process_docs.py

import re

CJK_RE = re.compile(r'[\u4e00-\u9fff]')
ASCII_RE = re.compile(r'[a-zA-Z]')
CJK_PUNCT_RE = re.compile(r'[。，、；：！？（）【】《》""''…]')

def cjk_count(text):
    return len(CJK_RE.findall(text))

def ascii_letter_count(text):
    return len(ASCII_RE.findall(text))

def english_word_count(text):
    return len(re.findall(r'[a-zA-Z]+', text))

def has_cjk_punct(text):
    return bool(CJK_PUNCT_RE.search(text))

def is_chinese_sentence(text):
    cjk = cjk_count(text)
    if cjk == 0:
        return False
    en_words = english_word_count(text)
    if has_cjk_punct(text):
        return True
    if cjk > en_words:
        return True
    return False

def split_slash(line):
    idx = line.find(' / ')
    if idx == -1:
        return None
    before = line[:idx]
    after = line[idx+3:]
    before_cjk = cjk_count(before)
    after_cjk = cjk_count(after)
    if after_cjk > 0 and before_cjk == 0:
        return (before, after)
    if before_cjk > 0 and after_cjk == 0:
        return (after, before)
    if before_cjk > 0 and after_cjk > 0:
        if english_word_count(before) > english_word_count(after):
            return (before, after)
        else:
            return (after, before)
    return None

def clean_en_line(line):
    s = split_slash(line)
    if s is not None:
        return s[0]
    m = re.match(r'^(.+?[.!?])\s+[\u4e00-\u9fff]', line)
    if m:
        return m.group(1)
    m = re.search(r'[\u4e00-\u9fff][。！？]\s+([A-Za-z].*)$', line)
    if m:
        return m.group(1)
    return line

def clean_zh_line(line):
    s = split_slash(line)
    if s is not None:
        return s[1]
    m = re.match(r'^([^\u4e00-\u9fff]*?[.!?])\s+([\u4e00-\u9fff])', line)
    if m:
        marker = re.match(r'^([\s>]*[-*]?\s*)', line)
        prefix = marker.group(1) if marker else ''
        return prefix + line[m.start(2):]
    m = re.search(r'[。！？]\s*[A-Za-z]', line)
    if m:
        return line[:m.start()+1]
    return line

def process_comment_en(indent, text):
    """Process a comment's text for en-US."""
    if cjk_count(text) == 0:
        return text
    s = split_slash(text)
    if s is not None:
        return s[0]
    if is_chinese_sentence(text):
        return None
    cleaned = clean_en_line(text)
    if cjk_count(cleaned) > 0:
        cleaned = re.sub(r'[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef，。、；：""''（）【】《》！？…]+', '', cleaned)
        cleaned = re.sub(r'  +', ' ', cleaned).strip()
    return cleaned if cleaned and ascii_letter_count(cleaned) > 0 else None

def process_comment_zh(text):
    if cjk_count(text) == 0:
        return None
    s = split_slash(text)
    if s is not None:
        return s[1]
    return text

def process_line_en(line, in_code_block):
    stripped = line.strip()
    if not stripped or stripped == '---':
        return line
    if stripped.startswith('```'):
        return line

    if in_code_block:
        # Find comment marker # (inline or full-line comment)
        hash_idx = line.find('#')
        if hash_idx != -1 and not re.match(r'^\s*#', line):
            # Inline comment: code # comment
            code_part = line[:hash_idx]
            comment_text = line[hash_idx+1:].strip()
            s = split_slash(comment_text)
            if s is not None:
                cleaned = s[0]
            elif cjk_count(comment_text) == 0:
                cleaned = comment_text
            elif is_chinese_sentence(comment_text):
                return code_part.rstrip()
            else:
                cleaned = clean_en_line(comment_text)
                if cjk_count(cleaned) > 0:
                    cleaned = re.sub(r'[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef，。、；：""''（）【】《》！？…]+', '', cleaned)
                    cleaned = re.sub(r'  +', ' ', cleaned).strip()
            if cleaned and ascii_letter_count(cleaned) > 0:
                return f'{code_part}# {cleaned}'
            return code_part.rstrip()
        # Full-line comment (# or //)
        cm = re.match(r'^(\s*)(#|//)\s*(.*)$', line)
        if cm:
            indent, cmarker, text = cm.group(1), cm.group(2), cm.group(3)
            if cjk_count(text) == 0:
                return line
            s = split_slash(text)
            if s is not None:
                return f'{indent}{cmarker} {s[0]}'
            if is_chinese_sentence(text):
                return None
            cleaned = clean_en_line(text)
            if cjk_count(cleaned) > 0:
                cleaned = re.sub(r'[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef，。、；：""''（）【】《》！？…]+', '', cleaned)
                cleaned = re.sub(r'  +', ' ', cleaned).strip()
            if cleaned and ascii_letter_count(cleaned) > 0:
                return f'{indent}{cmarker} {cleaned}'
            return None
        # No comment - try slash split on whole line (e.g. help text)
        s = split_slash(line)
        if s is not None:
            return s[0]
        # Handle translation key-value: "English" = "中文" -> "English" = "English"
        kv = re.match(r'^(\s*)"([^"]*)"\s*=\s*"([^"]*)"(\s*)$', line)
        if kv and cjk_count(kv.group(3)) > 0 and cjk_count(kv.group(2)) == 0:
            indent, key, val, tail = kv.group(1), kv.group(2), kv.group(3), kv.group(4)
            return f'{indent}"{key}" = "{key}"{tail}'
        return line

    # Outside code block
    # Heading: extract level and text, split only the text part
    if stripped.startswith('#'):
        hm = re.match(r'^(#{1,6})\s+(.*)$', line)
        if hm:
            level, text = hm.group(1), hm.group(2)
            s = split_slash(text)
            if s is not None:
                return f'{level} {s[0]}'
            if cjk_count(text) == 0:
                return line
            if is_chinese_sentence(text):
                return None
            cleaned = clean_en_line(text)
            return f'{level} {cleaned}' if cjk_count(cleaned) == 0 else None
        return line

    # Reference link list item with Chinese text -> remove for en-US
    link_match = re.match(r'^\s*[-*]\s*\[([^\]]*)\]', line)
    if link_match and cjk_count(link_match.group(1)) > 0:
        return None

    if cjk_count(line) == 0:
        return line

    # Try to extract English first (handles ' / ' separators)
    cleaned = clean_en_line(line)
    if cjk_count(cleaned) == 0:
        return cleaned

    # Still has CJK after cleaning - if primarily Chinese, remove
    if is_chinese_sentence(line):
        return None

    return cleaned

def process_line_zh(line, in_code_block):
    stripped = line.strip()
    if not stripped or stripped == '---':
        return line
    if stripped.startswith('```'):
        return line

    if in_code_block:
        # Find comment marker # (inline or full-line comment)
        hash_idx = line.find('#')
        if hash_idx != -1 and not re.match(r'^\s*#', line):
            code_part = line[:hash_idx]
            comment_text = line[hash_idx+1:].strip()
            s = split_slash(comment_text)
            if s is not None:
                cleaned = s[1]
            elif cjk_count(comment_text) == 0:
                return code_part.rstrip()  # pure English comment - remove
            else:
                cleaned = clean_zh_line(comment_text)
            if cleaned and cjk_count(cleaned) > 0:
                return f'{code_part}# {cleaned}'
            return code_part.rstrip()
        # Full-line comment (# or //)
        cm = re.match(r'^(\s*)(#|//)\s*(.*)$', line)
        if cm:
            indent, cmarker, text = cm.group(1), cm.group(2), cm.group(3)
            if cjk_count(text) == 0:
                return None
            s = split_slash(text)
            if s is not None:
                return f'{indent}{cmarker} {s[1]}'
            cleaned = clean_zh_line(text)
            if cjk_count(cleaned) > 0:
                return f'{indent}{cmarker} {cleaned}'
            return line
        # No comment - try slash split on whole line
        s = split_slash(line)
        if s is not None:
            return s[1]
        return line

    # Outside code block
    if stripped.startswith('#'):
        hm = re.match(r'^(#{1,6})\s+(.*)$', line)
        if hm:
            level, text = hm.group(1), hm.group(2)
            s = split_slash(text)
            if s is not None:
                return f'{level} {s[1]}'
            # No slash separator - keep heading as-is (may be filename/code)
            return line
        return line

    if cjk_count(line) == 0:
        if re.match(r'^\s*[-*]\s*\[', line) or stripped.startswith('http'):
            return line
        return None

    # Has Chinese - clean any leading/trailing English
    if ascii_letter_count(line) > 0:
        cleaned = clean_zh_line(line)
        if cjk_count(cleaned) > 0:
            return cleaned
    return line

def process_file(filepath, lang):
    with open(filepath, 'r', encoding='utf-8') as f:
        lines = f.readlines()

    processed = []
    in_code_block = False
    for line in lines:
        line = line.rstrip('\n')
        stripped = line.strip()
        if stripped.startswith('```'):
            in_code_block = not in_code_block
            processed.append(line)
            continue
        if lang == 'en':
            result = process_line_en(line, in_code_block)
        else:
            result = process_line_zh(line, in_code_block)
        if result is not None:
            processed.append(result)

    final = []
    prev_blank = False
    for l in processed:
        s = l.strip()
        if s == '>':
            continue
        if not s:
            if prev_blank:
                continue
            prev_blank = True
        else:
            prev_blank = False
        final.append(l)
    while final and not final[-1].strip():
        final.pop()

    with open(filepath, 'w', encoding='utf-8') as f:
        f.write('\n'.join(final) + '\n')
    return len(final)

import sys
import shutil
base = r'e:\Github_desktop\bomiot_example\greaterwms\media'
backup = r'e:\Github_desktop\bomiot_example\_backup_media'
names = [
    'django', 'fastapi_app', 'flask_app', 'bomiotconf', 'terminal',
    'interaction', 'server', 'scheduler', 'observer', 'data',
    'structure', 'permission', 'mysql', 'postgresql', 'inscription'
]
for name in names:
    shutil.copy2(f'{backup}\\{name}.en-US.md', f'{base}\\{name}.en-US.md')
    shutil.copy2(f'{backup}\\{name}.zh-CN.md', f'{base}\\{name}.zh-CN.md')
    en = process_file(f'{base}\\{name}.en-US.md', 'en')
    zh = process_file(f'{base}\\{name}.zh-CN.md', 'zh')
    print(f'{name}: EN={en} ZH={zh}')
print("All done")