#!/usr/bin/env python3 """ Translate Chinese comments/docstrings in Python files and Chinese prose in selected Markdown files into English. This intentionally skips: - generated frontend assets - dedicated localized docs such as *_CN.md / *_CH.md / *_TW.md - regular code string literals that are not docstrings, to avoid changing application behavior """ from __future__ import annotations import argparse import io import json import pathlib import re import sys import time import tokenize import urllib.parse import urllib.request from dataclasses import dataclass ROOT = pathlib.Path(__file__).resolve().parents[1] HAN_RE = re.compile(r"[\u3400-\u9FFF]") TRANSLATE_URL = ( "https://translate.googleapis.com/translate_a/single" "?client=gtx&sl=auto&tl=en&dt=t&q={query}" ) PYTHON_GLOBS = [ "backend_api_python/*.py", "backend_api_python/app/**/*.py", "scripts/*.py", ] MARKDOWN_TARGETS = [ "backend_api_python/README.md", "docs/FRONTEND_FAST_ANALYSIS.md", "docs/CHANGELOG.md", "docs/THIRD_PARTY_INTEGRATIONS.md", ] SKIP_PATH_PATTERNS = [ "/frontend/dist/", "/docs/README_CN.md", "_CN.md", "_CH.md", "_TW.md", "_JA.md", "_KO.md", ] @dataclass(order=True) class Replacement: start: int end: int text: str def contains_han(text: str) -> bool: return bool(HAN_RE.search(text)) _CACHE: dict[str, str] = {} def translate_text(text: str) -> str: text = text.strip("\n") if not text or not contains_han(text): return text if text in _CACHE: return _CACHE[text] url = TRANSLATE_URL.format(query=urllib.parse.quote(text)) last_error: Exception | None = None for attempt in range(3): try: with urllib.request.urlopen(url, timeout=20) as resp: payload = json.loads(resp.read().decode("utf-8")) translated = "".join(part[0] for part in payload[0] if part and part[0]) translated = normalize_translation(translated) _CACHE[text] = translated return translated except Exception as exc: # pragma: no cover - best effort script last_error = exc time.sleep(0.5 * (attempt + 1)) raise RuntimeError(f"Translation failed for text: {text!r}") from last_error def normalize_translation(text: str) -> str: text = text.replace("K line", "K-line") text = text.replace("K Line", "K-line") text = text.replace("candlestick line", "candlestick") text = text.replace("real-time market", "realtime market") text = text.replace("real-time quote", "realtime quote") return text def line_start_offsets(text: str) -> list[int]: starts = [0] for match in re.finditer(r"\n", text): starts.append(match.end()) return starts def to_offset(line_starts: list[int], lineno: int, col: int) -> int: return line_starts[lineno - 1] + col def translate_preserving_indent(text: str) -> str: lines = text.splitlines(keepends=True) out: list[str] = [] for line in lines: body = line.rstrip("\r\n") newline = line[len(body):] if not contains_han(body): out.append(line) continue leading = re.match(r"\s*", body).group(0) translated = translate_text(body[len(leading):]) out.append(f"{leading}{translated}{newline}") return "".join(out) DOCSTRING_ONE_LINE_RE = re.compile( r'^(?P\s*)(?P[rubfRUBF]*)(?P"""|\'\'\')(?P.*?)(?P=quote)(?P\s*(?:#.*)?)$' ) DOCSTRING_START_RE = re.compile( r'^(?P\s*)(?P[rubfRUBF]*)(?P"""|\'\'\')(?P.*)$' ) def translate_python_file(path: pathlib.Path) -> bool: src = path.read_text(encoding="utf-8") docstring_translated = translate_python_docstrings(src) line_starts = line_start_offsets(docstring_translated) replacements: list[Replacement] = [] for token in tokenize.generate_tokens(io.StringIO(docstring_translated).readline): if token.type != tokenize.COMMENT or not contains_han(token.string): continue leading = re.match(r"#\s*", token.string).group(0) translated = translate_text(token.string[len(leading):]) replacement = f"{leading}{translated}" replacements.append( Replacement( start=to_offset(line_starts, token.start[0], token.start[1]), end=to_offset(line_starts, token.end[0], token.end[1]), text=replacement, ) ) if not replacements and docstring_translated == src: return False new_src = apply_replacements(docstring_translated, replacements) if new_src != src: path.write_text(new_src, encoding="utf-8") return True return False def translate_python_docstrings(src: str) -> str: lines = src.splitlines(keepends=True) out: list[str] = [] in_docstring = False active_quote = "" for line in lines: stripped = line.lstrip() if not in_docstring: one_line = DOCSTRING_ONE_LINE_RE.match(line.rstrip("\n")) if one_line and stripped.startswith(one_line.group("prefix") + one_line.group("quote")): body = one_line.group("body") if contains_han(body): body = translate_text(body) newline = "\n" if line.endswith("\n") else "" out.append( f"{one_line.group('indent')}{one_line.group('prefix')}{one_line.group('quote')}" f"{body}{one_line.group('quote')}{one_line.group('trailing')}{newline}" ) continue start = DOCSTRING_START_RE.match(line.rstrip("\n")) if start and stripped.startswith(start.group("prefix") + start.group("quote")): in_docstring = True active_quote = start.group("quote") body = start.group("body") if contains_han(body): body = translate_text(body) newline = "\n" if line.endswith("\n") else "" out.append( f"{start.group('indent')}{start.group('prefix')}{start.group('quote')}{body}{newline}" ) continue out.append(line) continue line_no_newline = line.rstrip("\n") if active_quote in line_no_newline: before, after = line_no_newline.split(active_quote, 1) if contains_han(before): before = translate_preserving_indent(before).rstrip("\n") out.append(f"{before}{active_quote}{after}{'\n' if line.endswith('\n') else ''}") in_docstring = False active_quote = "" else: out.append(translate_preserving_indent(line)) return "".join(out) def translate_markdown_file(path: pathlib.Path) -> bool: src = path.read_text(encoding="utf-8") lines = src.splitlines(keepends=True) out: list[str] = [] in_fence = False changed = False for line in lines: stripped = line.strip() if stripped.startswith("```"): in_fence = not in_fence out.append(line) continue if in_fence or not contains_han(line): out.append(line) continue translated = translate_preserving_indent(line) out.append(translated) changed = changed or translated != line if changed: path.write_text("".join(out), encoding="utf-8") return changed def apply_replacements(text: str, replacements: list[Replacement]) -> str: dedup: list[Replacement] = [] seen: set[tuple[int, int]] = set() for item in sorted(replacements, reverse=True): key = (item.start, item.end) if key in seen: continue seen.add(key) dedup.append(item) for item in dedup: text = text[: item.start] + item.text + text[item.end :] return text def should_skip(path: pathlib.Path) -> bool: posix = path.as_posix() return any(pattern in posix for pattern in SKIP_PATH_PATTERNS) def iter_python_targets() -> list[pathlib.Path]: paths: set[pathlib.Path] = set() for pattern in PYTHON_GLOBS: paths.update(ROOT.glob(pattern)) return sorted(path for path in paths if path.is_file() and not should_skip(path)) def iter_markdown_targets() -> list[pathlib.Path]: return [ROOT / target for target in MARKDOWN_TARGETS if (ROOT / target).exists()] def resolve_targets(args: argparse.Namespace) -> tuple[list[pathlib.Path], list[pathlib.Path]]: if args.paths: python_files: list[pathlib.Path] = [] markdown_files: list[pathlib.Path] = [] for raw in args.paths: path = (ROOT / raw).resolve() if not pathlib.Path(raw).is_absolute() else pathlib.Path(raw) if not path.exists(): continue if path.suffix == ".py": python_files.append(path) elif path.suffix == ".md": markdown_files.append(path) return python_files, markdown_files return iter_python_targets(), iter_markdown_targets() def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("paths", nargs="*") parser.add_argument("--verbose", action="store_true") args = parser.parse_args() python_targets, markdown_targets = resolve_targets(args) changed_files: list[str] = [] for path in python_targets: if args.verbose: print(f"[py] {path.relative_to(ROOT).as_posix()}", file=sys.stderr) if translate_python_file(path): changed_files.append(path.relative_to(ROOT).as_posix()) for path in markdown_targets: if args.verbose: print(f"[md] {path.relative_to(ROOT).as_posix()}", file=sys.stderr) if translate_markdown_file(path): changed_files.append(path.relative_to(ROOT).as_posix()) for relpath in changed_files: print(relpath) return 0 if __name__ == "__main__": raise SystemExit(main())