vibe-coding-cn/scripts/check-research-raw.py

#!/usr/bin/env python3
"""Check research domains have local raw fact snapshots."""

from __future__ import annotations

import json
import subprocess
import sys
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
RESEARCH_DIR = ROOT / "research"
REQUIRED_RAW_FILES = [
    "README.md",
    "AGENTS.md",
    "sources.yml",
    "repository",
    "github-repo.raw.json",
    "github-readme.raw.md.txt",
    "github-root-contents.raw.json",
    "github-languages.raw.json",
]
REQUIRED_SOURCE_FIELDS = ["version: 1", "pulled_at:", "puller: scripts/fetch-research-raw.py", "files:"]
JSON_RAW_FILES = [
    "github-repo.raw.json",
    "github-root-contents.raw.json",
    "github-languages.raw.json",
]


def check_json(path: Path) -> str | None:
    try:
        json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError as error:
        rel = path.relative_to(ROOT)
        return f"{rel}:{error.lineno}:{error.colno}: invalid JSON raw file"
    return None


def check_git_worktree(path: Path) -> str | None:
    result = subprocess.run(
        ["git", "-C", str(path), "rev-parse", "--is-inside-work-tree"],
        cwd=ROOT,
        check=False,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    if result.returncode != 0 or result.stdout.strip() != "true":
        return f"{path.relative_to(ROOT)}: raw repository is not a valid Git working tree"
    return None


def main() -> int:
    errors: list[str] = []
    domain_files = sorted(RESEARCH_DIR.glob("*/domain.yml"))

    for domain_file in domain_files:
        domain_dir = domain_file.parent
        raw_dir = domain_dir / "raw"
        rel_raw = raw_dir.relative_to(ROOT)

        if not raw_dir.is_dir():
            errors.append(f"{rel_raw}: missing raw fact directory")
            continue

        for filename in REQUIRED_RAW_FILES:
            path = raw_dir / filename
            if filename == "repository":
                if not path.is_dir():
                    errors.append(f"{path.relative_to(ROOT)}: missing required raw repository directory")
                continue
            if not path.is_file():
                errors.append(f"{path.relative_to(ROOT)}: missing required raw fact file")

        sources = raw_dir / "sources.yml"
        if sources.is_file():
            text = sources.read_text(encoding="utf-8", errors="ignore")
            for field in REQUIRED_SOURCE_FIELDS:
                if field not in text:
                    errors.append(f"{sources.relative_to(ROOT)}: missing sources field '{field}'")
            if "path: repository/" not in text:
                errors.append(f"{sources.relative_to(ROOT)}: missing repository source record")

        raw_markdown = [
            path
            for path in raw_dir.glob("*.raw.md")
            if path.name not in {"README.md", "AGENTS.md"}
        ]
        for path in raw_markdown:
            errors.append(
                f"{path.relative_to(ROOT)}: raw external markdown must use '.raw.md.txt' to avoid local link checks"
            )

        for filename in JSON_RAW_FILES:
            path = raw_dir / filename
            if path.is_file():
                error = check_json(path)
                if error:
                    errors.append(error)

        repository = raw_dir / "repository"
        if repository.is_dir():
            error = check_git_worktree(repository)
            if error:
                errors.append(error)

    if errors:
        print("RESEARCH_RAW_ERRORS")
        for error in errors:
            print(error)
        print(f"TOTAL={len(errors)}")
        return 1

    print(f"OK research raw facts checked: {len(domain_files)} domains")
    return 0


if __name__ == "__main__":
    sys.exit(main())