#!/usr/bin/env python3
"""
fetch_readmes.py — 抓取每个项目的 README.md 头部 (前 12 KB)
用于推断:支持的平台、依赖、是否提供数据集、demo 视频链接等。
"""
from __future__ import annotations
import json
import pathlib
import re
import sys
import time
import urllib.request

ROOT = pathlib.Path(__file__).parent
PROJECTS_JSON = ROOT / "projects.json"
OUT = ROOT / "readmes"

UA = "Mozilla/5.0 (research) PRISM-comparison/1.0"


def get_text(url: str) -> str:
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    try:
        with urllib.request.urlopen(req, timeout=20) as r:
            return r.read().decode("utf-8", errors="replace")
    except Exception as e:
        return f"<<ERROR {e}>>"


def find_readme(repo: str, branch: str = "main") -> tuple[str, str]:
    """返回 (raw_url, content)。先试 main,再 master。"""
    for br in (branch, "master"):
        url = f"https://raw.githubusercontent.com/{repo}/{br}/README.md"
        txt = get_text(url)
        if not txt.startswith("<<ERROR"):
            return url, txt
    return url, txt


def main() -> None:
    data = json.loads(PROJECTS_JSON.read_text(encoding="utf-8"))
    OUT.mkdir(exist_ok=True)

    for key, v in data.items():
        repo = v["meta"]["gh"]
        # 部分项目用了非 main 默认分支
        default_branch = v.get("github", {}).get("default_branch") or "main"
        sys.stderr.write(f"[readme] {v['meta']['name']} <- {repo}@{default_branch} ...\n")
        url, txt = find_readme(repo, default_branch)
        out_path = OUT / f"{key}.md"
        out_path.write_text(f"<!-- src: {url} -->\n\n{txt[:16000]}\n",
                            encoding="utf-8")
        sys.stderr.write(f"  -> {out_path} ({len(txt)} chars)\n")
        time.sleep(1.0)

    sys.stderr.write(f"[ok] all readmes saved to {OUT}\n")


if __name__ == "__main__":
    main()