gaojie
61 lines
1.9 KiB
Python
61 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
fetch_readmes.py — 抓取每个项目的 README.md 头部 (前 12 KB)
|
|
用于推断:支持的平台、依赖、是否提供数据集、demo 视频链接等。
|
|
"""
|
|
from __future__ import annotations
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = pathlib.Path(__file__).parent
|
|
PROJECTS_JSON = ROOT / "projects.json"
|
|
OUT = ROOT / "readmes"
|
|
|
|
UA = "Mozilla/5.0 (research) PRISM-comparison/1.0"
|
|
|
|
|
|
def get_text(url: str) -> str:
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=20) as r:
|
|
return r.read().decode("utf-8", errors="replace")
|
|
except Exception as e:
|
|
return f"<<ERROR {e}>>"
|
|
|
|
|
|
def find_readme(repo: str, branch: str = "main") -> tuple[str, str]:
|
|
"""返回 (raw_url, content)。先试 main,再 master。"""
|
|
for br in (branch, "master"):
|
|
url = f"https://raw.githubusercontent.com/{repo}/{br}/README.md"
|
|
txt = get_text(url)
|
|
if not txt.startswith("<<ERROR"):
|
|
return url, txt
|
|
return url, txt
|
|
|
|
|
|
def main() -> None:
|
|
data = json.loads(PROJECTS_JSON.read_text(encoding="utf-8"))
|
|
OUT.mkdir(exist_ok=True)
|
|
|
|
for key, v in data.items():
|
|
repo = v["meta"]["gh"]
|
|
# 部分项目用了非 main 默认分支
|
|
default_branch = v.get("github", {}).get("default_branch") or "main"
|
|
sys.stderr.write(f"[readme] {v['meta']['name']} <- {repo}@{default_branch} ...\n")
|
|
url, txt = find_readme(repo, default_branch)
|
|
out_path = OUT / f"{key}.md"
|
|
out_path.write_text(f"<!-- src: {url} -->\n\n{txt[:16000]}\n",
|
|
encoding="utf-8")
|
|
sys.stderr.write(f" -> {out_path} ({len(txt)} chars)\n")
|
|
time.sleep(1.0)
|
|
|
|
sys.stderr.write(f"[ok] all readmes saved to {OUT}\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|