chore: initial commit — import worldmodel workspace (plans/, research/)

2026-05-20 21:43:57 +08:00
commit bec8a9a4a3
98 changed files with 44128 additions and 0 deletions
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""
+fetch_readmes.py — 抓取每个项目的 README.md 头部 (前 12 KB)
+用于推断:支持的平台、依赖、是否提供数据集、demo 视频链接等。
+"""
+from __future__ import annotations
+import json
+import pathlib
+import re
+import sys
+import time
+import urllib.request
+
+ROOT = pathlib.Path(__file__).parent
+PROJECTS_JSON = ROOT / "projects.json"
+OUT = ROOT / "readmes"
+
+UA = "Mozilla/5.0 (research) PRISM-comparison/1.0"
+
+
+def get_text(url: str) -> str:
+    req = urllib.request.Request(url, headers={"User-Agent": UA})
+    try:
+        with urllib.request.urlopen(req, timeout=20) as r:
+            return r.read().decode("utf-8", errors="replace")
+    except Exception as e:
+        return f"<<ERROR {e}>>"
+
+
+def find_readme(repo: str, branch: str = "main") -> tuple[str, str]:
+    """返回 (raw_url, content)。先试 main,再 master。"""
+    for br in (branch, "master"):
+        url = f"https://raw.githubusercontent.com/{repo}/{br}/README.md"
+        txt = get_text(url)
+        if not txt.startswith("<<ERROR"):
+            return url, txt
+    return url, txt
+
+
+def main() -> None:
+    data = json.loads(PROJECTS_JSON.read_text(encoding="utf-8"))
+    OUT.mkdir(exist_ok=True)
+
+    for key, v in data.items():
+        repo = v["meta"]["gh"]
+        # 部分项目用了非 main 默认分支
+        default_branch = v.get("github", {}).get("default_branch") or "main"
+        sys.stderr.write(f"[readme] {v['meta']['name']} <- {repo}@{default_branch} ...\n")
+        url, txt = find_readme(repo, default_branch)
+        out_path = OUT / f"{key}.md"
+        out_path.write_text(f"<!-- src: {url} -->\n\n{txt[:16000]}\n",
+                            encoding="utf-8")
+        sys.stderr.write(f"  -> {out_path} ({len(txt)} chars)\n")
+        time.sleep(1.0)
+
+    sys.stderr.write(f"[ok] all readmes saved to {OUT}\n")
+
+
+if __name__ == "__main__":
+    main()