#!/usr/bin/env python3 """ gen_stats.py ============ 统计 PRISM 全书章节的: - 中文字数 / 英文词数 / 标点数 - 各语言代码块行数 - Mermaid 图数 / 表格数 / 标题层级数 生成 16_stats.md(LaTeX longtable 版),供 build_book.sh 注入到 PDF 末尾。 用法: python3 gen_stats.py """ from __future__ import annotations import pathlib import re import sys from collections import Counter, defaultdict CJK_RE = re.compile(r"[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef]") EN_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9_'-]*") PUNCT_RE = re.compile(r"[，。;；:：!！?？、（）()\[\]【】「」“”\"']") # 章节顺序(同 build_book.sh) ORDER = [ "README.md", "00_overview.md", "01_capability_decomposition.md", "02_architecture.md", "03_data_schema.md", "04_pipeline_A_iphone_offline.md", "05_pipeline_B_relocalization.md", "06_pipeline_C_online_perception.md", "07_pipeline_D_consolidation.md", "08_runtime_timeline.md", "09_roadmap.md", "10_tech_stack.md", "11_world_model_bridge.md", "12_risks.md", "13_evaluation.md", "14_mvp.md", "15_glossary.md", "18_lyra_inspirations.md", ] def analyze(text: str) -> dict: """对一篇 md 做综合统计。先把代码块抠掉,然后再统计正文。""" # 抠出代码块(``` 包围) code_blocks: list[tuple[str, int]] = [] # (lang, line_count) def _strip_code(m: re.Match) -> str: fence_info = m.group(1) or "" body = m.group(2) lang = fence_info.strip().split()[0] if fence_info.strip() else "text" # 行数 = 内容行数(不含闭合 ```) n_lines = body.count("\n") code_blocks.append((lang, n_lines)) return "\n[[CODE_REMOVED]]\n" body = re.sub(r"```([^\n]*)\n(.*?)\n```", _strip_code, text, flags=re.S) # 统计 mermaid 块(已被剥离,从 code_blocks 找) n_mermaid = sum(1 for lang, _ in code_blocks if lang == "mermaid") n_code_total = sum(n for lang, n in code_blocks if lang not in ("mermaid",)) # 各语言代码行数 lang_counter: Counter[str] = Counter() for lang, n in code_blocks: if lang == "mermaid": continue # 归一化 lang_norm = lang.lower() if lang else "text" if lang_norm in ("py", "python3"): lang_norm = "python" if lang_norm in ("sh", "shell", "zsh"): lang_norm = "bash" lang_counter[lang_norm] += n # 表格行(markdown |...| 风格) n_table_rows = sum(1 for ln in body.splitlines() if re.match(r"^\s*\|.+\|\s*$", ln)) # 标题 n_h1 = sum(1 for ln in body.splitlines() if ln.startswith("# ")) n_h2 = sum(1 for ln in body.splitlines() if ln.startswith("## ")) n_h3 = sum(1 for ln in body.splitlines() if ln.startswith("### ")) # 中英标点 cjk_chars = len(CJK_RE.findall(body)) en_words = len(EN_WORD_RE.findall(body)) punct = len(PUNCT_RE.findall(body)) return dict( cjk_chars=cjk_chars, en_words=en_words, punct=punct, n_mermaid=n_mermaid, n_code_blocks=len(code_blocks) - n_mermaid, n_code_lines=n_code_total, lang_counter=lang_counter, n_table_rows=n_table_rows, n_h1=n_h1, n_h2=n_h2, n_h3=n_h3, raw_lines=text.count("\n"), ) def fmt_int(n: int) -> str: return f"{n:,}" def main(prism_dir: pathlib.Path, out_md: pathlib.Path) -> None: total = defaultdict(int) total_lang: Counter[str] = Counter() per_chap: list[tuple[str, dict]] = [] for fn in ORDER: p = prism_dir / fn if not p.exists(): continue text = p.read_text(encoding="utf-8") s = analyze(text) per_chap.append((fn, s)) for k, v in s.items(): if isinstance(v, int): total[k] += v elif isinstance(v, Counter): total_lang.update(v) # 生成 markdown(内嵌 LaTeX) lines: list[str] = [] lines.append("# Chapter 16 — 文档与代码统计 (Stats)") lines.append("") lines.append("> 本章自动生成,反映本 PRISM 文档集在**构建时**的体量、结构与代码资产。每次 `bash build_book.sh` 都会刷新。") lines.append("") lines.append("## 16.1 总览") lines.append("") # ── 总览表 lines.append("\\begin{center}") lines.append("\\begin{tabular}{@{}lr@{}}") lines.append("\\toprule") lines.append("\\textbf{指标} & \\textbf{数值} \\\\") lines.append("\\midrule") lines.append(f"章节数 & {fmt_int(len(per_chap))} \\\\") lines.append(f"中文字数 & {fmt_int(total['cjk_chars'])} \\\\") lines.append(f"英文词数 & {fmt_int(total['en_words'])} \\\\") lines.append(f"标点数 & {fmt_int(total['punct'])} \\\\") lines.append(f"Markdown 行数 & {fmt_int(total['raw_lines'])} \\\\") lines.append(f"一级标题 (\\#) & {fmt_int(total['n_h1'])} \\\\") lines.append(f"二级标题 (\\#\\#) & {fmt_int(total['n_h2'])} \\\\") lines.append(f"三级标题 (\\#\\#\\#) & {fmt_int(total['n_h3'])} \\\\") lines.append(f"Markdown 表格行数 & {fmt_int(total['n_table_rows'])} \\\\") lines.append(f"Mermaid 图块数 & {fmt_int(total['n_mermaid'])} \\\\") lines.append(f"代码块数(非 mermaid) & {fmt_int(total['n_code_blocks'])} \\\\") lines.append(f"代码总行数 & {fmt_int(total['n_code_lines'])} \\\\") lines.append("\\bottomrule") lines.append("\\end{tabular}") lines.append("\\end{center}") lines.append("") # ── 各章字数/行数 用宽列章节名(28%)+ 6 个等宽数值列(各 10%) lines.append("## 16.2 各章字数与代码分布") lines.append("") lines.append("\\begin{longtable}{@{}") lines.append(" >{\\raggedright\\arraybackslash}p{0.28\\linewidth}") for _ in range(6): lines.append(" >{\\raggedleft\\arraybackslash}p{0.10\\linewidth}") lines.append("@{}}") lines.append("\\toprule") header = "\\textbf{章节} & \\textbf{中文字} & \\textbf{英文词} & \\textbf{md 行} & \\textbf{mmd 图} & \\textbf{代码块} & \\textbf{代码行} \\\\" lines.append(header) lines.append("\\midrule") lines.append("\\endfirsthead") lines.append("\\toprule") lines.append(header) lines.append("\\midrule") lines.append("\\endhead") lines.append("\\bottomrule") lines.append("\\endfoot") def short_label(fn: str) -> str: stem = fn.replace(".md", "") if stem == "README": return "README" m = re.match(r"(\d+)_", stem) if m: return f"ch{m.group(1)}" return stem.replace("_", "\\_") for fn, s in per_chap: lines.append( f"{short_label(fn)} & {fmt_int(s['cjk_chars'])} & {fmt_int(s['en_words'])} & " f"{fmt_int(s['raw_lines'])} & {s['n_mermaid']} & {s['n_code_blocks']} & " f"{fmt_int(s['n_code_lines'])} \\\\" ) lines.append( f"\\midrule\\textbf{{合计}} & \\textbf{{{fmt_int(total['cjk_chars'])}}} & " f"\\textbf{{{fmt_int(total['en_words'])}}} & \\textbf{{{fmt_int(total['raw_lines'])}}} & " f"\\textbf{{{total['n_mermaid']}}} & \\textbf{{{total['n_code_blocks']}}} & " f"\\textbf{{{fmt_int(total['n_code_lines'])}}} \\\\" ) lines.append("\\end{longtable}") lines.append("") lines.append("> 章节短名:`ch00` = `00\\_overview.md`,依此类推;`ch15` 缩略语表,`ch16` 即本章。") lines.append("") # ── 各语言代码行数 lines.append("## 16.3 代码块语言分布") lines.append("") lines.append("\\begin{center}") lines.append("\\begin{tabular}{@{}lrr@{}}") lines.append("\\toprule") lines.append("\\textbf{语言} & \\textbf{代码块数} & \\textbf{代码行数} \\\\") lines.append("\\midrule") lang_block_counter: Counter[str] = Counter() # 重算块数(分语言) for _, s in per_chap: # 这里需要拿原 lang_counter — 但 analyze() 没把"块数"按语言留下,只留了行数。 # 直接重新跑一遍代码块抽取(快速,正则一遍) pass # 重新统计每语言的"块数"(独立循环) lang_block_n: Counter[str] = Counter() for fn in ORDER: p = prism_dir / fn if not p.exists(): continue for m in re.finditer(r"```([^\n]*)\n(.*?)\n```", p.read_text(encoding="utf-8"), flags=re.S): lang = (m.group(1).strip() or "text").split()[0].lower() if lang in ("py", "python3"): lang = "python" if lang in ("sh", "shell", "zsh"): lang = "bash" if lang == "mermaid": continue lang_block_n[lang] += 1 for lang, n_lines in total_lang.most_common(): lang_safe = lang.replace("_", "\\_") n_blk = lang_block_n.get(lang, 0) lines.append(f"{lang_safe} & {n_blk} & {fmt_int(n_lines)} \\\\") lines.append("\\midrule") lines.append( f"\\textbf{{合计}} & \\textbf{{{sum(lang_block_n.values())}}} & " f"\\textbf{{{fmt_int(sum(total_lang.values()))}}} \\\\" ) lines.append("\\bottomrule") lines.append("\\end{tabular}") lines.append("\\end{center}") lines.append("") lines.append("---") lines.append("") lines.append("**章节版本**:auto-generated ") lines.append("**生成时间**:由 `gen_stats.py` 在每次构建时即时计算 ") lines.append("**关键收获**:文档不只是文字,它本身也是一份可量化的工程产物。") lines.append("") out_md.write_text("\n".join(lines), encoding="utf-8") print(f"[stats] wrote {out_md}") print(f" 中文 {total['cjk_chars']:,} 字, 英文 {total['en_words']:,} 词, " f"代码 {total['n_code_lines']:,} 行, mermaid {total['n_mermaid']} 图") if __name__ == "__main__": if len(sys.argv) != 3: sys.stderr.write("usage: gen_stats.py \n") sys.exit(1) main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))