gaojie
272 lines
10 KiB
Python
272 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
gen_stats.py
|
||
============
|
||
统计 PRISM 全书章节的:
|
||
- 中文字数 / 英文词数 / 标点数
|
||
- 各语言代码块行数
|
||
- Mermaid 图数 / 表格数 / 标题层级数
|
||
|
||
生成 16_stats.md(LaTeX longtable 版),供 build_book.sh 注入到 PDF 末尾。
|
||
|
||
用法:
|
||
python3 gen_stats.py <prism_dir> <out_md>
|
||
"""
|
||
from __future__ import annotations
|
||
import pathlib
|
||
import re
|
||
import sys
|
||
from collections import Counter, defaultdict
|
||
|
||
CJK_RE = re.compile(r"[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef]")
|
||
EN_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9_'-]*")
|
||
PUNCT_RE = re.compile(r"[，。;；:：!！?？、（）()\[\]【】「」“”\"']")
|
||
|
||
# 章节顺序(同 build_book.sh)
|
||
ORDER = [
|
||
"README.md",
|
||
"00_overview.md",
|
||
"01_capability_decomposition.md",
|
||
"02_architecture.md",
|
||
"03_data_schema.md",
|
||
"04_pipeline_A_iphone_offline.md",
|
||
"05_pipeline_B_relocalization.md",
|
||
"06_pipeline_C_online_perception.md",
|
||
"07_pipeline_D_consolidation.md",
|
||
"08_runtime_timeline.md",
|
||
"09_roadmap.md",
|
||
"10_tech_stack.md",
|
||
"11_world_model_bridge.md",
|
||
"12_risks.md",
|
||
"13_evaluation.md",
|
||
"14_mvp.md",
|
||
"15_glossary.md",
|
||
"18_lyra_inspirations.md",
|
||
]
|
||
|
||
|
||
def analyze(text: str) -> dict:
|
||
"""对一篇 md 做综合统计。先把代码块抠掉,然后再统计正文。"""
|
||
# 抠出代码块(``` 包围)
|
||
code_blocks: list[tuple[str, int]] = [] # (lang, line_count)
|
||
def _strip_code(m: re.Match) -> str:
|
||
fence_info = m.group(1) or ""
|
||
body = m.group(2)
|
||
lang = fence_info.strip().split()[0] if fence_info.strip() else "text"
|
||
# 行数 = 内容行数(不含闭合 ```)
|
||
n_lines = body.count("\n")
|
||
code_blocks.append((lang, n_lines))
|
||
return "\n[[CODE_REMOVED]]\n"
|
||
|
||
body = re.sub(r"```([^\n]*)\n(.*?)\n```", _strip_code, text, flags=re.S)
|
||
|
||
# 统计 mermaid 块(已被剥离,从 code_blocks 找)
|
||
n_mermaid = sum(1 for lang, _ in code_blocks if lang == "mermaid")
|
||
n_code_total = sum(n for lang, n in code_blocks if lang not in ("mermaid",))
|
||
|
||
# 各语言代码行数
|
||
lang_counter: Counter[str] = Counter()
|
||
for lang, n in code_blocks:
|
||
if lang == "mermaid":
|
||
continue
|
||
# 归一化
|
||
lang_norm = lang.lower() if lang else "text"
|
||
if lang_norm in ("py", "python3"):
|
||
lang_norm = "python"
|
||
if lang_norm in ("sh", "shell", "zsh"):
|
||
lang_norm = "bash"
|
||
lang_counter[lang_norm] += n
|
||
|
||
# 表格行(markdown |...| 风格)
|
||
n_table_rows = sum(1 for ln in body.splitlines()
|
||
if re.match(r"^\s*\|.+\|\s*$", ln))
|
||
|
||
# 标题
|
||
n_h1 = sum(1 for ln in body.splitlines() if ln.startswith("# "))
|
||
n_h2 = sum(1 for ln in body.splitlines() if ln.startswith("## "))
|
||
n_h3 = sum(1 for ln in body.splitlines() if ln.startswith("### "))
|
||
|
||
# 中英标点
|
||
cjk_chars = len(CJK_RE.findall(body))
|
||
en_words = len(EN_WORD_RE.findall(body))
|
||
punct = len(PUNCT_RE.findall(body))
|
||
|
||
return dict(
|
||
cjk_chars=cjk_chars,
|
||
en_words=en_words,
|
||
punct=punct,
|
||
n_mermaid=n_mermaid,
|
||
n_code_blocks=len(code_blocks) - n_mermaid,
|
||
n_code_lines=n_code_total,
|
||
lang_counter=lang_counter,
|
||
n_table_rows=n_table_rows,
|
||
n_h1=n_h1,
|
||
n_h2=n_h2,
|
||
n_h3=n_h3,
|
||
raw_lines=text.count("\n"),
|
||
)
|
||
|
||
|
||
def fmt_int(n: int) -> str:
|
||
return f"{n:,}"
|
||
|
||
|
||
def main(prism_dir: pathlib.Path, out_md: pathlib.Path) -> None:
|
||
total = defaultdict(int)
|
||
total_lang: Counter[str] = Counter()
|
||
per_chap: list[tuple[str, dict]] = []
|
||
|
||
for fn in ORDER:
|
||
p = prism_dir / fn
|
||
if not p.exists():
|
||
continue
|
||
text = p.read_text(encoding="utf-8")
|
||
s = analyze(text)
|
||
per_chap.append((fn, s))
|
||
for k, v in s.items():
|
||
if isinstance(v, int):
|
||
total[k] += v
|
||
elif isinstance(v, Counter):
|
||
total_lang.update(v)
|
||
|
||
# 生成 markdown(内嵌 LaTeX)
|
||
lines: list[str] = []
|
||
lines.append("# Chapter 16 — 文档与代码统计 (Stats)")
|
||
lines.append("")
|
||
lines.append("> 本章自动生成,反映本 PRISM 文档集在**构建时**的体量、结构与代码资产。每次 `bash build_book.sh` 都会刷新。")
|
||
lines.append("")
|
||
lines.append("## 16.1 总览")
|
||
lines.append("")
|
||
# ── 总览表
|
||
lines.append("\\begin{center}")
|
||
lines.append("\\begin{tabular}{@{}lr@{}}")
|
||
lines.append("\\toprule")
|
||
lines.append("\\textbf{指标} & \\textbf{数值} \\\\")
|
||
lines.append("\\midrule")
|
||
lines.append(f"章节数 & {fmt_int(len(per_chap))} \\\\")
|
||
lines.append(f"中文字数 & {fmt_int(total['cjk_chars'])} \\\\")
|
||
lines.append(f"英文词数 & {fmt_int(total['en_words'])} \\\\")
|
||
lines.append(f"标点数 & {fmt_int(total['punct'])} \\\\")
|
||
lines.append(f"Markdown 行数 & {fmt_int(total['raw_lines'])} \\\\")
|
||
lines.append(f"一级标题 (\\#) & {fmt_int(total['n_h1'])} \\\\")
|
||
lines.append(f"二级标题 (\\#\\#) & {fmt_int(total['n_h2'])} \\\\")
|
||
lines.append(f"三级标题 (\\#\\#\\#) & {fmt_int(total['n_h3'])} \\\\")
|
||
lines.append(f"Markdown 表格行数 & {fmt_int(total['n_table_rows'])} \\\\")
|
||
lines.append(f"Mermaid 图块数 & {fmt_int(total['n_mermaid'])} \\\\")
|
||
lines.append(f"代码块数(非 mermaid) & {fmt_int(total['n_code_blocks'])} \\\\")
|
||
lines.append(f"代码总行数 & {fmt_int(total['n_code_lines'])} \\\\")
|
||
lines.append("\\bottomrule")
|
||
lines.append("\\end{tabular}")
|
||
lines.append("\\end{center}")
|
||
lines.append("")
|
||
|
||
# ── 各章字数/行数 用宽列章节名(28%)+ 6 个等宽数值列(各 10%)
|
||
lines.append("## 16.2 各章字数与代码分布")
|
||
lines.append("")
|
||
lines.append("\\begin{longtable}{@{}")
|
||
lines.append(" >{\\raggedright\\arraybackslash}p{0.28\\linewidth}")
|
||
for _ in range(6):
|
||
lines.append(" >{\\raggedleft\\arraybackslash}p{0.10\\linewidth}")
|
||
lines.append("@{}}")
|
||
lines.append("\\toprule")
|
||
header = "\\textbf{章节} & \\textbf{中文字} & \\textbf{英文词} & \\textbf{md 行} & \\textbf{mmd 图} & \\textbf{代码块} & \\textbf{代码行} \\\\"
|
||
lines.append(header)
|
||
lines.append("\\midrule")
|
||
lines.append("\\endfirsthead")
|
||
lines.append("\\toprule")
|
||
lines.append(header)
|
||
lines.append("\\midrule")
|
||
lines.append("\\endhead")
|
||
lines.append("\\bottomrule")
|
||
lines.append("\\endfoot")
|
||
|
||
def short_label(fn: str) -> str:
|
||
stem = fn.replace(".md", "")
|
||
if stem == "README":
|
||
return "README"
|
||
m = re.match(r"(\d+)_", stem)
|
||
if m:
|
||
return f"ch{m.group(1)}"
|
||
return stem.replace("_", "\\_")
|
||
|
||
for fn, s in per_chap:
|
||
lines.append(
|
||
f"{short_label(fn)} & {fmt_int(s['cjk_chars'])} & {fmt_int(s['en_words'])} & "
|
||
f"{fmt_int(s['raw_lines'])} & {s['n_mermaid']} & {s['n_code_blocks']} & "
|
||
f"{fmt_int(s['n_code_lines'])} \\\\"
|
||
)
|
||
lines.append(
|
||
f"\\midrule\\textbf{{合计}} & \\textbf{{{fmt_int(total['cjk_chars'])}}} & "
|
||
f"\\textbf{{{fmt_int(total['en_words'])}}} & \\textbf{{{fmt_int(total['raw_lines'])}}} & "
|
||
f"\\textbf{{{total['n_mermaid']}}} & \\textbf{{{total['n_code_blocks']}}} & "
|
||
f"\\textbf{{{fmt_int(total['n_code_lines'])}}} \\\\"
|
||
)
|
||
lines.append("\\end{longtable}")
|
||
lines.append("")
|
||
lines.append("> 章节短名:`ch00` = `00\\_overview.md`,依此类推;`ch15` 缩略语表,`ch16` 即本章。")
|
||
lines.append("")
|
||
|
||
# ── 各语言代码行数
|
||
lines.append("## 16.3 代码块语言分布")
|
||
lines.append("")
|
||
lines.append("\\begin{center}")
|
||
lines.append("\\begin{tabular}{@{}lrr@{}}")
|
||
lines.append("\\toprule")
|
||
lines.append("\\textbf{语言} & \\textbf{代码块数} & \\textbf{代码行数} \\\\")
|
||
lines.append("\\midrule")
|
||
lang_block_counter: Counter[str] = Counter()
|
||
# 重算块数(分语言)
|
||
for _, s in per_chap:
|
||
# 这里需要拿原 lang_counter — 但 analyze() 没把"块数"按语言留下,只留了行数。
|
||
# 直接重新跑一遍代码块抽取(快速,正则一遍)
|
||
pass
|
||
|
||
# 重新统计每语言的"块数"(独立循环)
|
||
lang_block_n: Counter[str] = Counter()
|
||
for fn in ORDER:
|
||
p = prism_dir / fn
|
||
if not p.exists():
|
||
continue
|
||
for m in re.finditer(r"```([^\n]*)\n(.*?)\n```", p.read_text(encoding="utf-8"), flags=re.S):
|
||
lang = (m.group(1).strip() or "text").split()[0].lower()
|
||
if lang in ("py", "python3"):
|
||
lang = "python"
|
||
if lang in ("sh", "shell", "zsh"):
|
||
lang = "bash"
|
||
if lang == "mermaid":
|
||
continue
|
||
lang_block_n[lang] += 1
|
||
|
||
for lang, n_lines in total_lang.most_common():
|
||
lang_safe = lang.replace("_", "\\_")
|
||
n_blk = lang_block_n.get(lang, 0)
|
||
lines.append(f"{lang_safe} & {n_blk} & {fmt_int(n_lines)} \\\\")
|
||
lines.append("\\midrule")
|
||
lines.append(
|
||
f"\\textbf{{合计}} & \\textbf{{{sum(lang_block_n.values())}}} & "
|
||
f"\\textbf{{{fmt_int(sum(total_lang.values()))}}} \\\\"
|
||
)
|
||
lines.append("\\bottomrule")
|
||
lines.append("\\end{tabular}")
|
||
lines.append("\\end{center}")
|
||
lines.append("")
|
||
|
||
lines.append("---")
|
||
lines.append("")
|
||
lines.append("**章节版本**:auto-generated ")
|
||
lines.append("**生成时间**:由 `gen_stats.py` 在每次构建时即时计算 ")
|
||
lines.append("**关键收获**:文档不只是文字,它本身也是一份可量化的工程产物。")
|
||
lines.append("")
|
||
|
||
out_md.write_text("\n".join(lines), encoding="utf-8")
|
||
print(f"[stats] wrote {out_md}")
|
||
print(f" 中文 {total['cjk_chars']:,} 字, 英文 {total['en_words']:,} 词, "
|
||
f"代码 {total['n_code_lines']:,} 行, mermaid {total['n_mermaid']} 图")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
if len(sys.argv) != 3:
|
||
sys.stderr.write("usage: gen_stats.py <prism_dir> <out_md>\n")
|
||
sys.exit(1)
|
||
main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))
|