gaojie/worldmodel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: initial commit — import worldmodel workspace (plans/, research/)

Browse Source
This commit is contained in:
gaojie
2026-05-20 21:43:57 +08:00
commit bec8a9a4a3
98 changed files with 44128 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plans/PRISM/.research/arxiv_data.json
+162
View File
@@ -0,0 +1,162 @@
{
"concept-graphs": {
"title": "ConceptGraphs: Open-Vocabulary 3D Scene Graphs for Perception and Planning",
"summary": "For robots to perform a wide variety of tasks, they require a 3D representation of the world that is semantically rich, yet compact and efficient for task-driven perception and planning. Recent approaches have attempted to leverage features from large vision-language models to encode semantics in 3D representations. However, these approaches tend to produce maps with per-point feature vectors, which do not scale well in larger environments, nor do they contain semantic spatial relationships between entities in the environment, which are useful for downstream planning. In this work, we propose ConceptGraphs, an open-vocabulary graph-structured representation for 3D scenes. ConceptGraphs is built by leveraging 2D foundation models and fusing their output to 3D by multi-view association. The resulting representations generalize to novel semantic classes, without the need to collect large 3D datasets or finetune models. We demonstrate the utility of this representation through a number of downstream planning tasks that are specified through abstract (language) prompts and require complex reasoning over spatial and semantic concepts. (Project page: https://concept-graphs.github.io/ Explainer video: https://youtu.be/mRhNkQwRYnc )",
"published": "2023-09-28T17:53:38Z",
"updated": "2023-09-28T17:53:38Z",
"authors": [
"Qiao Gu",
"Alihusein Kuwajerwala",
"Sacha Morin",
"Krishna Murthy Jatavallabhula",
"Bipasha Sen",
"Aditya Agarwal",
"Corban Rivera",
"William Paul",
"Kirsty Ellis",
"Rama Chellappa",
"Chuang Gan",
"Celso Miguel de Melo",
"Joshua B. Tenenbaum",
"Antonio Torralba",
"Florian Shkurti",
"Liam Paull"
],
"primary_cat": "cs.RO",
"journal_ref": "",
"doi": ""
},
"hov-sg": {
"title": "Hierarchical Open-Vocabulary 3D Scene Graphs for Language-Grounded Robot Navigation",
"summary": "Recent open-vocabulary robot mapping methods enrich dense geometric maps with pre-trained visual-language features. While these maps allow for the prediction of point-wise saliency maps when queried for a certain language concept, large-scale environments and abstract queries beyond the object level still pose a considerable hurdle, ultimately limiting language-grounded robotic navigation. In this work, we present HOV-SG, a hierarchical open-vocabulary 3D scene graph mapping approach for language-grounded robot navigation. Leveraging open-vocabulary vision foundation models, we first obtain state-of-the-art open-vocabulary segment-level maps in 3D and subsequently construct a 3D scene graph hierarchy consisting of floor, room, and object concepts, each enriched with open-vocabulary features. Our approach is able to represent multi-story buildings and allows robotic traversal of those using a cross-floor Voronoi graph. HOV-SG is evaluated on three distinct datasets and surpasses previous baselines in open-vocabulary semantic accuracy on the object, room, and floor level while producing a 75% reduction in representation size compared to dense open-vocabulary maps. In order to prove the efficacy and generalization capabilities of HOV-SG, we showcase successful long-horizon language-conditioned robot navigation within real-world multi-storage environments. We provide code and trial video data at http://hovsg.github.io/.",
"published": "2024-03-26T16:36:43Z",
"updated": "2024-06-03T17:12:25Z",
"authors": [
"Abdelrhman Werby",
"Chenguang Huang",
"Martin Büchner",
"Abhinav Valada",
"Wolfram Burgard"
],
"primary_cat": "cs.RO",
"journal_ref": "",
"doi": "10.15607/RSS.2024.XX.077"
},
"openscene": {
"title": "OpenScene: 3D Scene Understanding with Open Vocabularies",
"summary": "Traditional 3D scene understanding approaches rely on labeled 3D datasets to train a model for a single task with supervision. We propose OpenScene, an alternative approach where a model predicts dense features for 3D scene points that are co-embedded with text and image pixels in CLIP feature space. This zero-shot approach enables task-agnostic training and open-vocabulary queries. For example, to perform SOTA zero-shot 3D semantic segmentation it first infers CLIP features for every 3D point and later classifies them based on similarities to embeddings of arbitrary class labels. More interestingly, it enables a suite of open-vocabulary scene understanding applications that have never been done before. For example, it allows a user to enter an arbitrary text query and then see a heat map indicating which parts of a scene match. Our approach is effective at identifying objects, materials, affordances, activities, and room types in complex 3D scenes, all using a single model trained without any labeled 3D data.",
"published": "2022-11-28T18:58:36Z",
"updated": "2023-04-06T15:35:13Z",
"authors": [
"Songyou Peng",
"Kyle Genova",
"Chiyu \"Max\" Jiang",
"Andrea Tagliasacchi",
"Marc Pollefeys",
"Thomas Funkhouser"
],
"primary_cat": "cs.CV",
"journal_ref": "",
"doi": ""
},
"3d-llm": {
"title": "3D-LLM: Injecting the 3D World into Large Language Models",
"summary": "Large language models (LLMs) and Vision-Language Models (VLMs) have been proven to excel at multiple tasks, such as commonsense reasoning. Powerful as these models can be, they are not grounded in the 3D physical world, which involves richer concepts such as spatial relationships, affordances, physics, layout, and so on. In this work, we propose to inject the 3D world into large language models and introduce a whole new family of 3D-LLMs. Specifically, 3D-LLMs can take 3D point clouds and their features as input and perform a diverse set of 3D-related tasks, including captioning, dense captioning, 3D question answering, task decomposition, 3D grounding, 3D-assisted dialog, navigation, and so on. Using three types of prompting mechanisms that we design, we are able to collect over 300k 3D-language data covering these tasks. To efficiently train 3D-LLMs, we first utilize a 3D feature extractor that obtains 3D features from rendered multi- view images. Then, we use 2D VLMs as our backbones to train our 3D-LLMs. By introducing a 3D localization mechanism, 3D-LLMs can better capture 3D spatial information. Experiments on ScanQA show that our model outperforms state-of-the-art baselines by a large margin (e.g., the BLEU-1 score surpasses state-of-the-art score by 9%). Furthermore, experiments on our held-in datasets for 3D captioning, task composition, and 3D-assisted dialogue show that our model outperforms 2D VLMs. Qualitative examples also show that our model could perform more tasks beyond the scope of existing LLMs and VLMs. Project Page: : https://vis-www.cs.umass.edu/3dllm/.",
"published": "2023-07-24T17:59:02Z",
"updated": "2023-07-24T17:59:02Z",
"authors": [
"Yining Hong",
"Haoyu Zhen",
"Peihao Chen",
"Shuhong Zheng",
"Yilun Du",
"Zhenfang Chen",
"Chuang Gan"
],
"primary_cat": "cs.CV",
"journal_ref": "",
"doi": ""
},
"ok-robot": {
"title": "OK-Robot: What Really Matters in Integrating Open-Knowledge Models for Robotics",
"summary": "Remarkable progress has been made in recent years in the fields of vision, language, and robotics. We now have vision models capable of recognizing objects based on language queries, navigation systems that can effectively control mobile systems, and grasping models that can handle a wide range of objects. Despite these advancements, general-purpose applications of robotics still lag behind, even though they rely on these fundamental capabilities of recognition, navigation, and grasping. In this paper, we adopt a systems-first approach to develop a new Open Knowledge-based robotics framework called OK-Robot. By combining Vision-Language Models (VLMs) for object detection, navigation primitives for movement, and grasping primitives for object manipulation, OK-Robot offers a integrated solution for pick-and-drop operations without requiring any training. To evaluate its performance, we run OK-Robot in 10 real-world home environments. The results demonstrate that OK-Robot achieves a 58.5% success rate in open-ended pick-and-drop tasks, representing a new state-of-the-art in Open Vocabulary Mobile Manipulation (OVMM) with nearly 1.8x the performance of prior work. On cleaner, uncluttered environments, OK-Robot's performance increases to 82%. However, the most important insight gained from OK-Robot is the critical role of nuanced details when combining Open Knowledge systems like VLMs with robotic modules. Videos of our experiments and code are available on our website: https://ok-robot.github.io",
"published": "2024-01-22T18:42:20Z",
"updated": "2024-02-29T17:20:08Z",
"authors": [
"Peiqi Liu",
"Yaswanth Orru",
"Jay Vakil",
"Chris Paxton",
"Nur Muhammad Mahi Shafiullah",
"Lerrel Pinto"
],
"primary_cat": "cs.RO",
"journal_ref": "",
"doi": "10.15607/RSS.2024.XX.091"
},
"openmask3d": {
"title": "OpenMask3D: Open-Vocabulary 3D Instance Segmentation",
"summary": "We introduce the task of open-vocabulary 3D instance segmentation. Current approaches for 3D instance segmentation can typically only recognize object categories from a pre-defined closed set of classes that are annotated in the training datasets. This results in important limitations for real-world applications where one might need to perform tasks guided by novel, open-vocabulary queries related to a wide variety of objects. Recently, open-vocabulary 3D scene understanding methods have emerged to address this problem by learning queryable features for each point in the scene. While such a representation can be directly employed to perform semantic segmentation, existing methods cannot separate multiple object instances. In this work, we address this limitation, and propose OpenMask3D, which is a zero-shot approach for open-vocabulary 3D instance segmentation. Guided by predicted class-agnostic 3D instance masks, our model aggregates per-mask features via multi-view fusion of CLIP-based image embeddings. Experiments and ablation studies on ScanNet200 and Replica show that OpenMask3D outperforms other open-vocabulary methods, especially on the long-tail distribution. Qualitative experiments further showcase OpenMask3D's ability to segment object properties based on free-form queries describing geometry, affordances, and materials.",
"published": "2023-06-23T17:36:44Z",
"updated": "2023-10-29T14:04:25Z",
"authors": [
"Ayça Takmaz",
"Elisabetta Fedele",
"Robert W. Sumner",
"Marc Pollefeys",
"Federico Tombari",
"Francis Engelmann"
],
"primary_cat": "cs.CV",
"journal_ref": "NeurIPS 2023",
"doi": ""
},
"conceptfusion": {
"title": "ConceptFusion: Open-set Multimodal 3D Mapping",
"summary": "Building 3D maps of the environment is central to robot navigation, planning, and interaction with objects in a scene. Most existing approaches that integrate semantic concepts with 3D maps largely remain confined to the closed-set setting: they can only reason about a finite set of concepts, pre-defined at training time. Further, these maps can only be queried using class labels, or in recent work, using text prompts. We address both these issues with ConceptFusion, a scene representation that is (1) fundamentally open-set, enabling reasoning beyond a closed set of concepts and (ii) inherently multimodal, enabling a diverse range of possible queries to the 3D map, from language, to images, to audio, to 3D geometry, all working in concert. ConceptFusion leverages the open-set capabilities of today's foundation models pre-trained on internet-scale data to reason about concepts across modalities such as natural language, images, and audio. We demonstrate that pixel-aligned open-set features can be fused into 3D maps via traditional SLAM and multi-view fusion approaches. This enables effective zero-shot spatial reasoning, not needing any additional training or finetuning, and retains long-tailed concepts better than supervised approaches, outperforming them by more than 40% margin on 3D IoU. We extensively evaluate ConceptFusion on a number of real-world datasets, simulated home environments, a real-world tabletop manipulation task, and an autonomous driving platform. We showcase new avenues for blending foundation models with 3D open-set multimodal mapping. For more information, visit our project page https://concept-fusion.github.io or watch our 5-minute explainer video https://www.youtube.com/watch?v=rkXgws8fiDs",
"published": "2023-02-14T18:40:26Z",
"updated": "2023-10-23T14:56:15Z",
"authors": [
"Krishna Murthy Jatavallabhula",
"Alihusein Kuwajerwala",
"Qiao Gu",
"Mohd Omama",
"Tao Chen",
"Alaa Maalouf",
"Shuang Li",
"Ganesh Iyer",
"Soroush Saryazdi",
"Nikhil Keetha",
"Ayush Tewari",
"Joshua B. Tenenbaum",
"Celso Miguel de Melo",
"Madhava Krishna",
"Liam Paull",
"Florian Shkurti",
"Antonio Torralba"
],
"primary_cat": "cs.CV",
"journal_ref": "",
"doi": ""
},
"clio": {
"title": "Clio: Real-time Task-Driven Open-Set 3D Scene Graphs",
"summary": "Modern tools for class-agnostic image segmentation (e.g., SegmentAnything) and open-set semantic understanding (e.g., CLIP) provide unprecedented opportunities for robot perception and mapping. While traditional closed-set metric-semantic maps were restricted to tens or hundreds of semantic classes, we can now build maps with a plethora of objects and countless semantic variations. This leaves us with a fundamental question: what is the right granularity for the objects (and, more generally, for the semantic concepts) the robot has to include in its map representation? While related work implicitly chooses a level of granularity by tuning thresholds for object detection, we argue that such a choice is intrinsically task-dependent. The first contribution of this paper is to propose a task-driven 3D scene understanding problem, where the robot is given a list of tasks in natural language and has to select the granularity and the subset of objects and scene structure to retain in its map that is sufficient to complete the tasks. We show that this problem can be naturally formulated using the Information Bottleneck (IB), an established information-theoretic framework. The second contribution is an algorithm for task-driven 3D scene understanding based on an Agglomerative IB approach, that is able to cluster 3D primitives in the environment into task-relevant objects and regions and executes incrementally. The third contribution is to integrate our task-driven clustering algorithm into a real-time pipeline, named Clio, that constructs a hierarchical 3D scene graph of the environment online using only onboard compute, as the robot explores it. Our final contribution is an extensive experimental campaign showing that Clio not only allows real-time construction of compact open-set 3D scene graphs, but also improves the accuracy of task execution by limiting the map to relevant semantic concepts.",
"published": "2024-04-21T15:50:40Z",
"updated": "2024-09-26T20:34:51Z",
"authors": [
"Dominic Maggio",
"Yun Chang",
"Nathan Hughes",
"Matthew Trang",
"Dan Griffith",
"Carlyn Dougherty",
"Eric Cristofalo",
"Lukas Schmid",
"Luca Carlone"
],
"primary_cat": "cs.RO",
"journal_ref": "",
"doi": ""
}
}
plans/PRISM/.research/fetch_arxiv.py
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""
fetch_arxiv.py — 从 arXiv 原生 API 抓取每个项目的 title / abstract / year。
arXiv 没有 rate limit 烦恼;Semantic Scholar 429 改这里。
"""
from __future__ import annotations
import json
import pathlib
import re
import sys
import time
import urllib.request
import xml.etree.ElementTree as ET
PROJECTS_JSON = pathlib.Path(__file__).parent / "projects.json"
OUT = pathlib.Path(__file__).parent / "arxiv_data.json"
NS = {"a": "http://www.w3.org/2005/Atom",
"arxiv": "http://arxiv.org/schemas/atom"}
UA = "Mozilla/5.0 (research) PRISM-comparison/1.0"
def get(url: str, timeout: int = 30) -> str:
req = urllib.request.Request(url, headers={"User-Agent": UA})
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read().decode("utf-8", errors="replace")
def fetch_one(arxiv_id: str) -> dict:
"""arXiv id 形如 2309.16650 (可以带 v1 后缀,但去掉更稳)"""
aid = re.sub(r"v\d+$", "", arxiv_id)
url = f"http://export.arxiv.org/api/query?id_list={aid}"
try:
xml = get(url)
except Exception as e:
return {"_error": f"http: {e}"}
try:
root = ET.fromstring(xml)
entry = root.find("a:entry", NS)
if entry is None:
return {"_error": "no entry"}
title = (entry.findtext("a:title", default="", namespaces=NS) or "").strip()
summary = (entry.findtext("a:summary", default="", namespaces=NS) or "").strip()
published = (entry.findtext("a:published", default="", namespaces=NS) or "").strip()
updated = (entry.findtext("a:updated", default="", namespaces=NS) or "").strip()
authors = [
(a.findtext("a:name", default="", namespaces=NS) or "").strip()
for a in entry.findall("a:author", NS)
]
# primary category
prim = entry.find("arxiv:primary_category", NS)
primary = prim.get("term") if prim is not None else None
# journal-ref (= venue if author filled)
jref = entry.findtext("arxiv:journal_ref", default="", namespaces=NS)
# doi
doi = entry.findtext("arxiv:doi", default="", namespaces=NS)
return {
"title": re.sub(r"\s+", " ", title),
"summary": re.sub(r"\s+", " ", summary),
"published": published,
"updated": updated,
"authors": authors,
"primary_cat": primary,
"journal_ref": jref,
"doi": doi,
}
except Exception as e:
return {"_error": f"xml: {e}"}
def main() -> None:
data = json.loads(PROJECTS_JSON.read_text(encoding="utf-8"))
out = {}
for key, v in data.items():
aid = v["meta"]["arxiv"]
sys.stderr.write(f"[arxiv] {v['meta']['name']} ({aid}) ...\n")
out[key] = fetch_one(aid)
time.sleep(3.5) # arXiv 建议 ≥ 3s
OUT.write_text(json.dumps(out, ensure_ascii=False, indent=2),
encoding="utf-8")
sys.stderr.write(f"[ok] {OUT}\n")
if __name__ == "__main__":
main()
plans/PRISM/.research/fetch_projects.py
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
"""
fetch_projects.py — 抓取同档比较项目的元数据(arXiv + Semantic Scholar + GitHub)
输出 JSON 到 stdout 与 projects.json,供后续 comparison.md 编写引用。
"""
from __future__ import annotations
import json
import pathlib
import sys
import time
from typing import Any
import urllib.parse
import urllib.request
OUT = pathlib.Path(__file__).parent / "projects.json"
# 待研究项目列表
PROJECTS = [
{
"key": "concept-graphs",
"name": "ConceptGraphs",
"arxiv": "2309.16650",
"gh": "concept-graphs/concept-graphs",
"homepage": "https://concept-graphs.github.io/",
},
{
"key": "hov-sg",
"name": "HOV-SG",
"arxiv": "2403.17846",
"gh": "hovsg/HOV-SG",
"homepage": "https://hovsg.github.io/",
},
{
"key": "openscene",
"name": "OpenScene",
"arxiv": "2211.15654",
"gh": "pengsongyou/openscene",
"homepage": "https://pengsongyou.github.io/openscene",
},
{
"key": "3d-llm",
"name": "3D-LLM",
"arxiv": "2307.12981",
"gh": "UMass-Foundation-Model/3D-LLM",
"homepage": "https://vis-www.cs.umass.edu/3dllm/",
},
{
"key": "ok-robot",
"name": "OK-Robot",
"arxiv": "2401.12202",
"gh": "ok-robot/ok-robot",
"homepage": "https://ok-robot.github.io/",
},
{
"key": "openmask3d",
"name": "OpenMask3D",
"arxiv": "2306.13631",
"gh": "OpenMask3D/openmask3d",
"homepage": "https://openmask3d.github.io/",
},
{
"key": "conceptfusion",
"name": "ConceptFusion",
"arxiv": "2302.07241",
"gh": "concept-fusion/concept-fusion",
"homepage": "https://concept-fusion.github.io/",
},
{
"key": "clio",
"name": "Clio (MIT-SPARK)",
"arxiv": "2404.13696",
"gh": "MIT-SPARK/Clio",
"homepage": "https://clio-cmu.github.io/",
},
]
UA = "Mozilla/5.0 (research) PRISM-comparison/1.0"
def _get(url: str, headers: dict[str, str] | None = None,
timeout: int = 20) -> str:
h = {"User-Agent": UA, "Accept": "application/json"}
if headers:
h.update(headers)
req = urllib.request.Request(url, headers=h)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read().decode("utf-8", errors="replace")
except Exception as e:
return json.dumps({"_error": str(e)})
def fetch_arxiv(arxiv_id: str) -> dict[str, Any]:
"""Semantic Scholar 一次查到 abstract、引用数、年份"""
url = (
"https://api.semanticscholar.org/graph/v1/paper/"
f"arXiv:{arxiv_id}?fields=title,abstract,year,citationCount,"
"authors.name,venue,publicationVenue,referenceCount,influentialCitationCount,"
"openAccessPdf,fieldsOfStudy"
)
raw = _get(url)
try:
return json.loads(raw)
except Exception as e:
return {"_error": f"json: {e}", "_raw": raw[:300]}
def fetch_github(repo: str) -> dict[str, Any]:
url = f"https://api.github.com/repos/{repo}"
raw = _get(url, headers={"Accept": "application/vnd.github+json"})
try:
d = json.loads(raw)
# 只保留关心字段
if "stargazers_count" in d:
return {
"stars": d.get("stargazers_count"),
"forks": d.get("forks_count"),
"subscribers": d.get("subscribers_count"),
"language": d.get("language"),
"license": (d.get("license") or {}).get("spdx_id"),
"open_issues": d.get("open_issues_count"),
"created_at": d.get("created_at"),
"updated_at": d.get("updated_at"),
"pushed_at": d.get("pushed_at"),
"topics": d.get("topics"),
"description": d.get("description"),
"default_branch": d.get("default_branch"),
"archived": d.get("archived"),
"homepage": d.get("homepage"),
}
return d
except Exception as e:
return {"_error": f"json: {e}", "_raw": raw[:300]}
def main() -> None:
out: dict[str, Any] = {}
for p in PROJECTS:
sys.stderr.write(f"[fetch] {p['name']} ({p['arxiv']}) ...\n")
out[p["key"]] = {
"meta": p,
"arxiv_via_s2": fetch_arxiv(p["arxiv"]),
}
time.sleep(1.2) # 礼貌:S2 rate limit
out[p["key"]]["github"] = fetch_github(p["gh"])
time.sleep(0.8)
OUT.write_text(json.dumps(out, ensure_ascii=False, indent=2),
encoding="utf-8")
sys.stderr.write(f"[ok] wrote {OUT}\n")
print(f"saved {OUT}")
if __name__ == "__main__":
main()
plans/PRISM/.research/fetch_readmes.py
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""
fetch_readmes.py — 抓取每个项目的 README.md 头部 (前 12 KB)
用于推断:支持的平台、依赖、是否提供数据集、demo 视频链接等。
"""
from __future__ import annotations
import json
import pathlib
import re
import sys
import time
import urllib.request
ROOT = pathlib.Path(__file__).parent
PROJECTS_JSON = ROOT / "projects.json"
OUT = ROOT / "readmes"
UA = "Mozilla/5.0 (research) PRISM-comparison/1.0"
def get_text(url: str) -> str:
req = urllib.request.Request(url, headers={"User-Agent": UA})
try:
with urllib.request.urlopen(req, timeout=20) as r:
return r.read().decode("utf-8", errors="replace")
except Exception as e:
return f"<<ERROR {e}>>"
def find_readme(repo: str, branch: str = "main") -> tuple[str, str]:
"""返回 (raw_url, content)。先试 main,再 master。"""
for br in (branch, "master"):
url = f"https://raw.githubusercontent.com/{repo}/{br}/README.md"
txt = get_text(url)
if not txt.startswith("<<ERROR"):
return url, txt
return url, txt
def main() -> None:
data = json.loads(PROJECTS_JSON.read_text(encoding="utf-8"))
OUT.mkdir(exist_ok=True)
for key, v in data.items():
repo = v["meta"]["gh"]
# 部分项目用了非 main 默认分支
default_branch = v.get("github", {}).get("default_branch") or "main"
sys.stderr.write(f"[readme] {v['meta']['name']} <- {repo}@{default_branch} ...\n")
url, txt = find_readme(repo, default_branch)
out_path = OUT / f"{key}.md"
out_path.write_text(f"<!-- src: {url} -->\n\n{txt[:16000]}\n",
encoding="utf-8")
sys.stderr.write(f" -> {out_path} ({len(txt)} chars)\n")
time.sleep(1.0)
sys.stderr.write(f"[ok] all readmes saved to {OUT}\n")
if __name__ == "__main__":
main()
plans/PRISM/.research/projects.json
+247
View File
@@ -0,0 +1,247 @@
{
"concept-graphs": {
"meta": {
"key": "concept-graphs",
"name": "ConceptGraphs",
"arxiv": "2309.16650",
"gh": "concept-graphs/concept-graphs",
"homepage": "https://concept-graphs.github.io/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 868,
"forks": 124,
"subscribers": 8,
"language": "Python",
"license": "MIT",
"open_issues": 24,
"created_at": "2023-09-28T23:58:12Z",
"updated_at": "2026-05-14T11:33:15Z",
"pushed_at": "2025-10-16T22:09:00Z",
"topics": [],
"description": "Official code release for ConceptGraphs",
"default_branch": "main",
"archived": false,
"homepage": null
}
},
"hov-sg": {
"meta": {
"key": "hov-sg",
"name": "HOV-SG",
"arxiv": "2403.17846",
"gh": "hovsg/HOV-SG",
"homepage": "https://hovsg.github.io/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 470,
"forks": 33,
"subscribers": 5,
"language": "Python",
"license": "MIT",
"open_issues": 16,
"created_at": "2024-06-07T13:19:38Z",
"updated_at": "2026-05-14T12:23:03Z",
"pushed_at": "2026-01-19T08:58:18Z",
"topics": [
"3d-scene-graph",
"natural-language-understanding",
"open-vocabulary",
"robot-navigation",
"robot-planning"
],
"description": "[RSS2024] Official implementation of \"Hierarchical Open-Vocabulary 3D Scene Graphs for Language-Grounded Robot Navigation\"",
"default_branch": "main",
"archived": false,
"homepage": "https://hovsg.github.io"
}
},
"openscene": {
"meta": {
"key": "openscene",
"name": "OpenScene",
"arxiv": "2211.15654",
"gh": "pengsongyou/openscene",
"homepage": "https://pengsongyou.github.io/openscene"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 820,
"forks": 67,
"subscribers": 17,
"language": "Python",
"license": "Apache-2.0",
"open_issues": 17,
"created_at": "2023-03-18T09:29:23Z",
"updated_at": "2026-05-16T03:28:57Z",
"pushed_at": "2023-10-27T15:57:18Z",
"topics": [
"3d-scene-understanding",
"clip",
"cvpr2023",
"llm",
"matterport3d",
"nuscenes",
"point-cloud-segmentation",
"point-clouds",
"scannet",
"semantic-segmentation"
],
"description": "[CVPR'23] OpenScene: 3D Scene Understanding with Open Vocabularies",
"default_branch": "main",
"archived": false,
"homepage": "https://pengsongyou.github.io/openscene"
}
},
"3d-llm": {
"meta": {
"key": "3d-llm",
"name": "3D-LLM",
"arxiv": "2307.12981",
"gh": "UMass-Foundation-Model/3D-LLM",
"homepage": "https://vis-www.cs.umass.edu/3dllm/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 1196,
"forks": 74,
"subscribers": 18,
"language": "Python",
"license": "MIT",
"open_issues": 31,
"created_at": "2023-07-24T15:21:57Z",
"updated_at": "2026-05-16T07:35:31Z",
"pushed_at": "2024-06-06T19:14:00Z",
"topics": [],
"description": "Code for 3D-LLM: Injecting the 3D World into Large Language Models",
"default_branch": "main",
"archived": false,
"homepage": ""
}
},
"ok-robot": {
"meta": {
"key": "ok-robot",
"name": "OK-Robot",
"arxiv": "2401.12202",
"gh": "ok-robot/ok-robot",
"homepage": "https://ok-robot.github.io/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 596,
"forks": 42,
"subscribers": 10,
"language": "Python",
"license": "MIT",
"open_issues": 2,
"created_at": "2024-02-22T22:51:59Z",
"updated_at": "2026-05-16T03:11:25Z",
"pushed_at": "2024-03-04T15:56:19Z",
"topics": [
"home-robots",
"open-vocabulary",
"robotics"
],
"description": "An open, modular framework for zero-shot, language conditioned pick-and-drop tasks in arbitrary homes.",
"default_branch": "main",
"archived": false,
"homepage": "https://ok-robot.github.io"
}
},
"openmask3d": {
"meta": {
"key": "openmask3d",
"name": "OpenMask3D",
"arxiv": "2306.13631",
"gh": "OpenMask3D/openmask3d",
"homepage": "https://openmask3d.github.io/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 259,
"forks": 19,
"subscribers": 8,
"language": "Python",
"license": "MIT",
"open_issues": 8,
"created_at": "2023-10-31T14:58:50Z",
"updated_at": "2026-04-20T01:15:48Z",
"pushed_at": "2023-12-15T20:57:07Z",
"topics": [],
"description": null,
"default_branch": "main",
"archived": false,
"homepage": null
}
},
"conceptfusion": {
"meta": {
"key": "conceptfusion",
"name": "ConceptFusion",
"arxiv": "2302.07241",
"gh": "concept-fusion/concept-fusion",
"homepage": "https://concept-fusion.github.io/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 233,
"forks": 17,
"subscribers": 5,
"language": null,
"license": "MIT",
"open_issues": 13,
"created_at": "2023-05-21T11:25:39Z",
"updated_at": "2026-04-23T23:50:16Z",
"pushed_at": "2023-09-23T19:36:58Z",
"topics": [],
"description": "Code release for ConceptFusion [RSS 2023]",
"default_branch": "main",
"archived": false,
"homepage": null
}
},
"clio": {
"meta": {
"key": "clio",
"name": "Clio (MIT-SPARK)",
"arxiv": "2404.13696",
"gh": "MIT-SPARK/Clio",
"homepage": "https://clio-cmu.github.io/"
},
"arxiv_via_s2": {
"_error": "HTTP Error 429: "
},
"github": {
"stars": 239,
"forks": 17,
"subscribers": 11,
"language": "Python",
"license": "BSD-2-Clause",
"open_issues": 10,
"created_at": "2024-04-14T19:29:25Z",
"updated_at": "2026-04-29T01:32:29Z",
"pushed_at": "2025-09-01T14:57:43Z",
"topics": [],
"description": null,
"default_branch": "main",
"archived": false,
"homepage": null
}
}
}
plans/PRISM/.research/readmes/3d-llm.md
+273
View File
@@ -0,0 +1,273 @@
<br />
<p align="center">
<h1 align="center">3D-LLM: Injecting the 3D World into Large Language Models (NeurIPS 2023 Spotlight) </h1>
<p align="center">
<a href="https://evelinehong.github.io">Yining Hong</a>,
<a href="https://haoyuzhen.com">Haoyu Zhen</a>,
<a href="https://peihaochen.github.io">Peihao Chen</a>,
<a href="https://zsh2000.github.io">Shuhong Zheng</a>,
<a href="https://yilundu.github.io">Yilun Du</a>,
<a href="https://zfchenunique.github.io">Zhenfang Chen</a>,
<a href="https://people.csail.mit.edu/ganchuang">Chuang Gan</a>
</p>
<p align="center">
<a href='https://arxiv.org/abs/2307.12981'>
<img src='https://img.shields.io/badge/Paper-PDF-red?style=flat&logo=arXiv&logoColor=red' alt='Paper PDF'>
</a>
<a href='https://vis-www.cs.umass.edu/3dllm/' style='padding-left: 0.5rem;'>
<img src='https://img.shields.io/badge/Project-Page-blue?style=flat&logo=Google%20chrome&logoColor=blue' alt='Project Page'>
</a>
</p>
<p align="center">
<img src="figs/pipeline.png" alt="Logo" width="80%">
</p>
</p>
3D-LLM is the first Large Language Model that could take 3D representations as inputs. It is able to handle both object (e.g., objaverse) and scene data (e.g., scannet & hm3d).
## Installation
Install [salesforce-lavis](https://github.com/salesforce/LAVIS)
```shell
$ conda create -n lavis python=3.8
$ conda activate lavis
$ git clone https://github.com/salesforce/LAVIS.git SalesForce-LAVIS
$ cd SalesForce-LAVIS
$ pip install -e .
$ pip install positional_encodings
```
## Checkpoints
### Pretraining Checkpoints
[Pretrained checkpoints](https://drive.google.com/file/d/1tiis8mOdZGBzmR7vgZtRE4Ni_2FE4nTr/view?usp=drive_link) are released (Please use v2!)
### Finetuning Checkpoints
[Finetuning checkpoints](https://drive.google.com/drive/folders/1RKP1cz6R6H8YziEc4f3MHW9dCCXQChbA?usp=drive_link) for [ScanQA](https://drive.google.com/file/d/1sPynAO8pI_RPR4pwWTrx8weDTdMPsqtW/view?usp=drive_link), [SQA3d](https://drive.google.com/file/d/1Ka9TWv6cs6h-pPaaQG1auIiQma2xbNFk/view?usp=drive_link), and [3DMV_VQA](https://drive.google.com/file/d/1_h2wPPGO64HY5LUcA1bD8DlZx3WCsY8b/view?usp=drive_link) are released.
The results are better than preprint-version paper. We will update the camera-ready paper to the arxiv soon.
## Quick Start: Inference
Download the objaverse subset features [here](https://drive.google.com/file/d/1mJZONfWREfIUAPYXP65D65uS2EoplAfR/view?usp=drive_link). Download the [pretrained checkpoints](https://drive.google.com/drive/folders/1urI2I3S8SgLD8L9brl4ae1Mul_yhCxJe?usp=drive_link). For more details, please refer to `3DLLM_BLIP2-base/DEMO.md`.
```
$ cd 3DLLM_BLIP2-base
$ conda activate lavis
python inference.py # for objects
python inference.py --mode room # for scenes
```
TODO: huggingface auto load checkpoint.
## Finetuning
Finetuning config yaml files that need to be changed are in [this directory](https://github.com/UMass-Foundation-Model/3D-LLM/tree/main/3DLLM_BLIP2-base/lavis/projects/blip2/train)
1. Download the [pretrained checkpoints](https://drive.google.com/file/d/1tiis8mOdZGBzmR7vgZtRE4Ni_2FE4nTr/view?usp=drive_link). Modify the "resume_checkpoint_path" path in the yaml files
2. Download the [questions](https://drive.google.com/drive/folders/14MDiDl6Cch_B27Q0aZgdElhAEOBBpn2o?usp=drive_link), modify the "annotations" path in the yaml files
3. Download the [scannet features](https://drive.google.com/drive/folders/1H1SKZsK_XpMkXntwhY4BqSxsGCRP8cgp?usp=drive_link) or [3dmv-vqa features](https://drive.google.com/drive/folders/1NdFKKn_IZxGezi6fXA60rF1uxTOmhOet?usp=drive_link). Modify the path (both train and val) in lavis/datasets/datasets/threedvqa_datasets.py
4.
```
$ cd 3DLLM_BLIP2-base
$ conda activate lavis
python -m torch.distributed.run --nproc_per_node=8 train.py --cfg-path lavis/projects/blip2/train/<finetune_yaml_file>
```
You can also load the finetuning checkpoints [here](https://drive.google.com/drive/folders/1RKP1cz6R6H8YziEc4f3MHW9dCCXQChbA?usp=drive_link).
5.**Calculating scores**
```
cd calculate_scores
python calculate_score_<task>.py --folder <your result dir> --epoch <your epoch>
```
please also modify the feature and question path in the scripts
TODO: huggingface auto load checkpoint.
## Data
All data will be gradually released in [Google Drive](https://drive.google.com/drive/folders/188Yd7tmiUfyct-dVMpkQ8q_tnqkb-4bo?usp=sharing) and [Huggingface](https://huggingface.co/datasets/ShuhongZheng/3D-LLM) (All files are released in Google Drive first and then Huggingface. Please refer to the Google Drive for file structure)
### Pretraining Data
We are still cleaning the grounding & navigation part. All other pre-training data are released.
#### Object Data
&emsp;Language annotations of object data released [here](https://drive.google.com/file/d/17K1ZGb1HpvPj36a84GefGQK7DOkRp_32/view?usp=sharing).
&emsp;For downloading Objaverse data, please refer to [Objaverse website](https://objaverse.allenai.org/).
&emsp;To get 3D features and point clouds of the Objaverse data, please refer to [Step1](https://github.com/UMass-Foundation-Model/3D-LLM/blob/main/README.md#step1-render-images-from-different-views-of-a-scene) and [Step3](https://github.com/UMass-Foundation-Model/3D-LLM/blob/main/README.md#step3-3d-feature-construction-from-rendered-images) of 3DLanguage Data generation - ChatCaptioner based
&emsp;A small set of objaverse features is released [here](https://drive.google.com/file/d/1mJZONfWREfIUAPYXP65D65uS2EoplAfR/view?usp=drive_link).
&emsp;TODO: We will probably release the whole set of Objaverse 3D features
#### Scene Data
&emsp; [Language data released here](https://drive.google.com/file/d/18AVyAWwMwqP8nyZQfTpEMw1Wfhghjw0b/view?usp=drive_link).
&emsp; 3D features and point clouds (~250G) are released [here](https://drive.google.com/drive/folders/1bJpK00UFRZGH7tdx3PaqDrrTfHNP6B2K?usp=drive_link). However, if you want to explore generating the features yourself, please refer to the Three-step 3D Feature Extraction part [here](https://github.com/UMass-Foundation-Model/3D-LLM#three-step-3d-feature-extraction). Please use v2 to be consistent with the checkpoints (and also result in better performances).
&emsp;chat: 73103. task: 84531
### Finetuning Data
3D features and point clouds of Scannet (used for finetuning ScanQA and SQA3D) are released in [here](https://drive.google.com/drive/folders/1CsEt48jj5uCyelGcXXJBkGH86QYeCE8D?usp=drive_link). 3D features and point clouds of 3DMV-VQA are released [here](https://drive.google.com/drive/folders/1NdFKKn_IZxGezi6fXA60rF1uxTOmhOet?usp=drive_link) (3DMV-VQA data will be further updated for a clearer structure).
All questions can be found [here](https://drive.google.com/drive/folders/14MDiDl6Cch_B27Q0aZgdElhAEOBBpn2o?usp=drive_link).
## 3DLanguage Data Generation
### ChatCaptioner based / Three-step 3D Feature Extraction (Objaverse)
#### Step1: render images from different views of a scene
Follow the instruction in ``3DLanguage_data/ChatCaptioner_based/objaverse_render/README.md`` for installation.
The following code will render images of a objaverse scene (e.g. f6e9ec5953854dff94176c36b877c519). The rendered images will be saved at ``3DLanguage_data/ChatCaptioner_based/objaverse_render/output``.
(Please refer to ``3DLanguage_data/ChatCaptioner_based/objaverse_render/README.md`` for more details about the command)
```shell
$ cd ./3DLanguage_data/ChatCaptioner_based/objaverse_render
$ {path/to/blender} -b -P render.py -noaudio --disable-crash-handler -- --uid f6e9ec5953854dff94176c36b877c519
```
#### Step2: generate caption for this objaverse scene
Installation:
Please follow [ChatCaptioner](https://github.com/Vision-CAIR/ChatCaptioner) to install the environment/
The following code will read the rended images of an objaverse scene (e.g., f6e9ec5953854dff94176c36b877c519) and generate scene caption at ``3DLanguage_data/ChatCaptioner_based/output``
```shell
$ cd ./3DLanguage_data/ChatCaptioner_based
$ python chatcaption.py --specific_scene f6e9ec5953854dff94176c36b877c519
```
#### Step3: 3D feature construction from rendered images
Follow the instruction in ``3DLanguage_data/ChatCaptioner_based/gen_features/README.md`` for extracting 3D features from rendered images.
```shell
$ cd ./3DLanguage_data/ChatCaptioner_based/gen_features
```
### Box-Demonstration-Instruction based
TODO
### Revision based
TODO
## Three-step 3D Feature Extraction (Scene)
This section is for constructing 3D features for scene data. If you already downloaded our released scene data, please skip this section.
### First step
Installation:
Please follow [Mask2Former](https://github.com/facebookresearch/Mask2Former) to install the environment and download the [pretrained weight](https://dl.fbaipublicfiles.com/maskformer/mask2former/coco/instance/maskformer2_swin_large_IN21k_384_bs16_100ep/model_final_e5f453.pkl) to the current directory
if extracting the masks with [Mask2Former](https://openaccess.thecvf.com/content/CVPR2022/papers/Cheng_Masked-Attention_Mask_Transformer_for_Universal_Image_Segmentation_CVPR_2022_paper.pdf).
Please follow [Segment Anything](https://github.com/facebookresearch/segment-anything) to install the environment and download the [pretrained weight](https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth) to the current directory if extracting the masks with [SAM](https://arxiv.org/abs/2304.02643).
Extract masks with Mask2Former:
```shell
$ cd ./three_steps_3d_feature/first_step
$ python maskformer_mask.py --scene_dir_path DATA_DIR_WITH_RGB_IMAGES --save_dir_path DIR_YOU_WANT_TO_SAVE_THE_MASKS
```
Extract masks with Segment Anything:
```shell
$ cd ./three_steps_3d_feature/first_step
$ python sam_mask.py --scene_dir_path DATA_DIR_WITH_RGB_IMAGES --save_dir_path DIR_YOU_WANT_TO_SAVE_THE_MASKS
```
After the first step, we are expected to obtain a directory of masks (specified by ``--save_dir_path``) that contains extracted masks for
multi-view images of the scenes.
### Second step
Note: BLIP features are for LAVIS(BLIP2), CLIP features are for open-flamingo.
Installation: The same as the following ``3D-LLM_BLIP2-based`` section to install [salesforce-lavis](https://github.com/salesforce/LAVIS).
There are four options: (1) Extract CLIP feature with Mask2Former masks; (2) Extract CLIP feature with SAM masks;
(3) Extract BLIP feature with Mask2Former masks; (4) Extract BLIP feature with SAM masks.
Extract 2D CLIP features with Mask2Former masks:
```shell
$ cd ./three_steps_3d_feature/second_step/
$ python clip_maskformer.py --scene_dir_path DATA_DIR_WITH_RGB_IMAGES --mask_dir_path MASK_DIR_FROM_1ST_STEP --save_dir_path DIR_YOU_WANT_TO_SAVE_THE_FEAT
```
For the other options, the scripts are in similar format.
After the second step, we are expected to obtain a directory of features (specified by ``--save_dir_path``) that contains 2D features for
multi-view images of the scenes.
### Third step
#### Direct Reconstruction
Installation:
Please install the [Habitat environment](https://github.com/facebookresearch/habitat-lab/tree/challenge-2022).
Reconstruct 3D feature from multi-view 2D features:
```shell
$ cd ./three_steps_3d_feature/third_step/
$ python sam_mask.py --data_dir_path DATA_DIR_WITH_RGB_IMAGES --depth_dir_path DATA_DIR_WITH_DEPTH_IMAGES --feat_dir_path FEATURE_DIR_FROM_2ND_STEP
```
After the third step, we are expected to obtain two files (``pcd_pos.pt`` and ``pcd_feat.pt``) for each room inside the corresponding RGB directory.
``pcd_pos.pt`` contains the point positions of the 3D point cloud (shape: ``N * 3``). ``pcd_feat.pt`` contains the point features of the 3D point cloud (shape: ``N * n_dim``).
``N`` is the number of sampled points in the point cloud (default: 300000) and ``n_dim`` is the feature dimension (1024 for CLIP feature, 1408 for BLIP feature).
#### GradSLAM (Feature Fusion)
Refer to [Concept Fusion](https://github.com/concept-fusion/concept-fusion).
We will also release our reproduced version of Concept Fusion for our feature generation (we reproduced the paper before their official release).
#### Neural Field
Please refer to [3D-CLR](https://github.com/evelinehong/3D-CLR-Official) repository.
## Pre-training
```shell
$ cd 3DLLM_BLIP2-base
$ conda activate lavis
# use facebook/opt-2.7b:
$ TODO
# use flant5
$ python -m torch.distributed.run --nproc_per_node=8 train.py --cfg-path lavis/projects/blip2/train/pretrain.yaml
```
## 3D-LLM_flamingo-based
TODO.
## Citation
If you find our work useful, please consider citing:
```
@article{3dllm,
author = {Hong, Yining and Zhen, Haoyu and Chen, Peihao and Zheng, Shuhong and Du, Yilun and Chen, Zhenfang and Gan, Chuang},
title = {3D-LLM: Injecting the 3D World into Large Language Models},
journal = {NeurIPS},
year = {2023},
}
```
### Acknowledgements
https://github.com/salesforce/LAVIS
https://github.com/facebookresearch/Mask2Former
https://github.com/facebookresearch/segment-anything
https://github.com/mlfoundations/open_flamingo
https://github.com/concept-fusion/concept-fusion
https://github.com/evelinehong/3D-CLR-Official
plans/PRISM/.research/readmes/clio.md
+333
View File
@@ -0,0 +1,333 @@
# Clio
This repository contains the code for *Clio: Real-time Task-Driven Open-Set 3D Scene Graphs*.
![Clio](./assets/overview-figure.jpg)
Clio is a novel approach for building task-driven 3D scene graphs in real-time with open-set semantics. We draw inspiration from the classical Information Bottleneck principle to form task-relevant clusters of object primitives given a set of natural language tasks — such as ''Read brown textbook'' — and by clustering the scene into task-relevant semantic regions such as “Kitchenette” or “Workspace”. The map defines objects and regions at the correct semantic granularity to support tasks relevant for an agent.
# Table of Contents
* [Paper](#Paper)
* [News](#News)
* [Setup](#Setup)
* [Datasets](#Datasets)
* [Running Clio](#running-clio)
* [Clustering Pre-built Scene Graphs](#clustering-pre-built-scene-graphs)
* [Evaluation](#evaluation)
# Paper
If you find this useful for your research, please consider citing our paper:
* Dominic Maggio, Yun Chang, Nathan Hughes, Matthew Trang, Dan Griffith, Carlyn Dougherty, Eric Cristofalo, Lukas Schmid, Luca Carlone, "**Clio: Real-time Task-Driven Open-Set 3D Scene Graphs**", in IEEE Robotics and Automation Letters, vol. 9, no. 10, pp. 8921-8928, Oct. 2024. [ [IEEE](https://ieeexplore.ieee.org/document/10659066) | [ArXiv](http://arxiv.org/abs/2404.13696) | [Video](https://www.youtube.com/watch?v=m-HJO10qhSQ) ]
```bibtex
@ARTICLE{Maggio2024Clio,
title={Clio: Real-time Task-Driven Open-Set 3D Scene Graphs},
author={Maggio, Dominic and Chang, Yun and Hughes, Nathan and Trang, Matthew and Griffith, Dan and Dougherty, Carlyn and Cristofalo, Eric and Schmid, Lukas and Carlone, Luca},
journal={IEEE Robotics and Automation Letters},
year={2024},
volume={9},
number={10},
pages={8921-8928},
doi={10.1109/LRA.2024.3451395}
}
```
# News
- **[Bayesian Fields](https://arxiv.org/abs/2503.05949)** [Video](https://www.youtube.com/watch?v=DLBHpUP_-dk) – follow-up paper released showing improved results on the Clio datasets using better statistical understanding of CLIP and handling multi-view semantic measurements. Bayesian Fields also shows task-driven clustering with Gaussian Splatting.
- **[Ashita](https://arxiv.org/abs/2504.06553)** – follow-up paper released showing an LLM assisted task-driven reasoning framework that can use higher-level tasks and construct a 3D scene graph for all subtasks.
- Clio work was featured on the front page of **[MIT News](https://news.mit.edu/2024/helping-robots-focus-on-objects-that-matter-0930)** 🎉
# Setup
We recommend that everyone setup Clio by utilizing ROS.
To install ROS, you can follow the instructions [here](http://wiki.ros.org/ROS/Installation) if you haven't already.
> **Note**</br>
> We also provide a python-only implementation of Clio for offline processing of pre-built scene graphs and evaluation.
> If you want to avoid installing ROS and are only interested in this functionality, you can skip ahead to [these](#installing-without-ros) instructions instead.
### Installing with ROS
<details open>
<summary><b>Initial Requirements</b></summary>
Install the following requirements:
```
sudo apt install python3-rosdep python3-catkin-tools python3-vcstool python3-virtualenv
```
If you haven't set up rosdep yet run:
```
sudo rosdep init
rosdep update
```
</details>
<details open>
<summary><b>Getting and Building Clio</b></summary>
To clone and build Clio, first set up your catkin workspace:
```bash
mkdir -p ~/catkin_ws/src
cd ~/catkin_ws
catkin init
catkin config -DCMAKE_BUILD_TYPE=Release
catkin config --skiplist khronos_eval
```
> **Note**</br>
> By default, one of Clio's dependencies, [semantic_inference](https://github.com/MIT-SPARK/semantic_inference.git), will attempt to build against NVIDIA TensorRT.
> This is not required for Clio, and may cause issues when building if you already have CUDA set up on your system.
> You may wish to disable this by running `catkin config -a -DSEMANTIC_INFERENCE_USE_TRT=OFF` before building.
Then, clone the code and build:
```
cd src
git clone git@github.com:MIT-SPARK/Clio.git clio --recursive
vcs import . < clio/install/clio.rosinstall
rosdep install --from-paths . --ignore-src -r -y
cd ..
catkin build
```
> **Note**</br>
> For the rest of these instructions, we assume that you set up your catkin workspace at `~/catkin_ws`.
> If you used a different workspace path, you should substitute where appropriate.
</details>
<details open>
<summary><b>Setting up Open-Set Segmentation</b></summary>
Make a virtual environment and install:
```bash
python3 -m virtualenv --system-site-packages -p /usr/bin/python3 ~/environments/clio_ros
source ~/environments/clio_ros/bin/activate
pip install ~/catkin_ws/src/semantic_inference/semantic_inference[openset]
deactivate
```
> **Warning** :warning:</br>
> `--system-site-packages` is required when creating the environment.
</details>
<details open>
<summary><b>Setting up Clio Python Code</b></summary>
Make a virtual environment and install:
```bash
python3 -m virtualenv --download -p /usr/bin/python3 ~/environments/clio
source ~/environments/clio/bin/activate
pip install -e ~/catkin_ws/src/clio
```
> **Warning** :warning:</br>
> A devel install (i.e., using `-e` when installing Clio) is required.
</details>
### Installing without ROS
> **Warning** :warning:</br>
> This option does not include the open-set segmentation code or the real-time pipeline
First, setup a virtual environment:
```bash
python3 -m virtualenv -p /usr/bin/python3 --download ~/environments/clio
```
Then, clone and install Clio:
```bash
source ~/environments/clio/bin/activate
git clone https://github.com/MIT-SPARK/Clio.git clio --recursive
pip install -e clio
```
> **Note**</br>
> If you forgot to clone with `--recursive` you can run `git submodule update --init --recursive` instead.
# Datasets
Our custom datasets for the *Office*, *Apartment*, *Cubicle*, and *Building* scenes are available for download [here](https://www.dropbox.com/scl/fo/5bkv8rsa2xvwmvom6bmza/AOc8VW71kuZCgQjcw_REbWA?rlkey=wx1njghufcxconm1znidc1hgw&st=c809h8h3&dl=0). Each scene contains RGB images, depth images, a rosbag containing the RGB and depth images along with poses, and the list of tasks with ground truth object labels that was used in our paper. Each scene except *Building* contains a COLMAP dense reconstruction which can optionally be used to separately get a dense mesh view of the scene.
The task list is stored in a yaml file whose keys are the task and values are the ground truth oriented bounding boxes for the relevant objects. The folder structure is:
```
clio_datasets
├── apartment
│ ├── apartment.bag
│ ├── database.db
│ ├── dense
| ├── fused.ply
| |── meshed-poisson.ply
│ ├── depth
│ ├── images
│ ├── region_tasks_apartment.yaml
│ ├── rooms_apartment.yaml
│ ├── sparse
│ └── tasks_apartment.yaml
├── building
│ ├── ...
├── cubicle
│ ├── ...
├── office
│ ├── ...
```
### Pre-built Scene Graphs
Pre-built scene graph files can be downloaded from [here](https://www.dropbox.com/scl/fo/dmft0x6o0joxjxyy2q1qz/AIbgXuR4Y0nxMgsq1VtlWak?rlkey=g3orpk2itv1eyg0r1h7mpfgtt&st=po6btms5&dl=0),
which contain the 3D object primitives with corresponding meshes and semantic embedding vectors that Clio can use to form task-relevant objects.
These scene graph files can be used to test out Clio's Information Bottleneck clustering on a variety of tasks.
See [here](#clustering-pre-built-scene-graphs) for details.
### Pre-generating Open-set Semantics for a Scene
> **Warning** :warning: </br>
> This requires the [semantic_inference](https://github.com/MIT-SPARK/semantic_inference) package and ROS, which is installed by default if you follow the normal setup guide for Clio.
It may be convenient to generate the open-set segmentation and CLIP embeddings for a scene before running Clio.
You can run the following commands for any of the scenes, substituting the appropriate path to the rosbag for the scene.
First, source your `semantic_inference` environment and change to the directory containing the datasets if you haven't already:
```bash
source ~/environments/semantic_inference_ros/bin/activate
cd /path/to/clio/datasets
```
Using the apartment scene as an example, run:
```bash
rosrun semantic_inference_ros make_rosbag --clip-vec --copy \
apartment/apartment.bag -o apartment/apartment_with_semantics.bag \
/dominic/forward/color/image_raw:/dominic/forward/semantic/image_raw
```
to create a new bag, `apartment_with_semantics.bag` that contains the original contents of `apartment.bag` along with the open-set segmentation (under the `/dominic/forward/semantic/image_raw` topic).
# Running Clio
To run Clio on one of the provided [datasets](#datasets), first source your catkin workspace and python environment:
```bash
source ~/catkin_ws/devel/setup.bash
source ~/environments/clio_ros/bin/activate
```
In the following instructions, make sure to substitute the actual path to the datasets in place of `/path/to/datset`.
We'll use the *Office* scene for this example, but any of the datasets should work.
First, start Clio:
```bash
roslaunch clio_ros realsense.launch \
object_tasks_file:=/path/to/datasets/office/tasks_office.yaml \
place_tasks_file:=/path/to/datasets/office/region_tasks_office.yaml
```
If you want to use pre-generated segmentations and semantics instead, you can start Clio with the following:
```bash
roslaunch clio_ros realsense.launch run_segmentation:=false \
object_tasks_file:=/path/to/datasets/office/tasks_office.yaml \
place_tasks_file:=/path/to/datasets/office/region_tasks_office.yaml
```
> **Note**</br>
> Regardless of the `run_segmentation` setting, you should wait until Clio finishes initializing before starting the rosbag. You should see roughly this before proceeding:
> ```
> ...
> [INFO] [1728321782.786728, 0.000000]: '/semantic_inference': finished initializing!
> I1007 17:23:03.652261 3390159 ros_embedding_group.cpp:70] Got embeddings from '/task_server/objects'!
> I1007 17:23:03.654877 3390159 ros_embedding_group.cpp:47] Waiting for embeddings on '/task_server/places'
> I1007 17:23:03.857106 3390159 ros_embedding_group.cpp:70] Got embeddings from '/task_server/places'!
> I1007 17:23:04.047123 3390159 input_module.cpp:76] [Hydra Input] started!
> I1007 17:23:04.047190 3390159 active_window_module.cpp:72] [Active Window] started!
> I1007 17:23:04.047272 3390159 backend_module.cpp:134] [Hydra Backend] started!
> I1007 17:23:04.048808 3390159 graph_builder.cpp:175] [Hydra Frontend] started!
> [ INFO] [1728321784.049510656]: Running...
> ```
In a separate terminal, start the corresponding rosbag for the scene. For the *Office* dataset:
```bash
rosbag play path/to/datasets/office/office.bag --clock
```
You should see something like this after letting the bag play for a little bit:
![cluster](./assets/clio_online_example.png)
# Clustering Pre-built Scene Graphs
You can also try out Clio in an offline setting to perform task-oriented clustering on a pre-built scene graph of object primitives.
To try out Clio in this manner, run the following:
```
cd clio
python clio_batch/object_cluster.py <path-to-scene-graph-json> <path-to-task-yaml> <path-to-output-folder> <path-to-cluster-config>
```
For example, to run Clio just on the pre-built office scene graph (assuming you have made an empty folder first called clio_output), use the following command:
```
python clio_batch/object_cluster.py ~/clio_logs/office/trial_0/large_fine_ps/dsg.json ~/clio_datasets/office/tasks_office.yaml ~/clio_output clio_eval/experiments/configs/cluster/large_delta_00001.yaml
```
To quickly visualize results, you can visualize the estimated Clio bounding boxes (in green) and ground truth bounding boxes (in black) over a COLMAP mesh. To do this on the office scene, run:
```
python clio_eval/visualize_objects.py ~/clio_output/clio_dsg.json ~/clio_datasets/office/tasks_office.yaml ~/clio_datasets/office/dense/meshed-poisson.ply office
```
This will generate a 3D viewer such as this:
![cluster](./assets/sample_clustered.png)
Change to either apartment or cubicle with the following to generate the visualization on those datasets:
```
python clio_eval/visualize_objects.py clio_dsg.json ~/clio_datasets/apartment/tasks_apartment.yaml ~/clio_datasets/apartment/dense/meshed-poisson.ply apartment
python clio_eval/visualize_objects.py clio_dsg.json ~/clio_datasets/cubicle/tasks_cubicle.yaml ~/clio_datasets/cubicle/dense/meshed-poisson.ply cubicle
```
# Evaluation
> **Note**</br>
> To prepare Clio's codebase for release, we updated the versions of Hydra and Khronos that Clio is integrated with and changed how we integrated these codebases.
> An unintended consequence of these changes is that the performance of the real-time version of Clio has decreased compared to the results reported in our paper; this stems from what we believe is a bug associated with the reconstructed object primitives produced by Khronos.
> For a more complete discussion of the problem, please see [this issue](https://github.com/MIT-SPARK/Clio/issues/8).
> We are actively working on isolating the root cause of the issue.
> In the meantime, we've temporarily released a snapshot of the code used to generate the results in our paper [here](https://www.dropbox.com/scl/fi/q4ws6wnh5z9l1jdxztbf9/clio_ws.zip?rlkey=cgjrob8ddkyhof0rosizw7jmw&st=thdh8o5m&dl=0), which includes instructions for how to build and run this version of the code.
> If you plan on including results in a paper that require building new scene graphs with Clio, we politely request that you use this earlier version of Clio for now if possible.
> Please feel free to reach out if you have any questions!
We have included some of the evaluation code used to generate the results in our [paper](http://arxiv.org/abs/2404.13696).
In our paper, we refer to results obtained using the offline version of Clio (see [here](#clustering-pre-built-scene-graphs)) as *Clio-batch*, while the real-time ROS pipline (see [here](#running-clio)) is referred to as *Clio-online*.
We adopt this terminology here to make it easier to refer to the paper results.
### Evaluating Clio-Online
To obtain metrics on the quality of objects obtained by *Clio-Online*, set the paths in [realtime_clustering.yaml](clio_eval/experiments/configs/ablations/realtime_clustering.yaml) to the corresponding local task yaml files and folders containing the scene graphs produced by *Clio-online*. Then run:
```bash
python clio_eval/experiments/evaluate_ablations.py --config clio_eval/experiments/configs/ablations/realtime_clustering.yaml
```
### Ablating Clio-batch
This section is concerned with obtaining a partial ablation study of *Clio-batch*.
A config file for using CLIP model `ViT-L/14` used in the main results of the paper is provided in [3d_clustering.yaml](clio_eval/experiments/configs/ablations/3d_clustering.yaml).
Also note that a config file for Open-CLIP model `ViT-H-14` used to produce results in the appendix of the paper is provided in [3d_clustering_oc.yaml](clio_eval/experiments/configs/ablations/3d_clustering_oc.yaml).
The yaml files allow for running on one or on multiple datasets at a time.
First, set paths to your pre-built scene graph files and yaml files containing the ground truth objects and task labels in `3d_clustering.yaml` and `3d_clustering_oc.yaml`.
Then, to run *Clio-batch* and evaluate the results, run:
```
python clio_eval/experiments/run_3d_object_ablations.py clio_eval/experiments/configs/ablations/<3d_clustering.yaml/3d_clustering_oc.yaml>
```
The provided results will include for each dataset:
- Clio Primitives (labeled as `<dataset>_fine`)
- Khronos (labeled as `<dataset>_khronos`)
- Khronos thresholded (labeled as `<dataset>_khronos_thres`)
- Clio batch (labeled as `<dataset>_d<IB delta stop value>`)
By default, the config files are set to run on all three of the office, apartment, and cubicle datasets. Results from clustering will be saved in a newly created scene graph files for each dataset.
plans/PRISM/.research/readmes/concept-graphs.md
+318
View File
@@ -0,0 +1,318 @@
<!-- src: https://raw.githubusercontent.com/concept-graphs/concept-graphs/main/README.md -->
# ConceptGraphs: Open-Vocabulary 3D Scene Graphs for Perception and Planning
[**Project Page**](https://concept-graphs.github.io/) |
[**Paper**](https://concept-graphs.github.io/assets/pdf/2023-ConceptGraphs.pdf) |
[**ArXiv**](https://arxiv.org/abs/2309.16650) |
[**Video**](https://www.youtube.com/watch?v=mRhNkQwRYnc&feature=youtu.be&ab_channel=AliK)
[Qiao Gu](https://georgegu1997.github.io/)\*,
[Ali Kuwajerwala](https://www.alihkw.com/)\*,
[Sacha Morin](https://sachamorin.github.io/)\*,
[Krishna Murthy Jatavallabhula](https://krrish94.github.io/)\*,
[Bipasha Sen](https://bipashasen.github.io/),
[Aditya Agarwal](https://skymanaditya1.github.io/),
[Corban Rivera](https://www.jhuapl.edu/work/our-organization/research-and-exploratory-development/red-staff-directory/corban-rivera),
[William Paul](https://scholar.google.com/citations?user=92bmh84AAAAJ),
[Kirsty Ellis](https://mila.quebec/en/person/kirsty-ellis/),
[Rama Chellappa](https://engineering.jhu.edu/faculty/rama-chellappa/),
[Chuang Gan](https://people.csail.mit.edu/ganchuang/),
[Celso Miguel de Melo](https://celsodemelo.net/),
[Joshua B. Tenenbaum](http://web.mit.edu/cocosci/josh.html),
[Antonio Torralba](https://groups.csail.mit.edu/vision/torralbalab/),
[Florian Shkurti](http://www.cs.toronto.edu//~florian/),
[Liam Paull](http://liampaull.ca/)
![Splash Figure](./assets/splash-final.png)
# Updates
* The codebase has been significantly refactored in the `ali-dev` [branch](https://github.com/concept-graphs/concept-graphs/tree/ali-dev), which provides a real-time, streamlined re-implementation that supports RGB-D video from iPhone and has a better visualization using [Rerun.io](https://rerun.io/). We also provide this [getting started video tutorial](https://youtu.be/56jEFyrqqpo?si=jo-qto5Gv8qxqEw2). Please check it out!
* The code for real-world mapping and navigation using a Jackal robot is open-sourced [here](https://github.com/sachaMorin/concept_graphs_jackal).
* The code for localization and mapping in AI2Thor is released in the codebase. See [here](https://github.com/concept-graphs/concept-graphs/tree/main?tab=readme-ov-file#ai2thor-related-experiments) for instructions.
## Setup
The env variables needed can be found in `env_vars.bash.template`. When following the setup guide below, you can duplicate that files and change the variables accordingly for easy setup.
### Install the required libraries
We recommend setting up a virtual environment using virtualenv or conda. Our code has been tested with Python 3.10.12. It may also work with other later versions. We also provide the `environment.yml` file for Conda users. In generaly, directly installing conda env using `.yml` file may cause some unexpected issues, so we recommand setting up the environment by the following instructions and only using the `.yml` file as a reference.
Sample instructions for `conda` users.
```bash
conda create -n conceptgraph anaconda python=3.10
conda activate conceptgraph
# Install the required libraries
pip install tyro open_clip_torch wandb h5py openai hydra-core distinctipy
# for yolo
pip install ultralytics
# Install the Faiss library (CPU version should be fine)
conda install -c pytorch faiss-cpu=1.7.4 mkl=2021 blas=1.0=mkl
##### Install Pytorch according to your own setup #####
# For example, if you have a GPU with CUDA 11.8 (We tested it Pytorch 2.0.1)
conda install pytorch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 pytorch-cuda=11.8 -c pytorch -c nvidia
# Install Pytorch3D (https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md)
# conda install pytorch3d -c pytorch3d # This detects a conflict. You can use the command below, maybe with a different version
conda install https://anaconda.org/pytorch3d/pytorch3d/0.7.4/download/linux-64/pytorch3d-0.7.4-py310_cu118_pyt201.tar.bz2
# Install the gradslam package and its dependencies
# Please clone and install them in separate folders, not within the concept-graphs folder.
git clone https://github.com/krrish94/chamferdist.git
cd chamferdist
pip install .
cd ..
git clone https://github.com/gradslam/gradslam.git
cd gradslam
git checkout conceptfusion
pip install .
```
### Install [Grounded-SAM](https://github.com/IDEA-Research/Grounded-Segment-Anything) package
Follow the instructions on the original [repo](https://github.com/IDEA-Research/Grounded-Segment-Anything#install-without-docker). ConceptGraphs has been tested with the codebase at this [commit](https://github.com/IDEA-Research/Grounded-Segment-Anything/commit/a4d76a2b55e348943cba4cd57d7553c354296223). Grounded-SAM codebase at later commits may require some adaptations.
First checkout the package by
```bash
git clone git@github.com:IDEA-Research/Grounded-Segment-Anything.git
```
Then, install the package Following the commands listed in the original GitHub repo. You can skip the `Install osx` step and the "optional dependencies".
During this process, you will need to set the `CUDA_HOME` to be where the CUDA toolkit is installed.
The CUDA tookit can be set up system-wide or within a conda environment. We tested it within a conda environment, i.e. installing [cudatoolkit-dev](https://anaconda.org/conda-forge/cudatoolkit-dev) using conda.
```bash
# i.e. You can install cuda toolkit using conda
conda install -c conda-forge cudatoolkit-dev
# and you need to replace `export CUDA_HOME=/path/to/cuda-11.3/` by
export CUDA_HOME=/path/to/anaconda3/envs/conceptgraph/
```
You also need to download `ram_swin_large_14m.pth`, `groundingdino_swint_ogc.pth`, `sam_vit_h_4b8939.pth` (and optionally `tag2text_swin_14m.pth` if you want to try Tag2Text) following the instruction [here](https://github.com/IDEA-Research/Grounded-Segment-Anything#label-grounded-sam-with-ram-or-tag2text-for-automatic-labeling).
After installation, set the path to Grounded-SAM as an environment variable
```bash
export GSA_PATH=/path/to/Grounded-Segment-Anything
```
### (Optional) Set up the EfficientSAM variants
Follow the installation instructions on this [page](https://github.com/IDEA-Research/Grounded-Segment-Anything/tree/main/EfficientSAM). The major steps are:
* Install FastSAM codebase following [here](https://github.com/CASIA-IVA-Lab/FastSAM#installation). You don't have to create a new conda env. Just installing it in the same env as the Grounded-SAM is fine.
* Download FastSAM checkpoints [FastSAM-x.pt](https://github.com/CASIA-IVA-Lab/FastSAM#model-checkpoints) and save it to `Grounded-Segment-Anything/EfficientSAM`.
* Download MobileSAM checkpoints [mobile_sam.pt](https://github.com/ChaoningZhang/MobileSAM/blob/master/weights/mobile_sam.pt) and save it to `Grounded-Segment-Anything/EfficientSAM`.
* Download Light HQ-SAM checkpoints [sam_hq_vit_tiny.pth](https://huggingface.co/lkeab/hq-sam/resolve/main/sam_hq_vit_tiny.pth) and save it to `Grounded-Segment-Anything/EfficientSAM`.
### Install this repo
```bash
git clone git@github.com:concept-graphs/concept-graphs.git
cd concept-graphs
pip install -e .
```
### Set up LLaVA (used for scene graph generation)
Follow the instructions on the [LLaVA repo](https://github.com/haotian-liu/LLaVA) to set it up. You also need to prepare the LLaVA checkpoints and save them to `$LLAVA_CKPT_PATH`. We have tested with model checkpoint `LLaVA-7B-v0` and [LLaVA code](https://github.com/haotian-liu/LLaVA) at this [commit](https://github.com/haotian-liu/LLaVA/tree/8fc54a09a6be74b2abd913c468fb3d42ae826194). LLaVA codebase at later commits may require some adaptations.
```bash
# Set the env variables as follows (change the paths accordingly)
export LLAVA_PYTHON_PATH=/path/to/llava
export LLAVA_CKPT_PATH=/path/to/LLaVA-7B-v0
```
## Prepare dataset (Replica as an example)
ConceptGraphs takes posed RGB-D images as input. Here we show how to prepare the dataset using [Replica](https://github.com/facebookresearch/Replica-Dataset) as an example. Instead of the original Replica dataset, download the scanned RGB-D trajectories of the Replica dataset provided by [Nice-SLAM](https://github.com/cvg/nice-slam). It contains rendered trajectories using the mesh models provided by the original Replica datasets.
Download the Replica RGB-D scan dataset using the downloading [script](https://github.com/cvg/nice-slam/blob/master/scripts/download_replica.sh) in [Nice-SLAM](https://github.com/cvg/nice-slam#replica-1) and set `$REPLICA_ROOT` to its saved path.
```bash
export REPLICA_ROOT=/path/to/Replica
export CG_FOLDER=/path/to/concept-graphs/
export REPLICA_CONFIG_PATH=${CG_FOLDER}/conceptgraph/dataset/dataconfigs/replica/replica.yaml
```
ConceptGraphs can also be easily run on other dataset. See `dataset/datasets_common.py` for how to write your own dataloader.
## Run ConceptGraph
The following commands should be run in the `conceptgraph` folder.
```bash
cd conceptgraph
```
### (Optional) Run regular 3D reconstruction for sanity check
The following command runs a 3D RGB reconstruction ([GradSLAM](https://github.com/gradslam/gradslam)) of a replica scene and also visualize it. This is useful for sanity check.
* `--visualize` requires it to be run with GUI.
```bash
SCENE_NAME=room0
python scripts/run_slam_rgb.py \
--dataset_root $REPLICA_ROOT \
--dataset_config $REPLICA_CONFIG_PATH \
--scene_id $SCENE_NAME \
--image_height 480 \
--image_width 640 \
--stride 5 \
--visualize
```
### Extract 2D (Detection) Segmentation and per-resgion features
First, (Detection) Segmentation results and per-region CLIP features are extracted. In the following, we provide two options.
* The first one (ConceptGraphs) uses SAM in the "segment all" mode and extract class-agnostic masks.
* The second one (ConceptGraphs-Detect) uses a tagging model and a detection model to extract class-aware bounding boxes first, and then use them as prompts for SAM to segment each object.
```bash
SCENE_NAME=room0
# The CoceptGraphs (without open-vocab detector)
python scripts/generate_gsa_results.py \
--dataset_root $REPLICA_ROOT \
--dataset_config $REPLICA_CONFIG_PATH \
--scene_id $SCENE_NAME \
--class_set none \
--stride 5
# The ConceptGraphs-Detect
CLASS_SET=ram
python scripts/generate_gsa_results.py \
--dataset_root $REPLICA_ROOT \
--dataset_config $REPLICA_CONFIG_PATH \
--scene_id $SCENE_NAME \
--class_set $CLASS_SET \
--box_threshold 0.2 \
--text_threshold 0.2 \
--stride 5 \
--add_bg_classes \
--accumu_classes \
--exp_suffix withbg_allclasses
```
The above commands will save the detection and segmentation results in `$REPLICA_ROOT/$SCENE_NAME/`.
The visualization of the detection and segmentation can be viewed in `$REPLICA_ROOT/$SCENE_NAME/gsa_vis_none` and `$REPLICA_ROOT/$SCENE_NAME/gsa_vis_ram_withbg_allclasses` respectively.
You can ignore the `There's a wrong phrase happen, this is because of our post-process merged wrong tokens, which will be modified in the future. We will assign it with a random label at this time.` message for now.
### Run the 3D object mapping system
The following command builds an object-based 3D map of the scene, using the image segmentation results from above.
* Use `save_objects_all_frames=True` to save the mapping results at every frame, which can be used for animated visualization by `scripts/animate_mapping_interactive.py` and `scripts/animate_mapping_save.py`.
* Use `merge_interval=20 merge_visual_sim_thresh=0.8 merge_text_sim_thresh=0.8` to also perform overlap-based merging during the mapping process.
```bash
# Using the CoceptGraphs (without open-vocab detector)
THRESHOLD=1.2
python slam/cfslam_pipeline_batch.py \
dataset_root=$REPLICA_ROOT \
dataset_config=$REPLICA_CONFIG_PATH \
stride=5 \
scene_id=$SCENE_NAME \
spatial_sim_type=overlap \
mask_conf_threshold=0.95 \
match_method=sim_sum \
sim_threshold=${THRESHOLD} \
dbscan_eps=0.1 \
gsa_variant=none \
class_agnostic=True \
skip_bg=True \
max_bbox_area_ratio=0.5 \
save_suffix=overlap_maskconf0.95_simsum${THRESHOLD}_dbscan.1_merge20_masksub \
merge_interval=20 \
merge_visual_sim_thresh=0.8 \
merge_text_sim_thresh=0.8
# On the ConceptGraphs-Detect
SCENE_NAME=room0
THRESHOLD=1.2
python slam/cfslam_pipeline_batch.py \
dataset_root=$REPLICA_ROOT \
dataset_config=$REPLICA_CONFIG_PATH \
stride=5 \
scene_id=$SCENE_NAME \
spatial_sim_type=overlap \
mask_conf_threshold=0.25 \
match_method=sim_sum \
sim_threshold=${THRESHOLD} \
dbscan_eps=0.1 \
gsa_variant=ram_withbg_allclasses \
skip_bg=False \
max_bbox_area_ratio=0.5 \
save_suffix=overlap_maskconf0.25_simsum${THRESHOLD}_dbscan.1
```
The above commands will save the mapping results in `$REPLICA_ROOT/$SCENE_NAME/pcd_saves`. It will create two `pkl.gz` files, where the one with `_post` suffix indicates results after some post processing, which we recommend using.`
If you run the above command with `save_objects_all_frames=True`, it will create a folder in `$REPLICA_ROOT/$SCENE_NAME/objects_all_frames`. Then you can run the following command to visualize the mapping process or save it to a video. Also see the relevant files for available key callbacks for viusalization options.
```
python scripts/animate_mapping_interactive.py --input_folder $REPLICA_ROOT/$SCENE_NAME/objects_all_frames/<folder_name>
python scripts/animate_mapping_save.py --input_folder $REPLICA_ROOT/$SCENE_NAME/objects_all_frames/<folder_name>
```
### Visualize the object-based mapping results
```bash
python scripts/visualize_cfslam_results.py --result_path /path/to/output.pkl.gz
```
Then in the open3d visualizer window, you can use the following key callbacks to change the visualization.
* Press `b` to toggle the background point clouds (wall, floor, ceiling, etc.). Only works on the ConceptGraphs-Detect.
* Press `c` to color the point clouds by the object class from the tagging model. Only works on the ConceptGraphs-Detect.
* Press `r` to color the point clouds by RGB.
* Press `f` and type text in the terminal, and the point cloud will be colored by the CLIP similarity with the input text.
* Press `i` to color the point clouds by object instance ID.
### Evaluate semantic segmentation from the object-based mapping results on Replica datasets
First, download the GT point cloud with per-point semantic segmentation labels from this [Google Drive link](https://drive.google.com/file/d/1NhQIM5PCH5L5vkZDSRq6YF1bRaSX2aem/view?usp=sharing). Please refer to [this issue](https://github.com/concept-graphs/concept-graphs/issues/18#issuecomment-1876673985) for a brief description of how they are generated. Unzip the file and record its location in `REPLICA_SEMANTIC_ROOT`.
Then run the following command to evaluate the semantic segmentation results. The results will be saved in the `results` folder, where the mean recall `mrecall` is the mAcc and `fmiou` is the F-mIoU reported in the paper.
```bash
# CoceptGraphs (without open-vocab detector)
python scripts/eval_replica_semseg.py \
--replica_root $REPLICA_ROOT \
--replica_semantic_root $REPLICA_SEMANTIC_ROOT \
--n_exclude 6 \
--pred_exp_name none_overlap_maskconf0.95_simsum1.2_dbscan.1_merge20_masksub
# On the ConceptGraphs-Detect (Grounding-DINO as the object detector)
python scripts/eval_replica_semseg.py \
--replica_root $REPLICA_ROOT \
--replica_semantic_root $REPLICA_SEMANTIC_ROOT \
--n_exclude 6 \
--pred_exp_name ram_withbg_allclasses_overlap_maskconf0.25_simsum1.2_dbscan.1_masksub
```
### Extract object captions and build scene graphs
Ensure that the `openai` package is installed and that your APIKEY is set. We recommend using GPT-4, since GPT-3.5 often produces inconsistent results on this task.
```bash
export OPENAI_API_KEY=<your GPT-4 API KEY here>
```
Also note that if you are using the same [commit](https://github.com/haotian-liu/LLaVA/tree/8fc54a09a6be74b2abd913c468fb3d42ae826194) as we did, you may need to
plans/PRISM/.research/readmes/conceptfusion.md
+218
View File
@@ -0,0 +1,218 @@
<!-- src: https://raw.githubusercontent.com/concept-fusion/concept-fusion/main/README.md -->
# ConceptFusion: Open-set Multimodal 3D Mapping
> Code release for our [RSS 2023](https://roboticsconference.org/) publication
[Project page](https://concept-fusion.github.io/) | [Video explainer](https://www.youtube.com/watch?v=rkXgws8fiDs) | [arXiv](https://arxiv.org/abs/2302.07241)
[Krishna Murthy Jatavallabhula](https://krrish94.github.io/), [Alihusein Kuwajerwala](https://www.alihkw.com/), [Qiao Gu](https://georgegu1997.github.io/), [Mohd Omama](https://scholar.google.com/citations?user=jFH3ShsAAAAJ&hl=en), [Tao Chen](https://taochenshh.github.io/), [Alaa Maalouf](https://www.csail.mit.edu/person/alaa-maalouf), [Shuang Li](https://people.csail.mit.edu/lishuang/), [Ganesh Iyer](https://epiception.github.io/), [Soroush Saryazdi](https://saryazdi.github.io/), [Nikhil Keetha](https://nik-v9.github.io/), [Ayush Tewari](https://ayushtewari.com/), [Joshua B. Tenenbaum](http://web.mit.edu/cocosci/josh.html), [Celso Miguel de Melo](https://celsodemelo.net/), [Madhava Krishna](https://robotics.iiit.ac.in/), [Liam Paull](http://liampaull.ca/), [Florian Shkurti](http://www.cs.toronto.edu//~florian/), [Antonio Torralba](https://groups.csail.mit.edu/vision/torralbalab/)
<p align="center">
<img src="assets/splash.gif" />
</p>
> **Note**: WIP repo with the following key deviations from the ConceptFusion paper
1. Employ the segment anything model (SAM) opposed to Mask2Former to generate mask proposals
2. Remove the mask-to-mask similarity term (uniqueness; Eq. 4) -- needed for Mask2Former, but SAM seems to work okay without
3. (TODO) Add parser and download links for UnCoCo data
## Prerequisites
We recommend setting up a python virtualenv or conda environment to help manage dependencies. Our code has been tested primarily with Python 3.10 (although this should technically work with Python 3.8 with minimal modifications).
Sample instruction for `conda` users.
```
conda create -n conceptfusion python=3.10.8
conda activate conceptfusion
```
**Pytorch**: Install PyTorch using an appropriate Python-CUDA-CuDNN config from the [pytorch webpage](https://pytorch.org/).
**gradslam**: Install the `conceptfusion` branch of [gradlsam](https://gradslam.github.io/) by following these instructions (Note: the `main` branch does not have the feature fusion functionality, and will therefore, not work).
```
git clone https://github.com/gradslam/gradslam.git
cd gradslam
git checkout conceptfusion
pip install -e .
```
**segment-anything**: Install `segment-anything` by following [instructions here](https://github.com/facebookresearch/segment-anything).
**openclip**: Install `openclip` following [instructions here](https://github.com/mlfoundations/open_clip).
**(Optional) OpenAI CLIP**: If interested in using the OpenAI CLIP models, install `clip`.
```
pip install git+https://github.com/openai/CLIP.git
```
Note, however, that our released code isn't set up to use these CLIP models, and may require a few low-effort edits.
## Download and extract datasets
Depending on the dataset you would like to use, download and set it up for gradslam. To extend our (general-enough) dataset class to your own dataset, we recommend looking into the `gradslam` package (again, the `conceptfusion` branch), particularly `gradslam/datasets/` directory. A number of datasets have already been implemented.
### ICL dataset
Download it from [here](https://www.doc.ic.ac.uk/~ahanda/VaFRIC/iclnuim.html).
For the first two "scenes", i.e. living rooms 'lr kt0' and 'lr kt1', we want to download the files from these two links:
- "TUM RGB-D Compatible PNGs"
- "Global Poses [R | t]: Global_RT_Trajectory_GT" files.
### ScanNet and other datasets
**TODO**
## Setup models and download checkpoints
By default, the commandline arguments are setup to run conceptfusion feature extraction (i.e., CLIP features from an `openclip` model). If you would like to use DINO or LSeg features instead, follow these setup instructions. Else, this section may safely be ignored.
### DINO and/or LSeg models
From the dino repo [here](https://github.com/facebookresearch/dino) get the "ViT-B/8 backbone only" checkpoint
From the Lseg-minimal repo [here](https://github.com/krrish94/lseg-minimal#download-pretrained-weights) , get the checkpoint from the onedrive link.
Put these files in the checkpoints folder:
`/your/path/to/concept-fusion/examples/checkpoints/`
Full structure:
```
├── checkpoints
│ ├── dino_vitbase8_pretrain.pth
│ └── lseg_minimal_e200.ckpt
```
Also, clone and setup the `lseg-minimal` and `dino-minimal` repos -- used in running the pretrained networks for feature extraction. (These can be installed anywhere in your env, and do not have to be within the `concept-fusion` directory)
```
cd /path/to/where/you/keep/repos
git clone https://github.com/krrish94/lseg-minimal
cd lseg-minimal
python setup.py build develop
```
> Download the pretrained weights for the model(s) used in the `lseg-minimal` repo (instructions/links in readme)
```
cd /path/to/where/you/keep/repos
git clone https://github.com/krrish94/dino-minimal
cd dino-minimal
python setup.py build develop
```
> The pretrained weights for DINO will automatically be downloaded by this library when you run it for the first time
## Extract ConceptFusion features
To extract pixel-aligned CLIP features from a `GradSLAMDataset`, run
```
cd examples
python extract_conceptfusion_features.py
```
This script can parse any dataset compatible with the `GradSLAMDataset` format. It extracts mask proposals from SAM, computes CLIP features per-mask (and for the full image), and applies the pixel-aligned feature extraction scheme proposed in the paper (with the caveats at the top of this README).
The extracted features are saved in the `saved-feat` directory by default (this can be overridden by passing a `--feat_dir` argument).
## (Optional) Extract DINO and/or LSeg features
To extract features from other models like DINO or LSeg, run `run_feature_fusion_and_save_map.py` with `--mode extract` (and other flags as appropriate; importantly `--checkpoint_path`).
## Fuse extracted features to a 3D map
After extracting features, fuse them to 3D by running
```
python run_feature_fusion_and_save_map.py
```
This script fuses the extracted features into a 3D pointcloud map, and saved this by default to the `saved-map` directory.
## Try out the examples (and interactive demos)
### Interactive demo - click a point on the scene; visualize self similar points
```
python demo_click_query.py --load_path saved-map
```
This script will load the map saved in the `saved-map` directory. An Open3D window will pop up, where you can click on a point (`SHIFT + LEFT_MOUSE_BUTTON`). (while you can technically click multiple points, we discard all but the first clicked point). The script will then plot a similarity heatmap indicating all other scene points and their similarities, visualized in a `jet` colormap (red => higher similarity; blue => lower similarity).
### Interactive demo - text query
```
python demo_text_query.py --load_path saved-map
```
This script will load the map saved in the `saved-map` directory. You may type in a text query on the console (or press `q` to quit), and a similarity map will be displayed (again, using a `jet` colormap).
### TODO
(TODO - add K-means clustering demo)
If interested in running the K-Means clustering demo, you will need to install `fast-pytorch-kmeans`
```
pip install fast-pytorch-kmeans
```
## Outdated instructions
The instructions that follow are outdated, but are retained here, to help understand typical commandline arguments for other datasets such as ScanNet.
### Examples on use run_feature_fusion_and_save_map.py
The fourth step should also work on OpenSeg by setting `--model_type ovseg`.
```
# 0. Change these settings to your own path
SCENE_ID=scene0568_00
SCANNET_ROOT=/home/qiao/data/scannet/scans
DIR_FEAT=/home/qiao/data/scannet/results/${SCENE_ID}-lseg-0-500
DIR_SAVE_MAP=/home/qiao/data/scannet/results/saved-maps-${SCENE_ID}-lseg-0-500
DIR_SAVE_GT=/home/qiao/data/scannet/results/saved-maps-gt-${SCENE_ID}-lseg-0-500
DIR_SAVE_METRICS=/home/qiao/data/scannet/results/metrics-${SCENE_ID}-lseg-0-500
# 1. Extract feature map for each frame and save them to disk
python run_feature_fusion_and_save_map.py --mode extract --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 480 --image_width 640 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --checkpoint_path checkpoints/lseg_minimal_e200.ckpt
# 2. Load the saved feature map, fuse them and save the result to disk
python run_feature_fusion_and_save_map.py --mode fusion --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --checkpoint_path checkpoints/lseg_minimal_e200.ckpt
# 3. Fuse the GT semantic labels, get per-point GT classification and save them to disk
python run_scannet_feature_fusion_and_save_map.py --mode fusion-gt --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_gt $DIR_SAVE_GT --checkpoint_path checkpoints/lseg_minimal_e200.ckpt
# 4. Evaluate the result and compute metrics (4 variants below). Remember to change DIR_SAVE_METRICS to your own path.
## 4-1. Use the text embeddings as the query features
python run_scannet_feature_fusion_and_save_map.py --mode metrics --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --dir_to_save_gt $DIR_SAVE_GT --dir_to_save_metrics $DIR_SAVE_METRICS --checkpoint_path checkpoints/lseg_minimal_e200.ckpt
## 4-2. Use the feature means (Oracle) as the query features
DIR_SAVE_METRICS=/home/qiao/data/scannet/results/metrics-oracle-${SCENE_ID}-lseg-0-500
python run_scannet_feature_fusion_and_save_map.py --mode metrics --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --dir_to_save_gt $DIR_SAVE_GT --dir_to_save_metrics $DIR_SAVE_METRICS --checkpoint_path checkpoints/lseg_minimal_e200.ckpt --query_feat oracle
## 4-3. Use the feature means of random 1 points as the query features (multiple runs recommended)
DIR_SAVE_METRICS=/home/qiao/data/scannet/results/metrics-rand1p-${SCENE_ID}-lseg-0-500
python run_scannet_feature_fusion_and_save_map.py --mode metrics --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --dir_to_save_gt $DIR_SAVE_GT --dir_to_save_metrics $DIR_SAVE_METRICS --checkpoint_path checkpoints/lseg_minimal_e200.ckpt --query_feat random --n_point_query 1
## 4-4. Use the feature means of random 3 points as the query features (multiple runs recommended)
DIR_SAVE_METRICS=/home/qiao/data/scannet/results/metrics-rand3p-${SCENE_ID}-lseg-0-500
python run_scannet_feature_fusion_and_save_map.py --mode metrics --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --dir_to_save_gt $DIR_SAVE_GT --dir_to_save_metrics $DIR_SAVE_METRICS --checkpoint_path checkpoints/lseg_minimal_e200.ckpt --query_feat random --n_point_query 3
```
For 4-3 and 4-4, it would be better to run the commands multiple times with different seeds and average the results. For example:
```
DIR_SAVE_METRICS=/home/qiao/data/scannet/results/metrics-rand3p-${SCENE_ID}-lseg-0-500
for SEED in {1..10}
do
python run_scannet_feature_fusion_and_save_map.py --mode metrics --model_type lseg --dataconfig_path dataconfigs/scannet/${SCENE_ID}.yaml --dataset_path $SCANNET_ROOT --sequence ${SCENE_ID} --image_height 240 --image_width 320 --frame_start 0 --frame_end 500 --stride 25 --desired_feature_height 240 --desired_feature_width 320 --feat_dir $DIR_FEAT --dir_to_save_map $DIR_SAVE_MAP --dir_to_save_gt $DIR_SAVE_GT --dir_to_save_metrics $DIR_SAVE_METRICS --checkpoint_path checkpoints/lseg_minimal_e200.ckpt --query_feat random --n_point_query 3 --seed $SEED
done
```
plans/PRISM/.research/readmes/hov-sg.md
+314
View File
@@ -0,0 +1,314 @@
<!-- src: https://raw.githubusercontent.com/hovsg/HOV-SG/main/README.md -->
# HOV-SG
[![Static Badge](https://img.shields.io/badge/-arXiv-B31B1B?logo=arxiv)](https://arxiv.org/abs/2403.17846)
[![Static Badge](https://img.shields.io/badge/Project-Page-a)](https://hovsg.github.io/)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
[![Static Badge](https://img.shields.io/badge/-Video-FF0000?logo=youtube)](https://hovsg.github.io/static/images/hovsg_rss_final.mp4)
This repository is the official implementation of the paper:
> **Hierarchical Open-Vocabulary 3D Scene Graphs for Language-Grounded Robot Navigation**
>
> [Abdelrhman Werby]()&ast;, [Chenguang Huang](http://www2.informatik.uni-freiburg.de/~huang/)&ast;, [Martin Büchner](https://rl.uni-freiburg.de/people/buechner)&ast;, [Abhinav Valada](https://rl.uni-freiburg.de/people/valada), and [Wolfram Burgard](https://www.utn.de/person/wolfram-burgard/). <br>
> &ast;Equal contribution. <br>
>
> *arXiv preprint arXiv:2403.17846*, 2024 <br>
> (Accepted for *Robotics: Science and Systems (RSS), Delft, Netherlands*, 2024.)
<p align="center">
<img src="media/teaser-hovsg-white.png" alt="HOV-SG allows the construction of accurate, open-vocabulary 3D scene graphs for large-scale and multi-story environments and enables robots to effectively navigate in them with language instructions." width="600" />
</p>
## 📰 Major Updates
- **[29 Aug 2024]** **We added `hm3dsem_walks` dataset generation and hierarchical scene graph evaluation code.** <br>
Please review the updated code structure and newly added dependencies for dataset construction. <br><br>
- [01 Jul 2024] Initial release of HOV-SG including mapping and graph construction engine.
## 🏗 Setup
1. Clone and set up the HOV-SG repository
```bash
git clone https://github.com/hovsg/HOV-SG.git
cd HOV-SG
# set up virtual environment and install habitat-sim afterwards separately to avoid errors.
conda env create -f environment.yaml
conda activate hovsg
conda install habitat-sim -c conda-forge -c aihabitat
# set up the HOV-SG python package
pip install -e .
```
### OpenCLIP
HOV-SG uses the Open CLIP model to extract features from RGB-D frames. To download the Open CLIP model checkpoint `CLIP-ViT-H-14-laion2B-s32B-b79K` please refer to [Open CLIP](https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K).
```bash
mkdir checkpoints
wget https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/resolve/main/open_clip_pytorch_model.bin?download=true -O checkpoints/temp_open_clip_pytorch_model.bin && mv checkpoints/temp_open_clip_pytorch_model.bin checkpoints/laion2b_s32b_b79k.bin
```
Another option is to use the OVSeg fine-tuned Open CLIP model, which is available under [here](https://github.com/facebookresearch/ov-seg):
```bash
pip install gdown
gdown --fuzzy https://drive.google.com/file/d/17C9ACGcN7Rk4UT4pYD_7hn3ytTa3pFb5/view -O checkpoints/ovseg_clip.pth
```
### SAM
HOV-SG uses [SAM](https://github.com/facebookresearch/segment-anything) to generate class-agnostic masks for the RGB-D frames. To download the SAM model checkpoint `sam_v2` execute the following:
```bash
wget https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth -O checkpoints/sam_vit_h_4b8939.pth
```
## 🖼️ Dataset Preparation
### Habitat Matterport 3D Semantics
HOV-SG takes posed RGB-D sequences as input. In order to produce hierarchical multi-story scenes we make use of the Habitat 3D Semantics dataset ([HM3DSem](https://aihabitat.org/datasets/hm3d-semantics/)).
- Download the [Habitat Matterport 3D Semantics](https://github.com/matterport/habitat-matterport-3dresearch) dataset. More specifically, download through the links corresponding to these filenames: [hm3d-val-habitat-v0.2.tar](https://api.matterport.com/resources/habitat/hm3d-val-habitat-v0.2.tar), [hm3d-val-semantic-annots-v0.2.tar](https://api.matterport.com/resources/habitat/hm3d-val-semantic-annots-v0.2.tar), [hm3d-val-semantic-configs-v0.2.tar]( https://api.matterport.com/resources/habitat/hm3d-val-semantic-configs-v0.2.tar).
<details>
<summary>Make sure that the raw HM3D dataset has the following structure:</summary>
```
├── hm3d
│ ├── hm3d_annotated_basis.scene_dataset_config.json # this file is necessary
│ ├── val
│ │ └── 00824-Dd4bFSTQ8gi
│ │ ├── Dd4bFSTQ8gi.basis.glb
│ │ ├── Dd4bFSTQ8gi.basis.navmesh
│ │ ├── Dd4bFSTQ8gi.glb
│ │ ├── Dd4bFSTQ8gi.semantic.glb
│ │ └── Dd4bFSTQ8gi.semantic.txt
...
...
...
```
</details>
We used the following scenes from the Habitat Matterport 3D Semantics dataset in our evaluation:
<details>
<summary>Show Scenes ID</summary>
1. `00824-Dd4bFSTQ8gi`
2. `00829-QaLdnwvtxbs`
3. `00843-DYehNKdT76V`
4. `00861-GLAQ4DNUx5U`
5. `00862-LT9Jq6dN3Ea`
6. `00873-bxsVRursffK`
7. `00877-4ok3usBNeis`
8. `00890-6s7QHgap2fW`
</details>
1. Our method requires posed input data. Because of that, we recorded trajectories for each sequence we evaluate on. We provide a script (`hovsg/data/hm3dsem/gen_hm3dsem_walks_from_poses.py`) that turns a set of camera poses (`hovsg/data/hm3dsem/metadata/poses`) into a sequence of RGB-D observations using the [habitat-sim](https://github.com/facebookresearch/habitat-sim) simulator. The output includes RGB, depth, poses and frame-wise semantic/panoptic ground truth:
```bash
python data/habitat/gen_hm3dsem_from_poses.py --dataset_dir <hm3dsem_dir> --save_dir data/hm3dsem_walks/
```
2. Secondly, we construct a new hierarchical graph-structured dataset that is called `hm3dsem_walks` that includes ground truth based on all observations recorded. To produce this ground-truth data please execute the following: First, define the following config paths: `main.package_path`, `main.dataset_path`, `main.raw_data_path`, and `main.save_path` under `config/create_graph.yaml`. For each scene, define the `main.scene_id`, `main.split`. Next, execute the following to obtain floor-, region-, and object-level ground truth data per scene. We utilize every recorded frame without skipping (see parameter `dataset.hm3dsem.gt_skip_frames`) and recommend 128 GB of RAM to compile this as the scenes differ in size:
```bash
cd HOV-SG
python hovsg/data/hm3dsem/create_hm3dsem_walks_gt.py
```
To evaluate semantic segmentation cababilities, we used [ScanNet](http://www.scan-net.org/) and [Replica](https://github.com/facebookresearch/Replica-Dataset).
### ScanNet
To get an RGBD sequence for ScanNet, download the ScanNet dataset from the [official website](http://www.scan-net.org/). The dataset contains RGB-D frames compressed as .sens files. To extract the frames, use the [SensReader/python](https://github.com/ScanNet/ScanNet/blob/master/SensReader/python).
We used the following scenes from the ScanNet dataset:
<details>
<summary>Show Scenes ID</summary>
1. `scene0011_00`
2. `scene0050_00`
2. `scene0231_00`
3. `scene0378_00`
4. `scene0518_00`
</details>
### Replica
To get an RGBD sequence for Replica, Instead of the original Replica dataset, download the scanned RGB-D trajectories of the Replica dataset provided by [Nice-SLAM](https://github.com/cvg/nice-slam). It contains rendered trajectories using the mesh models provided by the original Replica datasets.
Download the Replica RGB-D scan dataset using the downloading [script](https://github.com/cvg/nice-slam/blob/master/scripts/download_replica.sh) in [Nice-SLAM](https://github.com/cvg/nice-slam#replica-1).
```bash
wget https://cvg-data.inf.ethz.ch/nice-slam/data/Replica.zip -O data/Replica.zip && unzip data/Replica.zip -d data/Replica_RGBD && rm data/Replica.zip
```
To evaluate against the ground truth semantics labels, you also need also to download the original Replica dataset from the [Replica](https://github.com/facebookresearch/Replica-Dataset) as it contains the ground truth semantics labels as .ply files.
```bash
git clone https://github.com/facebookresearch/Replica-Dataset.git data/Replica-Dataset
chmod +x data/Replica-Dataset/download.sh && data/Replica-Dataset/download.sh data/Replica_original
```
We only used the following scenes from the Replica dataset:
<details>
<summary>Show Scenes ID</summary>
1. `office0`
2. `office1`
3. `office2`
4. `office3`
5. `office4`
6. `room0`
7. `room1`
8. `room2`
</details>
## 📂 Datasets file strutcre
The Data folder should have the following structure:
<details>
<summary>Show data folder structure</summary>
```
├── hm3dsem_walks
│ ├── val
│ │ ├── 00824-Dd4bFSTQ8gi
│ │ │ ├── depth
│ │ │ │ ├── Dd4bFSTQ8gi-000000.png
│ │ │ │ ├── ...
│ │ │ ├── rgb
│ │ │ │ ├── Dd4bFSTQ8gi-000000.png
│ │ │ │ ├── ...
│ │ │ ├── semantic
│ │ │ │ ├── Dd4bFSTQ8gi-000000.png
│ │ │ │ ├── ...
│ │ │ ├── pose
│ │ │ │ ├── Dd4bFSTQ8gi-000000.png
│ │ │ │ ├── ...
| | ├── 00829-QaLdnwvtxbs
| | ├── ..
├── Replica
│ ├── office0
│ │ ├── results
│ │ │ ├── depth0000.png
│ │ │ ├── ...
│ │ | ├── rgb0000.png
│ │ | ├── ...
│ │ ├── traj.txt
│ ├── office1
│ ├── ...
├── ScanNet
│ ├── scans
│ │ ├── scene0011_00
│ │ │ ├── color
│ │ │ │ ├── 0.jpg
│ │ │ │ ├── ...
│ │ │ ├── depth
│ │ │ │ ├── 0.png
│ │ │ │ ├── ...
│ │ │ ├── poses
│ │ │ │ ├── 0.txt
│ │ │ │ ├── ...
│ │ │ ├── internsics
│ │ │ │ ├── intrinsics_color.txt
│ │ │ │ ├── intrinsics_depth.txt
│ │ ├── ..
```
</details>
## :rocket: Run
### Create scene graphs (only for Habitat Matterport 3D Semantics):
```bash
python application/create_graph.py main.dataset=hm3dsem main.dataset_path=data/hm3dsem_walks/val/00824-Dd4bFSTQ8gi/ main.save_path=data/scene_graphs/00824-Dd4bFSTQ8gi
```
<details>
<summary>This will generate a scene graph for the specified RGB-D sequence and save it. The following files are generated:</summary>
```
├── graph
│ ├── floors
│ │ ├── 0.json
│ │ ├── 0.ply
│ │ ├── 1.json
│ │ ├── ...
│ ├── rooms
│ │ ├── 0_0.json
│ │ ├── 0_0.ply
│ │ ├── 0_1.json
│ │ ├── ...
│ ├── objects
│ │ ├── 0_0_0.json
│ │ ├── 0_0_0.ply
│ │ ├── 0_0_1.json
│ │ ├── ...
│ ├── nav_graph
├── tmp
├── full_feats.pt
├── mask_feats.pt
├── full_pcd.ply
├── masked_pcd.ply
```
The `graph` folder contains the generated scene graph hierarchy, the first number in the file name represents the floor number, the second number represents the room number, and the third number represents the object number. The `tmp` folder holds intermediate results obtained throughout graph construction. The `full_feats.pt` and `mask_feats.pt` contain the features extracted from the RGBD frames using the Open CLIP and SAM models. the former contains per point features and the latter contains the features for the object masks. The `full_pcd.ply` and `masked_pcd.ply` contain the point cloud representation of the RGB-D frames and the instance masks of all objects, respectively.
</details>
### Visualize scene graph
```bash
python application/visualize_graph.py graph_path=data/scene_graphs/hm3dsem/00824-Dd4bFSTQ8gi/graph
```
![hovsg_graph_vis](media/hovsg_graph_vis.gif)
### Interactive visualization of scene graphs and natural language queries
#### Setup OpenAI
In order to test graph queries with HOV-SG, you need to setup an OpenAI API account with the following steps:
1. [Sign up an OpenAI account](https://openai.com/blog/openai-api), login your account, and bind your account with at least one payment method.
2. [Get you OpenAI API keys](https://platform.openai.com/account/api-keys), copy it.
3. Open your `~/.bashrc` file, paste a new line `export OPENAI_KEY=<your copied key>`, save the file, and source it with command `source ~/.bashrc`. Another way would be to run `export OPENAI_KEY=<your copied key>` in the teminal where you want to run the query code.
#### Evaluate query against pre-built hierarchical scene graph
```bash
python application/visualize_query_graph.py main.graph_path=data/scene_graphs/hm3dsem/00824-Dd4bFSTQ8gi/graph
```
After launching the code, you will be asked to input the hierarchical query. An example is `chair in the living room on floor 0`. You can see the visualization of the top 5 target objects and the room it lies in.
![hovsg_graph_query](media/hovsg_graph_query.gif)
### Extract feature map for semantic segmentation (only ScanNet and Replica)
```bash
python application/semantic_segmentation.py main.dataset=replica main.dataset_path=Replica/office0 main.save_path=data/sem_seg/office0
```
### Evaluate semantic segmentation (only ScanNet and Replica)
```bash
python application/eval/evaluate_sem_seg.py dataset=replica scene_name=office0 feature_map_path=data/sem_seg/office0
```
### Evaluate predicted scene graphs (only Habitat 3D Semantics)
- Define the scene identifiers and paths of ground truth and the predicted scene graph in the `config/eval_graph.yaml`.
- Run the graph evaluation method:
```bash
python application/eval/evaluate_graph.py
```
## 📔 Abstract
Recent open-vocabulary robot mapping methods enrich dense geometric maps with pre-trained visual-language features. While these maps allow for the prediction of point-wise saliency maps when queried for a certain language concept, largescale environments and abstract queries beyond the object level still pose a considerable hurdle, ultimately limiting languagegrounded robotic navigation. In this work, we present HOVSG, a hierarchical open-vocabulary 3D scene graph mapping approach for language-grounded indoor robot navigation. Leveraging open-vocabulary vision foundation models, we first obtain state-of-the-art open-vocabulary segment-level maps in 3D and subsequently construct a 3D scene graph hierarchy consisting of floor, room, and object concepts, each enriched with openvocabulary features. Our approach is able to represent multistory buildings and allows robotic traversal of those using a cross-floor Voronoi graph. HOV-SG is evaluated on three distinct datasets and surpasses previous baselines in open-vocabulary semantic accuracy on the object, room, and floor level while producing a 75% reduction in representation size compared to dense open-vocabulary maps. In order to prove the efficacy and generalization capabilities of HOV-SG, we showcase successful long-horizon language-conditioned robot navigation within realworld multi-story environments.
If you find our work useful, please consider citing our paper:
```
@article{werby23hovsg,
Author = {Abdelrhman Werby and Chenguang Huang and Martin Büchner and Abhinav Valada and Wolfram Burgard},
Title = {Hierarchical Open-Vocabulary 3D Scene Graphs for Language-Grounded Robot Navigation},
Year = {2024},
journal = {Robotics: Science and Systems},
}
```
## 👩‍⚖️ License
For academic usage, the code is released under the [MIT](https://opensource.org/licenses/MIT) license.
For any commercial purpose, please contact the authors.
## 🙏 Acknowledgment
This work was funded by the German Research Foundation
(DFG) Emmy Noether Program grant number 468878300, the
BrainLinks-BrainTools Center of the University of Freiburg,
and an academic grant from NVIDIA.
plans/PRISM/.research/readmes/ok-robot.md
+138
View File
@@ -0,0 +1,138 @@
[![Intro image](https://github.com/ok-robot/ok-robot/assets/156981262/5634372e-9820-4681-bb1a-1c14541c229c)](https://ok-robot.github.io)
# `OK-Robot`
[![arXiv](https://img.shields.io/badge/arXiv-2401.12202-163144.svg?style=for-the-badge)](https://arxiv.org/abs/2401.12202)
![License](https://img.shields.io/github/license/notmahi/bet?color=873a7e&style=for-the-badge)
[![Code Style: Black](https://img.shields.io/badge/Code%20Style-Black-262626?style=for-the-badge)](https://github.com/psf/black)
[![PyTorch](https://img.shields.io/badge/Videos-Website-db6a4b.svg?style=for-the-badge&logo=airplayvideo)](https://ok-robot.github.io/)
[![Discord](https://img.shields.io/discord/1210368035324436551?style=for-the-badge&logo=discord&color=5865F2&label=discord&logoColor=white)](https://discord.gg/wzzZJxqKYC)
**Authors**: [<u>Peiqi Liu</u>*](https://leo20021210.github.io/), [<u>Yaswanth Orru</u>*](https://www.linkedin.com/in/yaswanth-orru/), [<u>Jay Vakil</u>](https://www.linkedin.com/in/jdvakil/), [<u>Chris Paxton</u>](https://cpaxton.github.io/), [<u>Mahi Shafiuallah</u>](https://mahis.life/)<sup>†</sup>, [<u>Lerrel Pinto</u>](https://www.lerrelpinto.com/)<sup>†</sup>
\* equal contribution, † equal advising.
OK-Robot is a zero-shot modular framework that effectively combines the state-of-art navigation and manipulation models to perform pick and place tasks in real homes. It has been tested in 10 real homes on 170+ objects and achieved a total success rate of 58.5%.
https://github.com/ok-robot/ok-robot/assets/156981262/7221264b-2fa3-4008-a757-0604cdc04e2b
## Hardware and software requirements
Hardware required:
* An iPhone Pro with Lidar sensors
* [Hello Robot Stretch](https://hello-robot.com/) with Dex Wrist installed
* A workstation with GPU to run pretrained models
Software required:
* Python 3.9
* Record3D (>1.18.0)
* [CloudCompare](https://www.danielgm.net/cc/release/)
## Installation
* You need to get anygrasp [license and checkpoint](./ok-robot-manipulation/anygrasp_license_registration/README.md).
* [Install](./docs/workspace-installation.md) the necessary environment on workstation to run the navigation and manipulation modules
* [Verify the workspace installation](./docs/installation-verification.md) once the above steps are completed.
* [Install](./docs/robot-installation.md) the necessary packages on robot to be able to properly communicate with backend workstation.
* You might also need to get a [new calibrated URDF](./docs/robot-calibration.md) for accurate robot manipulation.
Once both the robot and workstation are complete. You are good to start the experiments.
## Run Experiments
First [set up the environment](./docs/environment-setup.md) with the tapes, position the robot properly and scan the environment to get a r3d file from Record3D. Place it in `/navigation/r3d/` run following commands.
### On Workstation:
In one terminal run the [Navigation Module](./ok-robot-navigation/).
```
mamba activate ok-robot-env
cd ok-robot-navigation
python path_planning.py debug=False min_height={z coordinates of the ground tapes + 0.1} dataset_path='r3d/{your_r3d_filename}.r3d' cache_path='{your_r3d_filename}.pt' pointcloud_path='{your_r3d_filename}.ply'
```
In another terminal run the [Manipulation module](./ok-robot-manipulation/README.md)
```
mamba activate ok-robot-env
cd ok-robot-manipulation/src
python demo.py --open_communication --debug
```
### On Robot:
Before running anything on the robot, you need to calibrate it by
```
stretch_robot_home.py
```
Our robot codes rely on robot controllers provided by [home-robot](https://github.com/facebookresearch/home-robot). Just like running other home-robot based codes, you need to run two processes synchronously in two terminals.
In one terminal start the home-robot
```
roslaunch home_robot_hw startup_stretch_hector_slam.launch
```
In another terminal run the robot control. More details in [ok-robot-hw](./ok-robot-hw/README.md)
```
cd ok-robot-hw
python run.py -x1 [x1] -y1 [y1] -x2 [x2] -y2 [y2] -ip [your workstation ip]
```
## Citation
If you find this work useful, please consider citing:
```bibtex
@article{liu2024okrobot,
title={OK-Robot: What Really Matters in Integrating Open-Knowledge Models for Robotics},
author={Liu, Peiqi and Orru, Yaswanth and Paxton, Chris and Shafiullah, Nur Muhammad Mahi and Pinto, Lerrel},
journal={arXiv preprint arXiv:2401.12202},
year={2024}
}
```
Our work is reliant on a lot of other publications and open source projects, if you find a particular component useful, please consider citing the original authors as well.
<details>
<summary>List of citations</summary>
```bibtex
@article{fang2023anygrasp,
title={Anygrasp: Robust and efficient grasp perception in spatial and temporal domains},
author={Fang, Hao-Shu and Wang, Chenxi and Fang, Hongjie and Gou, Minghao and Liu, Jirong and Yan, Hengxu and Liu, Wenhai and Xie, Yichen and Lu, Cewu},
journal={IEEE Transactions on Robotics},
year={2023},
publisher={IEEE}
}
@article{minderer2024scaling,
title={Scaling open-vocabulary object detection},
author={Minderer, Matthias and Gritsenko, Alexey and Houlsby, Neil},
journal={Advances in Neural Information Processing Systems},
volume={36},
year={2024}
}
@article{yenamandra2023homerobot,
title={HomeRobot: Open-Vocabulary Mobile Manipulation},
author={Yenamandra, Sriram and Ramachandran, Arun and Yadav, Karmesh and Wang, Austin and Khanna, Mukul and Gervet, Theophile and Yang, Tsung-Yen and Jain, Vidhi and Clegg, Alexander William and Turner, John and others},
journal={arXiv preprint arXiv:2306.11565},
year={2023}
}
```
</details>
## Roadmap
While OK-Robot can do quite a bit by itself, we think there are plenty of room for improvement for a zero-shot, home-dwelling robot. That's why we consider OK-Robot a living release, and will try to occassionally add new features to this. We also encourage you to take a look at the list below, and if you are interested, share your improvements with the community by contributing to this project.
- [x] Create OK-Robot, a shared platform for a zero-shot, open-vocab pick-and-place robot.
- [x] Integrate grasping primitive with AnyGrasp.
- [x] Integrate open-vocabulary navigation with VoxelMap.
- [x] Integrate heuristic based dropping.
- [ ] Improve documentation.
- [ ] Add error detection/recovery from failure while manipulating.
- [ ] Figure out interactive navigation: if an object is not found or a query is ambiguous, ask the end-user.
- [ ] Integrate with an open-source grasp perception model so that we can MIT-license all the dependencies.
plans/PRISM/.research/readmes/openmask3d.md
+193
View File
@@ -0,0 +1,193 @@
<!-- src: https://raw.githubusercontent.com/OpenMask3D/openmask3d/master/README.md -->
<p align="center">
<h1 align="center">OpenMask3D🛋: Open-Vocabulary 3D Instance Segmentation</h1>
<p align="center">
<a href="https://aycatakmaz.github.io/">Ay&#231;a Takmaz</a><sup>1*</sup></span>,
<a href="https://elisabettafedele.github.io/">Elisabetta Fedele</a><sup>1*</sup>
<br>
<a href="https://studios.disneyresearch.com/people/bob-sumner/">Robert W. Sumner</a><sup>1</sup>,
<a href="https://people.inf.ethz.ch/pomarc/">Marc Pollefeys</a><sup>1,2</sup>,
<a href="https://federicotombari.github.io/">Federico Tombari</a><sup>1,3</sup>,
<a href="https://francisengelmann.github.io/">Francis Engelmann</a><sup>1,3</sup>
<br>
<sup>1</sup>ETH Zurich,
<sup>2</sup>Microsoft,
<sup>3</sup>Google <br>
<sup>*</sup>equal contribution
</p>
<h2 align="center">NeurIPS 2023</h2>
<h3 align="center"><a href="https://github.com/OpenMask3D/openmask3d">Code</a> | <a href="https://arxiv.org/abs/2306.13631">Paper</a> | <a href="https://openmask3d.github.io">Project Page</a> </h3>
<div align="center"></div>
</p>
<p align="center">
<a href="">
<img src="https://openmask3d.github.io/static/images/teaser.jpeg" alt="Logo" width="100%">
</a>
</p>
<p align="center">
<strong>OpenMask3D</strong> is a zero-shot approach for 3D instance segmentation with open-vocabulary queries.
Guided by predicted class-agnostic 3D instance masks, our model aggregates per-mask features via multi-view fusion of CLIP-based image embeddings.
</p>
<br>
---
## Setup 🛠
Clone the repository, create conda environment and install the required packages as follows:
```bash
conda create --name=openmask3d python=3.8.5 # create new virtual environment
conda activate openmask3d # activate it
bash install_requirements.sh # install requirements
pip install -e . # install current repository in editable mode
```
Note: If you encounter any issues in the `bash install_requirements.sh` step, we recommend you to run the commands in that script one-by-one, especially for performing the MinkowskiEngine installation manually.
---
## Run the pipeline on a single scene 🛋
In this section we provide some information about how to run the pipeline on a single scene. In particular, we divide this section into four parts:
1. Download **checkpoints**
2. Check the format of **scene's data**
3. Set-up **configurations**
4. **Run** OpenMask3D
### Step 1: Download the checkpoints 📍
Create a folder `resources` in the main directory of the repository. Then, add to this folder the checkpoints for:
* **Mask module network**: use [this link](https://drive.google.com/file/d/1emtZ9xCiCuXtkcGO3iIzIRzcmZAFfI_B/view?usp=sharing) (model trained on ScanNet200 training set) for evaluating on **ScanNet validation scenes**, or [this link](https://drive.google.com/file/d/1rD2Uvbsi89X4lSkont_jUTT7X9iaox9y/view?usp=share_link) for running the model on an **arbitrary scene**.
* **Segment Anything Model** (in our case we used ViT-H): use this [link](https://drive.google.com/file/d/1WHi0hBi0iqMZfk8l3rDXLrW4lEEgHm_y/view?usp=sharing) or the [official repository](https://github.com/facebookresearch/segment-anything#model-checkpoints).
### Step 2: Check the folder structure of the data for your scene 🛢
In order to run OpenMask3D you need to have access to the point cloud of the scene as well to the posed RGB-D frames.
We recommend creating a folder `scene_example` inside the `resources` folder where the data is saved with the following structure ([here](https://drive.google.com/file/d/1UOwBZMCrTMg-_MFwmYkKOrex1YS6Nw-i/view?usp=sharing) we provide a scene as an example).
```
scene_example
├── pose <- folder with camera poses
│ ├── 0.txt
│ ├── 1.txt
│ └── ...
├── color <- folder with RGB images
│ ├── 0.jpg (or .png/.jpeg)
│ ├── 1.jpg (or .png/.jpeg)
│ └── ...
├── depth <- folder with depth images
│ ├── 0.png (or .jpg/.jpeg)
│ ├── 1.png (or .jpg/.jpeg)
│ └── ...
├── intrinsic
│ └── intrinsic_color.txt <- camera intrinsics
└── scene_example.ply <- point cloud of the scene
```
Please note the followings:
* The **point cloud** should be provided as a `.ply` file and the points are expected to be in the z-up right-handed coordinate system.
* The **camera intrinsics** and **camera poses** should be provided in a `.txt` file, containing a 4x4 matrix.
* The **RGB images** and the **depths** can be either in `.png`, `.jpg`, `.jpeg` format; the used format should be specified as explained in **Step 3**.
* The **RGB images** and their corresponding **depths** and **camera poses** should be named as `{FRAME_ID}.extension`, without zero padding for the frame ID, starting from index 0.
### Step 3: Set-up the paths to data and to output folders 🛤
Before running OpenMask3D make sure to fill all the required parameters in [this script](run_openmask3d_single_scene.sh). In particular, if you have followed the structure provided in Step 2, you should adapt only the following fields:
* `SCENE_DIR`: directory to `scene_example`
* `SCENE_INTRINSIC_RESOLUTION`: resolution on which intrinsics are computed
* `IMG_EXTENSION`: extension of RGB pictures. Either `.png`, `.jpg`, `.jpeg`
* `DEPTH_EXTENSION`: extension of depth pictures. Either `.png`, `.jpg`, `.jpeg`
* `DEPTH_SCALE`: factor by which the depth of the sensor should be divided to obtain a measure in terms of meters. It should be set to 1000 for ScanNet depth images and to 6553.5 for Replica depth images. You should set this value based on the scale of your depth maps.
* `MASK_MODULE_CKPT_PATH`: path to the mask module network checkpoint
* `SAM_CKPT_PATH`: path to the Segment Anything Model (SAM) checkpoint
* `OUTPUT_FOLDER_DIRECTORY`: path to the folder in which you wish to save the outputs
* `SAVE_VISUALIZATIONS`: set to true if you wish to save the visualizations of the class-agnostic masks
* `SAVE_CROPS`: set to true if you wish to save the 2D crops of the masks from which the CLIP features are extracted. It can be helpful for debugging and for a qualitative evaluation of the quality of the masks.
* `OPTIMIZE_GPU_USAGE`: set to true if you have some memory constraints and wish to minimize GPU memory footprint. Please note that this version is slower compared to the our default version.
### Step 4: Run OpenMask3D 🚀
Now you can run OpenMask3D by using the following command.
```bash
bash run_openmask3d_single_scene.sh
```
This script first extracts and saves the class-agnostic masks, and then computes the per-mask features. Masks and mask-features are saved into the directory specified by the user at the beginning of [this script](run_openmask3d_single_scene.sh). In particular, the output has the following structure.
```
OUTPUT_FOLDER_DIRECTORY
└── date-time-experiment_name <- folder with the output of a specific experiment
├── crops <- folder with crops (if SAVE_CROPS=true)
├── hydra_outputs <- folder with outputs from hydra (config.yaml files are useful)
├── scene_example_masks.pt <- class-agnostic instance masks - dim. (num_points, num_masks) indicating the masks in which a given point is included
└── scene_example_openmask3d_features.npy <- per-mask features for each object instance - dim. (num_masks, num_features), the mask-feature vecture for each instance mask.
```
Note: For the ScanNet validation, we use available segments on ScanNet and obtain more robust and less noisy masks compared to directly running the mask predictor on the point cloud. Therefore, the results we obtain for a single scene from ScanNet directly using the point cloud can be different then the masks obtained during the overall ScanNet evaluation described in the section below.
---
## Other Configs ⚙️
Other configuration parameters can be modified from [this file](openmask3d/configs/openmask3d_inference.yaml). Here we provide some clarifications of other configuration parameters:
- `multi_level_expansion_ratio`: factor of increment of the crop dimension for using multi-level image crops
- `openmask3d.frequency`: the frequency with which we want to process the frames given in input (e.g. a frequency of 10 takes 1 image in every 10 frames)
- `openmask3d.num_random_rounds` and `openmask3d.num_selected_points`: sets the number of iterations and the number of sampled points for SAM.
---
## Closed-vocabulary 3D instance segmentation evaluation on ScanNet200 📊
In this section we outline the steps to take in order to reproduce our results on the ScanNet200 validation set. In particular, we divide this section into four parts:
1. Download and preprocess the **ScanNet200** dataset
2. Check the format of ScanNet200 dataset
3. Set-up the paths to data and to output folders
2. Run evaluation
### Step 1: Download and pre-process the ScanNet200 dataset 📍
First, you need to download the ScanNet200 dataset as explained [here](https://kaldir.vc.in.tum.de/scannet_benchmark/documentation).
Once you have the dataset, you have to clone the [ScanNet repository](https://github.com/ScanNet/ScanNet) and process the dataset by using the following command.
```
cd class_agnostic_mask_computation
python -m datasets.preprocessing.scannet_preprocessing preprocess \
--data_dir="PATH_TO_ORIGINAL_SCANNET_DATASET" \
--save_dir="data/processed/scannet" \
--git_repo="PATH_TO_SCANNET_GIT_REPO" \
--scannet200=true
```
### Step 2: Check the format of ScanNet200 dataset 🛢
Make sure to have the data in the following form.
```bash
scans <- out folder
├── scene_0011_00
│ ├── data
│ │ ├── intrinsic <- folder with the intrinsics
│ │ └── pose <- folder with the poses
│ ├── data_compressed
│ │ ├── color <- folder with the color images
│ │ └── depth <- folder with the depth images
│ └── scene_0011_00_vh_clean_2.ply <- path to the point cloud/mesh ply file
├── scene0011_01
│ ├── data
│ │ ├── intrinsic
│ │ └── pose
│ ├── data_compressed
│ │ ├── color
│ │ └── depth
│ └── scene_0011_01_vh_clean_2.ply
...
```
### Step 3: Set-up paths to data and to output folders 🛤
Modify the paths and parameters in [this script](run_openmask3d_scannet200_eval.sh), following the instructions provided there.
### Step 4: Run OpenMask3D on ScanNet200 🚀
Now you can compute the per-mask scene features and run the evaluation of OpenMask3D on the whole ScanNet200 dataset by using the following command:
```bash
bash run_openmask3d_scannet200_eval.sh
```
This script first extracts and saves the class-agnostic masks, and then computes the mask features associated with each extracted mask. Afterwards, the evaluation script automatically runs in order to obtain 3D closed-vocabulary semantic instance segmentation scores.
---
## Citation :pray:
```
@inproceedings{takmaz2023openmask3d,
title={{OpenMask3D: Open-Vocabulary 3D Instance Segmentation}},
author={Takmaz, Ay{\c{c}}a and Fedele, Elisabetta and Sumner, Robert W. and Pollefeys, Marc and Tombari, Federico and Engelmann, Francis},
booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
year={2023}
}
```
plans/PRISM/.research/readmes/openscene.md
+250
View File
@@ -0,0 +1,250 @@
<!-- src: https://raw.githubusercontent.com/pengsongyou/openscene/main/README.md -->
<!-- PROJECT LOGO -->
<p align="center">
<h1 align="center"><img src="https://pengsongyou.github.io/media/openscene/logo.png" width="40">OpenScene: 3D Scene Understanding with Open Vocabularies</h1>
<p align="center">
<a href="https://pengsongyou.github.io"><strong>Songyou Peng</strong></a>
·
<a href="https://www.kylegenova.com/"><strong>Kyle Genova</strong></a>
·
<a href="https://www.maxjiang.ml/"><strong>Chiyu "Max" Jiang</strong></a>
·
<a href="https://taiya.github.io/"><strong>Andrea Tagliasacchi</strong></a>
<br>
<a href="https://people.inf.ethz.ch/pomarc/"><strong>Marc Pollefeys</strong></a>
·
<a href="https://www.cs.princeton.edu/~funk/"><strong>Thomas Funkhouser</strong></a>
</p>
<h2 align="center">CVPR 2023</h2>
<h3 align="center"><a href="https://arxiv.org/abs/2211.15654">Paper</a> | <a href="https://youtu.be/jZxCLHyDJf8">Video</a> | <a href="https://pengsongyou.github.io/openscene">Project Page</a></h3>
<div align="center"></div>
</p>
<p align="center">
<a href="">
<img src="https://pengsongyou.github.io/media/openscene/teaser.jpg" alt="Logo" width="100%">
</a>
</p>
<p align="center">
<strong>OpenScene</strong> is a zero-shot approach to perform a series of novel 3D scene understanding tasks using open-vocabulary queries.
</p>
<br>
<!-- TABLE OF CONTENTS -->
<details open="open" style='padding: 10px; border-radius:5px 30px 30px 5px; border-style: solid; border-width: 1px;'>
<summary>Table of Contents</summary>
<ol>
<li>
<a href="#interactive-demo">Interactive Demo</a>
</li>
<li>
<a href="#installation">Installation</a>
</li>
<li>
<a href="#data-preparation">Data Preparation</a>
</li>
<li>
<a href="#run">Run</a>
</li>
<li>
<a href="#applications">Applications</a>
</li>
<li>
<a href="#todo">TODO</a>
</li>
<li>
<a href="#acknowledgement">Acknowledgement</a>
</li>
<li>
<a href="#citation">Citation</a>
</li>
</ol>
</details>
## News :triangular_flag_on_post:
- [2023/10/27] Add the code for LSeg per-pixel feature extraction and multi-view fusion. Check [this repo](https://github.com/pengsongyou/lseg_feature_extraction).
- [2023/03/31] Code is released.
## Interactive Demo
### No GPU is needed! Follow **[this instruction](./demo)** to set up and play with the real-time demo yourself.
<p align="center">
<img src="./media/demo.gif" width="75%" />
</p>
Here we present a **real-time**, **interactive**, **open-vocabulary** scene understanding tool. A user can type in an arbitrary query phrase like **`snoopy`** (rare object), **`somewhere soft`** (property), **`made of metal`** (material), **`where can I cook?`** (activity), **`festive`** (abstract concept) etc, and the correponding regions are highlighted.
## Installation
Follow the [installation.md](installation.md) to install all required packages so you can do the evaluation & distillation afterwards.
## Data Preparation
We provide the **pre-processed 3D&2D data** and **multi-view fused features** for the following datasets:
- ScanNet
- Matterport3D
- nuScenes
- Replica
### Pre-processed 3D&2D Data
You can preprocess the dataset yourself, see the [data pre-processing instruction](scripts/preprocess/README.md).
Alternatively, we have provided the preprocessed datasets. One can download the pre-processed datasets by running the script below, and following the command line instruction to download the corresponding datasets:
```bash
bash scripts/download_dataset.sh
```
The script will download and unpack data into the folder `data/`. One can also download the dataset somewhere else, but link to the corresponding folder with the symbolic link:
```bash
ln -s /PATH/TO/DOWNLOADED/FOLDER data
```
<details>
<summary><strong>List of provided processed data</strong> (click to expand):</summary>
- ScanNet 3D (point clouds with GT semantic labels)
- ScanNet 2D (RGB-D images with camera poses)
- Matterport 3D (point clouds with GT semantic labels)
- Matterport 2D (RGB-D images with camera poses)
- nuScenes 3D (lidar point clouds with GT semantic labels)
- nuScenes 2D (RGB images with camera poses)
- Replica 3D (point clouds)
- Replica 2D (RGB-D images)
- Matterport 3D with top 40 NYU classes
- Matterport 3D with top 80 NYU classes
- Matterport 3D with top 160 NYU classes
</details>
**Note**: 2D processed datasets (e.g. `scannet_2d`) are only needed if you want to do multi-view feature fusion on your own. If so, please follow the [instruction for multi-view fusion](./scripts/feature_fusion/README.md).
### Multi-view Fused Features
To evaluate our OpenScene model or distill a 3D model, one needs to have the multi-view fused image feature for each 3D point (see method in Sec. 3.1 in the paper).
You can run the following to directly download provided fused features:
```bash
bash scripts/download_fused_features.sh
```
<details>
<summary><strong>List of provided fused features</strong> (click to expand):</summary>
- ScanNet - Multi-view fused OpenSeg features, train/val (234.8G)
- ScanNet - Multi-view fused LSeg features, train/val (175.8G)
- Matterport - Multi-view fused OpenSeg features, train/val (198.3G)
- Matterport - Multi-view fused OpenSeg features, test set (66.7G)
- Replica - Multi-view fused OpenSeg features (9.0G)
- Matterport - Multi-view fused LSeg features (coming)
- nuScenes - Multi-view fused OpenSeg features (coming)
- nuScenes - Multi-view fused LSeg features (coming)
</details>
Alternatively, you can also generate multi-view features yourself following the [instruction](./scripts/feature_fusion/README.md).
## Run
When you have installed the environment and obtained the **processed 3D data** and **multi-view fused features**, you are ready to run our OpenScene disilled/ensemble model for 3D semantic segmentation, or distill your own model from scratch.
### Evaluation for 3D Semantic Segmentation with a Pre-defined Labelsets
<p align="center">
<img src="./media/benchmark_screenshot.jpg" width="80%" />
</p>
Here you can evaluate OpenScene features on different dataset (ScanNet/Matterport3D/nuScenes/Replica) that have pre-defined labelsets.
We already include the following labelsets in [label_constants.py](dataset/label_constants.py):
- ScanNet 20 classes (`wall`, `door`, `chair`, ...)
- Matterport3D 21 classes (ScanNet 20 classes + `floor`)
- Matterport top 40, 80, 160 NYU classes (more rare object classes)
- nuScenes 16 classes (`road`, `bicycle`, `sidewalk`, ...)
The general command to run evaluation:
```bash
sh run/eval.sh EXP_DIR CONFIG.yaml feature_type
```
where you specify your experiment directory `EXP_DIR`, and replace `CONFIG.yaml` with the correct config file under [`config/`](./config/). **`feature_type`** corresponds to per-point OpenScene features:
- `fusion`: The 2D multi-view fused features
- `distill`: features from 3D distilled model
- `ensemble`: Our 2D-3D ensemble features
To evaluate with `distill` and `ensemble`, the easiest way is to use a pre-trained 3D distilled model. You can do this by using one of the config files with postfix `_pretrained`.
For example, to evaluate the semantic segmentation on Replica, you can simply run:
```bash
# 2D-3D ensemble
sh run/eval.sh out/replica_openseg config/replica/ours_openseg_pretrained.yaml ensemble
# Run 3D distilled model
sh run/eval.sh out/replica_openseg config/replica/ours_openseg_pretrained.yaml distill
# Evaluate with 2D fused features
sh run/eval.sh out/replica_openseg config/replica/ours_openseg_pretrained.yaml fusion
```
The script will automatically download the pretrained 3D model and run the evaluation for Matterport 21 classes.
You can find all outputs in the `out/replica_openseg`.
For evaluation options, see under `TEST` inside `config/replica/ours_openseg_pretrained.yaml`. Below are important evaluation options that you might want to modify:
- `labelset` (default: None, `scannet`| `matterport` | `matterport40`| `matterport80`|`matterport160`): Evaluate on a specific pre-defined labelset in [label_constants.py](./dataset/label_constants.py). If not specified, same as your 3D point cloud folder name
- `eval_iou` (default: True): whether evaluating the mIoU. Set to `False` if there is no GT labels
- `save_feature_as_numpy` (default: False): save the per-point features as `.npy`
- `prompt_eng` (default: True): input class name X -> "a X in a scene"
- `vis_gt` (default: True): visualize point clouds with GT semantic labels
- `vis_pred` (default: True): visualize point clouds with our predicted semantic labels
- `vis_input` (default: True): visualize input point clouds
If you want to use a 3D model distilled from scratch, specify the `model_path` to the correponding checkpoints `EXP/model/model_best.pth.tar`.
### Distillation
Finally, if you want to distill a new 3D model from scratch, run:
- Start distilling:
```sh run/distill.sh EXP_NAME CONFIG.yaml```
- Resume:
```sh run/resume_distill.sh EXP_NAME CONFIG.yaml```
For available distillation options, please take a look at `DISTILL` inside `config/matterport/ours_openseg.yaml`
### Using Your Own Datasets
1. Follow the [data preprocessing instruction](./scripts/preprocess/README.md), modify codes accordingly to obtain the processed 2D&3D data
2. Follow the [feature fusion instruction](./scripts/feature_fusion/README.md), modify codes to obtain multi-view fused features.
3. You can distill a model on your own, or take our provided 3D distilled model weights (e.g. our 3D model for ScanNet or Matterport3D), and modify the `model_path` accordingly.
4. If you want to evaluate on a specific labelset, change the `labelset` in config.
## Applications
Besides the zero-shot 3D semantic segmentation, we can perform also the following tasks:
- **Open-vocabulary 3D scene understanding and exploration**: query a 3D scene to understand properties that extend beyond fixed category labels, e.g. materials, activity, affordances, room type, abstract concepts...
- **Rare object search**: query a 3D scene database to find rare examples based on their names
- **Image-based 3D object detection**: query a 3D scene database to retrieve examples based on similarities to a given input image
## Acknowledgement
We sincerely thank Golnaz Ghiasi for providing guidance on using OpenSeg model. Our appreciation extends to Huizhong Chen, Yin Cui, Tom Duerig, Dan Gnanapragasam, Xiuye Gu, Leonidas Guibas, Nilesh Kulkarni, Abhijit Kundu, Hao-Ning Wu, Louis Yang, Guandao Yang, Xiaoshuai Zhang, Howard Zhou, and Zihan Zhu for helpful discussion. We are also grateful to Charles R. Qi and Paul-Edouard Sarlin for their proofreading.
We build some parts of our code on top of the [BPNet repository](https://github.com/wbhu/BPNet).
## TODO
- [ ] Support demo for arbitrary scenes
- [ ] Support in-webiste demo
- [x] Support multi-view feature fusion with LSeg
- [x] Add missing multi-view fusion LSeg feature for Matterport & nuScenes
- [x] Add missing multi-view fusion OpenSeg feature for nuScenes
- [x] Multi-view feature fusion code for nuScenes
- [ ] Support the latest PyTorch version
We are very much welcome all kinds of contributions to the project.
## Citation
If you find our code or paper useful, please cite
```bibtex
@inproceedings{Peng2023OpenScene,
title = {OpenScene: 3D Scene Understanding with Open Vocabularies},
author = {Peng, Songyou and Genova, Kyle and Jiang, Chiyu "Max" and Tagliasacchi, Andrea and Pollefeys, Marc and Funkhouser, Thomas},
booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)},
year = {2023}
```
plans/PRISM/00_overview.md
+173
View File
@@ -0,0 +1,173 @@
# Chapter 00 — 总览与命名由来
> **PRISM** = **P**rior-**R**egistered **I**ntegrated **S**patial **M**emory
> 一种把"离线扫描的先验地图"与"在线相机的实时感知"统一进机器人大脑的四层空间记忆架构。
---
## 0.1 我们要解决什么问题
机器人要在真实室内环境（酒店客房、家居、办公）里完成任务，它至少要回答 4 个问题：
| 问题 | 解决它需要的信息 | 现有单一传感器够吗 |
|------|------------------|---------------------|
| **Q1 我在哪？** | 全局坐标系下的位姿 | ❌ VIO 漂移；无先验无法重定位 |
| **Q2 我面前是什么？** | 当前视场内的几何与语义 | ✅ ZED 2i 自给 |
| **Q3 那个房间长什么样？** | 没去过的房间的结构 | ❌ 单凭实时相机做不到 |
| **Q4 那把椅子被搬过吗？** | 当下与历史的对比 | ❌ 需要"记忆"而非"感知" |
**单一传感器答不全 Q1+Q3+Q4。**
**PRISM 的答案**：用 iPhone 做一次性的"先验灌入"，用 ZED 2i 做持续的"在线感知"，两者通过**重定位握手**绑定到同一世界坐标系，并按**四层记忆**分层存放、互相校准。
---
## 0.2 一图看懂 PRISM
```mermaid
flowchart LR
subgraph IN[" "]
direction TB
IPHONE["📱 iPhone<br/>RoomPlan<br/><i>离线一次</i>"]
ZED["📷 ZED 2i<br/>VIO + Depth<br/><i>实时 30 Hz</i>"]
end
subgraph BRAIN["PRISM Brain"]
direction TB
L4["L4<br/>语义图"]
L3["L3<br/>拓扑图"]
L2["L2<br/>度量"]
L1["L1<br/>感知缓冲"]
REG["Registrar<br/>CLIP + ICP"]
CON["Consolidator<br/>（充电时跑）<br/>delta → 写回 LTM + 更新锚点"]
L4 <--> L3
L3 <--> L2
L2 <--> L1
L4 --> REG
L3 --> REG
L2 --> REG
L1 --> REG
end
subgraph OUT[" "]
direction TB
AGENT["🤖 Agent / LLM"]
ACT["Plan / Act"]
AGENT -- 查询 --> ACT
end
IPHONE --> L4
ZED --> REG
L4 --> AGENT
L3 --> AGENT
L2 --> AGENT
L1 --> AGENT
style BRAIN fill:#f6f7fb,stroke:#444,stroke-width:1px
style REG fill:#ffe9b3,stroke:#c97a00
style CON fill:#d4f0d4,stroke:#2e7d32
style IPHONE fill:#e3f2fd,stroke:#1565c0
style ZED fill:#e3f2fd,stroke:#1565c0
```
- **左**：两路输入（iPhone 偶发、ZED 2i 持续）
- **中**：四层记忆 + 配准器 (Registrar) + 巩固器 (Consolidator)
- **右**：上层 Agent 通过统一 API 查询记忆
---
## 0.3 为什么叫 PRISM
| 字母 | 含义 | 在架构中的角色 |
|------|------|----------------|
| **P** Prior | 先验 | iPhone RoomPlan 灌入的离线知识——机器人没去过也"知道" |
| **R** Registered | 已配准 | 实时观测必须先**和先验对齐**才能写入大脑（CLIP→ICP） |
| **I** Integrated | 融合 | 两路数据共享同一 `SpatialMemory` Schema，不分库 |
| **S** Spatial | 空间 | 表达对象是 3D 空间，而非纯文本/纯图像 |
| **M** Memory | 记忆 | 四层 + 巩固机制，明确区分"短期/长期/语义" |
**类比**：
- 棱镜把**白光分成七色** ↔ PRISM 把**异构观测分到四层**
- 棱镜也能把**七色合回白光** ↔ 查询时四层信息汇聚为决策
- **不可逆性**：先验是"宪法"，在线感知是"补丁"——区分明确才能避免互相污染
---
## 0.4 PRISM 的三条核心设计原则
### 原则 1：**分层而非分库**
不要给 iPhone 数据建一个数据库、ZED 2i 数据建另一个，那样查询时会困惑。
所有数据进入**同一个 `SpatialMemory`**，靠 `level ∈ {L1,L2,L3,L4}` 和 `source ∈ {iphone,zed2i,vlm,fused}` 两个字段区分。
### 原则 2：**先验是宪法，实时是补丁**
iPhone 写入的内容默认**只读、高置信**；ZED 2i 发现的变化先写到 `delta/`（补丁区），经多次确认 + 巩固才允许修改 LTM。
这避免"今天客人坐了沙发 → 机器人以为沙发被搬走"这类一次性观测带来的灾难性遗忘。
### 原则 3：**记忆要老化、要巩固**
模仿人脑：
- **工作记忆 (L1)** 几秒内被覆盖
- **短期记忆 (delta/)** 几小时到几天
- **长期记忆 (LTM)** 月级稳定
- 充电时 (`Consolidator`) 像"睡眠"把短期转长期，并淘汰旧锚点
---
## 0.5 一句话定义 PRISM
> **PRISM 是一个把"地图扫描 (iPhone)"和"具身感知 (ZED 2i)"分别落到"长期记忆"和"工作记忆"，再用配准握手与巩固机制粘合的，面向室内服务机器人的空间记忆操作系统。**
---
## 0.6 PRISM 不做什么（边界说明）
为了避免范围发散：
| 不做 | 理由 | 推荐换什么 |
|------|------|-----------|
| 自动 SLAM 建图（机器人自己跑一遍把图建出来） | iPhone RoomPlan 已经足够好且 10 分钟搞定 | 如要自动化：可后续接 FAST-LIO2，但**先验主入口仍是 RoomPlan** |
| 训练新的 3D 基础模型 | 重点是"系统"而非"模型"，调用现成 CLIP/SAM/YOLO-World 即可 | 模型层放在 [`11_world_model_bridge.md`](11_world_model_bridge.md) 讨论 |
| 户外/大尺度（公里级） | RoomPlan 不支持，ZED 2i 漂移会爆炸 | 户外用 LiDAR-Inertial 替代，整体架构理念可借鉴 |
| 多机器人同步 | 单机版先跑通；多机协同是 v2.0 | 留作 future work |
---
## 0.7 谁是 PRISM 的用户
| 用户 | 用 PRISM 干什么 | 推荐入口章节 |
|------|------------------|--------------|
| **机器人产品经理** | 评估"能不能让酒店机器人记住整层楼" | 00, 01, 09, 12 |
| **机器人工程师** | 实现一个能跑的 demo | 03, 04, 05, 06, 10 |
| **研究者** | 与 hierarchical SLAM / 世界模型对比 | 02, 11, 13 |
| **运维/部署人员** | 部署到新场景、维护先验地图 | 04, 07, 12 |
---
## 0.8 与既有工作的关系（5 分钟版）
PRISM **不是从零发明**，它是把以下三股思想合到一起：
1. **认知科学的"多重记忆系统"**（Tulving, 1985）→ 启发四层结构
2. **分层 SLAM (Hydra, Kimera-Multi, Hovsg, ConceptGraphs)** → 启发场景图作为 L4
3. **预扫描先验 + 在线 SLAM 配准** (BIM-aided SLAM, RoomPlan-as-prior) → 启发握手机制
PRISM 的独特贡献：**首次把消费级 iPhone RoomPlan 当成"先验地图源"**，并设计配套的差异检测与巩固机制，使方案在 **3 万元成本以内**可落地。
详细对比见 [`11_world_model_bridge.md`](11_world_model_bridge.md) 末尾的"相关工作"小节。
---
## 0.9 下一步
读完本章，你应该能回答：
- ✅ PRISM 想解决的核心问题是什么
- ✅ "棱镜"比喻在哪里成立
- ✅ 为什么需要分层而非分库
- ✅ 什么不属于 PRISM 的范围
下一章 [`01_capability_decomposition.md`](01_capability_decomposition.md) 会把 iPhone 和 ZED 2i 两个传感器**逐项拆开**，告诉你它们各自只擅长什么、不擅长什么——这是后续所有分层决策的依据。
---
**章节版本**：v1.0
**估计阅读时间**：8 分钟
**关键收获**：理解 PRISM 命名、目标、边界、设计三原则
plans/PRISM/01_capability_decomposition.md
+244
View File
@@ -0,0 +1,244 @@
# Chapter 01 — 两方案能力解构
> 本章目标：把 iPhone 与 ZED 2i 两个采集方案**逐项拆开**，建立"谁该干什么"的明确边界，作为后续所有分层决策的依据。
---
## 1.1 决策框架：传感器画像 (Sensor Persona)
在做任何融合之前，必须先回答 7 个问题：
```
1. WHEN 何时采集？ （离线 vs 在线）
2. WHO 谁来用？ （人 vs 机器人）
3. WHERE 覆盖多大？ （场景级 vs 视锥级）
4. HOW MUCH 精度多少？ （cm vs %）
5. WHAT 语义多丰富？ （结构化 vs 像素级）
6. HOW FAST 多快？ （秒/分钟 vs Hz）
7. WHY FAIL 什么时候坏？ （失效模式）
```
按这 7 个问题给两个传感器各做一张画像。
---
## 1.2 iPhone (RoomPlan + LiDAR) 画像
```yaml
名称: iPhone Pro (15/14/13 Pro 系列)
角色: 先验地图源 (Prior Map Provider)
WHEN:
- 部署前一次性扫描
- 场景大改时重扫（建议季度级）
- 不参与机器人运行时
WHO:
- 由"人"持机扫描（10–20 分钟/房间）
- 机器人**不携带** iPhone
WHERE:
- 房间级 30–100 m²（单次扫描）
- 多房间需多次扫描 + 走廊串联
- 最大有效范围 ≤ 5 m（LiDAR 物理限制）
HOW MUCH:
- 墙面定位精度: ±3 cm
- 家具尺寸误差: ±5 cm
- 朝向误差: ±2°
- 全局拓扑（房间相对位置）: ±10 cm
WHAT:
RoomPlan 直接产出：
- 结构化几何：参数化的 wall/door/window 平面方程 + OBB
- 家具语义：bed/sofa/chair/table/storage/... (≈ 16 类)
- USDZ / JSON 格式
额外可得（用 ARKit 原始数据）：
- 稠密点云（LiDAR 深度，640×480）
- RGB 图像序列
- ARKit 位姿轨迹
HOW FAST:
- 扫描期间: 30 fps 实时预览
- 处理出图: 10–60 秒（设备端）
- 不是"实时机器人感知"
WHY FAIL:
- ❌ 强光直射（窗边）：LiDAR 失效
- ❌ 全黑环境：RGB 失效，IMU 漂移
- ❌ 完全镜面 / 玻璃墙：几何错乱（与 ZED 2i 共同弱点）
- ❌ 弱纹理大白墙：照片建模失败（LiDAR 可补）
- ⚠️ 家具被移动后：所有家具节点过时
- ⚠️ 动态物体（人/宠物）：会被错误烘到 mesh
```
**一句话**：iPhone 是**"建筑师的图纸 + 摄影师的照片"**——稀疏精确、有语义，但**冻结在过去某一时刻**。
---
## 1.3 ZED 2i (双目 + IMU) 画像
```yaml
名称: Stereolabs ZED 2i
角色: 在线感知器 (Live Perceptor)
WHEN:
- 机器人运行时持续工作
- 30 fps 流式输出
- 与控制循环同步
WHO:
- 装在机器人前向（建议 1.0–1.3 m 高度）
- 机器人随身携带
- 没有人参与
WHERE:
- 视锥级：水平 110° × 垂直 70°
- 深度有效范围: 0.3–20 m（QUALITY 模式下 0.5–10 m 最稳）
- 单次只能看一个视锥（≈ 30% 房间）
HOW MUCH:
- 深度精度: ±1% @ 1 m, ±2% @ 3 m, ±4% @ 5 m
- VIO 位姿漂移: 0.1% 行驶距离（即 100 m 内 < 10 cm）
- 长时间累计漂移: 1 km 后约 1 m
- IMU 短时积分: 角速度 ±2000 dps, 加速度 ±16 g, 400 Hz
WHAT:
原始数据：
- 左右 RGB 图像（2208×1242 @ 15fps，或 1280×720 @ 60fps）
- 深度图（与左目对齐）
- IMU (陀螺 + 加速度 + 磁力计 + 气压计)
SDK 直接给出：
- 6-DoF 位姿（VIO）
- 稀疏关键点轨迹
- 实时 spatial mapping（mesh）
- 物体检测（内置 SOTA 类别有限，建议外挂 YOLO-World）
没有语义：
- 默认不识别房间、不分割物体
- 需自己接 CLIP/SAM/Grounding-DINO
HOW FAST:
- 深度 + VIO: 30 Hz
- 1080p RGB: 30 Hz
- IMU: 400 Hz
- 端到端延迟（采集→深度可用）: 50–80 ms
WHY FAIL:
- ❌ 镜面、玻璃：双目失配，深度伪影（与 iPhone 共同弱点）
- ❌ 极弱纹理（白墙、纯色地毯）：匹配失败
- ❌ 强光直射镜头：过曝
- ⚠️ 长时间静止 + 转动：VIO 漂移大（IMU bias 累积）
- ⚠️ 快速旋转（> 180°/s）：图像运动模糊
- ⚠️ 跨房间（穿门瞬间）：可能丢失定位 → 需重定位
```
**一句话**：ZED 2i 是**"驾驶员的眼睛 + 内耳前庭"**——视锥窄但连续、稠密、随身，但**只看到当下、不记得过去**。
---
## 1.4 并排对比表
| 维度 | iPhone (RoomPlan/LiDAR) | ZED 2i (双目+IMU) | 谁占优 |
|------|-------------------------|-------------------|--------|
| 部署成本 | ¥10 k（一次性） | ¥4 k（一次性） | ZED |
| 单次覆盖 | 30–100 m² | 一个视锥 | **iPhone** |
| 几何精度 | ±3 cm（墙） | ±1–4%（深度） | iPhone（近场打平，远场胜出） |
| 语义丰富度 | 16 类家具，开箱即用 | 0 类（需外挂） | **iPhone** |
| 时间分辨率 | 离线 | 30 Hz | **ZED** |
| 动态物体处理 | 无 | 可在线检测 | **ZED** |
| 镜面/玻璃 | 失败 | 失败 | 平手（都差） |
| 弱纹理 | LiDAR 可救 | 失败 | **iPhone** |
| 大场景拼接 | 多扫多拼，易漂 | VIO 长漂 | 平手（都需校正） |
| 是否上机器人 | 否 | 是 | — |
---
## 1.5 互补性可视化
```mermaid
quadrantChart
title iPhone vs ZED 2i 能力象限
x-axis 低更新率 --> 高更新率
y-axis 低覆盖 --> 高覆盖
quadrant-1 高覆盖+高频率 理想不存在
quadrant-2 高覆盖+低频率 iPhone
quadrant-3 低覆盖+低频率 盲区
quadrant-4 低覆盖+高频率 ZED 2i
iPhone RoomPlan: [0.15, 0.85]
ZED 2i VIO Depth: [0.85, 0.20]
L2 度量层 会合点: [0.5, 0.5]
```
- **iPhone 在"高覆盖×低更新"象限** → 自然落位为 **L3/L4** 的稳定结构
- **ZED 2i 在"低覆盖×高更新"象限** → 自然落位为 **L1/L2** 的即时感知
- 中间的 **L2 度量层**是两者**必然相遇**的地方 → 配准与差异检测必须发生在这里
---
## 1.6 互补性的三个具体维度
### 维度 1：空间分布互补
- iPhone 覆盖**全场景**但**静态**
- ZED 2i 覆盖**视锥**但**穿越所有视锥后**也能覆盖全场景
- → ZED 2i 是 iPhone 的**时间扩展**（补未来的变化）；iPhone 是 ZED 2i 的**空间扩展**（补它没看到的部分）
### 维度 2：语义粒度互补
- iPhone 给**粗粒度建筑语义**：房间、墙、门、大件家具
- ZED 2i + VLM 给**细粒度物品语义**：遥控器、水杯、毛巾
- → 大小语义嵌套形成 **L4 场景图的两级**（房间→家具→物品）
### 维度 3：失败模式互补
- iPhone 在**镜面**失败 → ZED 2i 也失败 → 这是**真共同弱点**，需第三方案（偏振相机，见 [`hotel_scene_implementation_plan_part2.md`](../hotel_scene_implementation_plan_part2.md)）
- iPhone 在**弱纹理**仍能 LiDAR 救场 → ZED 2i 在此处需依赖 iPhone 的先验填充
- ZED 2i 在**动态场景**胜出 → 在此处需 ZED 来告诉 iPhone "你的图过期了"
---
## 1.7 反例：什么情况下单一传感器够用？
为了诚实，列出 PRISM **不必要**的场景：
| 场景 | 推荐 | 原因 |
|------|------|------|
| 仅做静态房间数字孪生展示（不上机器人） | 只用 iPhone | 不需要在线感知 |
| 已有完整 BIM/CAD 图纸的工厂 | BIM + ZED 2i（跳过 iPhone） | BIM 比 RoomPlan 更准 |
| 完全开放的户外环境（公里级） | LiDAR 主导（如 Livox + FAST-LIO2） | RoomPlan 不支持户外 |
| 实验室桌面操作（毫米级） | 桌面级 RGB-D + AprilTag | RoomPlan/VIO 精度都不够 |
| 一次性快速 demo（不留长期记忆） | 只用 ZED 2i | 不需要先验 |
**适用 PRISM 的甜区**：**室内 + 多房间 + 服务机器人 + 需要长期运行 + 场景偶尔变化**。酒店、家居、办公、医院走廊都属于此。
---
## 1.8 把画像翻译为分层决策
| 画像观察 | 引出的设计决策 | 在 PRISM 中的体现 |
|----------|----------------|--------------------|
| iPhone 离线 + 高精度 + 有语义 | 应充当"先验/宪法" | L3/L4 主写者 |
| ZED 2i 在线 + 视锥窄 + 无语义 | 应充当"补丁/快讯" | L1 唯一写者，L2/L4 增量写者 |
| 两者都在镜面失败 | 镜面区单独打标"no-update" | 见 [`12_risks.md`](12_risks.md) |
| 两者精度量级相近（cm 级） | 配准用 ICP 足够，不必更复杂 | 见 [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) |
| iPhone 偶尔过时 | 必须有"差异检测 + 巩固"机制 | 见 [`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md), [`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md) |
| ZED 2i 会漂 | 必须有"锚点重定位刷新" | 见 [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) |
---
## 1.9 本章小结
| 关键词 | 一句话 |
|--------|--------|
| **iPhone 的本质** | 高质量但冻结的"过去快照"，给 L3/L4 |
| **ZED 2i 的本质** | 实时但视锥窄的"当下流"，给 L1/L2 |
| **不可调和的共同弱点** | 镜面/玻璃 → 单独打掩膜 |
| **天然互补的关键点** | 覆盖×频率象限完全错开 |
| **甜区** | 室内多房间服务机器人 |
读完本章你应知道：**为什么必须分层，以及每一层应该由谁主导。**
下一章 [`02_architecture.md`](02_architecture.md) 会把这 4 层结构完整展开，说明每一层的数据结构、更新规则、查询接口。
---
**章节版本**：v1.0
**估计阅读时间**：12 分钟
**关键收获**：建立"传感器画像 → 分层决策"的映射逻辑
plans/PRISM/02_architecture.md
+494
View File
@@ -0,0 +1,494 @@
# Chapter 02 — 四层空间记忆架构
> 本章目标：把机器人大脑里的"空间记忆"拆成 **L1–L4 四层**，逐层说清楚：**存什么 / 怎么存 / 谁写 / 谁读 / 何时过期。**
---
## 2.1 为什么是四层？而不是 1 层、3 层、7 层？
### 不能是 1 层的原因
单一表示（如纯点云）既不能高频更新（点云重写太慢），也不能直接被 LLM 查询（没有语义）。
### 不能是 3 层的原因
若合并 L3 拓扑与 L4 语义为一层，**路径规划**（需要拓扑边权）与**问答**（需要属性查询）会互相干扰。
### 不能更多（如 7 层）的原因
分层越多，**层间同步成本越高**。四层来自三个现实约束的交集：
```mermaid
flowchart TB
subgraph C["认知科学"]
direction LR
C1["Working"] <--> C2["Short-term"]
C2 <--> C3["Long-term"]
end
subgraph S["SLAM 工程"]
direction LR
S1["Metric"] <--> S2["Topological"]
S2 <--> S3["Semantic"]
end
subgraph R["机器人控制"]
direction LR
R1["Reactive (Hz 级)"] <--> R2["Deliberative (秒级)"]
end
subgraph L["PRISM 四层（三家传统的最小公倍数）"]
direction LR
L1["L1<br/>感知缓冲"] --- L2["L2<br/>度量"] --- L3["L3<br/>拓扑"] --- L4["L4<br/>语义"]
end
C -. 分化 .-> S
S -. 加感知缓冲 .-> R
R -. 取并集 .-> L
style L fill:#fff7d6,stroke:#c97a00,stroke-width:2px
```
四层是这三套传统在工程上的**最小公倍数**。
---
## 2.2 四层全景图
```mermaid
flowchart LR
subgraph PRISM["PRISM Spatial Memory（左：高频・易变・低抽象 → 右：低频・稳定・高抽象）"]
direction LR
L1["<b>L1 感知缓冲</b><br/>Perceptual<br/>──────────<br/>位姿环形缓冲<br/>深度环形缓冲<br/>──────────<br/>ZED 2i 主写<br/>更新：30 Hz<br/>寿命：ms 级"]
L2["<b>L2 度量</b><br/>Metric<br/>──────────<br/>占据栅格<br/>TSDF<br/>3DGS<br/>──────────<br/>iPhone+ZED 共写<br/>更新：5 Hz<br/>寿命：秒级"]
L3["<b>L3 拓扑</b><br/>Topological<br/>──────────<br/>房间节点 + 边<br/>锚点列表<br/>──────────<br/>iPhone 主写<br/>更新：低频<br/>寿命：分钟级"]
L4["<b>L4 语义</b><br/>Semantic<br/>──────────<br/>场景图 Neo4j<br/>bbox 属性<br/>──────────<br/>iPhone+ZED 共写<br/>更新：极低频<br/>寿命：小时级"]
L1 --> L2 --> L3 --> L4
end
style L1 fill:#fde2e2,stroke:#a33
style L2 fill:#fff1c1,stroke:#a87a00
style L3 fill:#d4f0d4,stroke:#2e7d32
style L4 fill:#d8e4ff,stroke:#1565c0
```
记住三条直觉：
1. **越往右越"抽象"**：L1 是像素和位姿，L4 是 "the lamp is on the nightstand"
2. **越往右越"慢"**：L1 ms 级覆盖，L4 小时级更新
3. **越往右越"小"**：L1 几 GB/小时，L4 几 MB/全场景
---
## 2.3 L1 — 感知缓冲 (Perceptual Buffer)
### 2.3.1 定位
"机器人最近几秒看到的所有东西"——纯粹的**工作记忆**，类似人的视觉残留。
### 2.3.2 存什么
```python
@dataclass
class PerceptualFrame:
timestamp: float
pose: Pose # T_robot→map (ZED VIO 输出)
rgb_left: np.ndarray # (H,W,3) 可选保存
depth: np.ndarray # (H,W)
imu_packet: List[IMUSample] # 自上一帧以来的 IMU
keypoints: Optional[np.ndarray] # ORB/SuperPoint 关键点
class L1Buffer:
capacity_seconds: float = 10.0 # 环形缓冲容量
keyframe_interval: float = 0.5 # 关键帧采样间隔
ring: Deque[PerceptualFrame] # 满则覆盖
keyframes: Deque[PerceptualFrame] # 关键帧池（保留更久）
```
### 2.3.3 写者 / 读者
| 角色 | 频率 | 操作 |
|------|------|------|
| **写**：ZED 2i 驱动 | 30 Hz | `ring.append(frame)` |
| **读**：避障 / 局部规划 | 10 Hz | 取最近 1 s 的 frames |
| **读**：L2 融合器 | 5 Hz | 取一个关键帧融入 TSDF |
| **读**：回环检测 | 1 Hz | 与关键帧池做相似度匹配 |
### 2.3.4 过期规则
- 超过 `capacity_seconds` 的非关键帧 → 直接丢弃
- 关键帧：若已成功融入 L2 → 5 分钟后可丢弃
- 全部仅在内存，不落盘（除非 debug）
### 2.3.5 失效与降级
- VIO 跟丢 → L1 暂停接收 → 触发重定位（见 [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md)）
- IMU 异常 → 仅用视觉位姿，标记 `confidence=low`
---
## 2.4 L2 — 度量记忆 (Metric Memory)
> ⚠️ **v1.5 更新**:本节后续提及"L2 voxel 直接生成 L3/L4 节点内容"的写法已被 [§ 2.7b L2-L3 数据流:路由 vs 内容(v1.5 新原则)](#27b-l2-l3-数据流路由-vs-内容v15-新原则) 取代。L2 几何在 v1.5 起仅用作**路由信号**,不再承担"决定节点内容"的职责。旧描述保留以备历史追溯。
### 2.4.1 定位
"3D 几何长什么样"——机器人能在上面**规划路径、避障、渲染**的稠密表示。
### 2.4.2 存什么（多重表示）
```python
@dataclass
class L2Memory:
# 用途 1：导航——2D/2.5D 占据栅格
occupancy_grid: OctoMap # 5 cm 分辨率，0=空 1=占 -1=未知
# 用途 2：精细几何——TSDF / Mesh
tsdf: VoxelBlockGrid # 2 cm 分辨率
mesh_uri: str # 离线烘焙的 .glb
# 用途 3：渲染 / 视觉相似度——3DGS
gaussians_uri: Optional[str] # .ply，每房间一份
# 用途 4：先验掩膜
prior_mask: np.ndarray # 0=ZED 可写 1=iPhone 静态保护
no_update_zone: np.ndarray # 1=镜面/玻璃，禁止写入
```
为什么要 4 套表示？
| 表示 | 谁用 | 为什么不能替代 |
|------|------|----------------|
| OctoMap | 路径规划器 | 路径规划只关心"能不能走"，cm 级足够 |
| TSDF | 差异检测 / 抓取 | 需要带符号距离才能算 SDF 差 |
| Mesh | 可视化 / Unity 仿真 | 渲染流水线友好 |
| 3DGS | 视觉重定位 / 新视角生成 | 比 mesh 真实得多 |
它们**共享同一份原始点云**，只是不同的"派生视图"。
### 2.4.3 写者 / 读者
| 区域 | 主写者 | 来源 | 频率 |
|------|--------|------|------|
| **静态结构**（墙、门、固定家具） | iPhone | RoomPlan mesh | 1 次/场景 |
| **可变区域**（家具间空隙、地面） | ZED 2i | TSDF 增量 | 5 Hz 局部 |
| **新发现区**（iPhone 没扫到） | ZED 2i | TSDF 新建 voxel | 5 Hz |
| **镜面/玻璃** | 都不写 | — | — |
### 2.4.4 关键规则：先验保护
```python
def fuse_zed_to_l2(zed_tsdf_local, l2: L2Memory):
for voxel in zed_tsdf_local:
if l2.no_update_zone[voxel.idx]:
continue # 镜面区，跳过
if l2.prior_mask[voxel.idx]:
# 先验区：只允许微调（低权重）
l2.tsdf.update(voxel, weight=0.1)
else:
# 自由区：正常融合
l2.tsdf.update(voxel, weight=1.0)
```
→ iPhone 提供的墙不会被一次 ZED 噪声毁掉，但小幅度（< 5 cm）的修正可累积起效。
### 2.4.5 过期规则
- iPhone 写入的体素 → 长期保留，但有 `last_seen` 字段
- ZED 写入的体素：30 天内未被再次确认 → 衰减 confidence
- 若某区域被打上 `delta`（家具搬走）→ 在 Consolidator 跑后才物理移除
---
## 2.5 L3 — 拓扑记忆 (Topological Memory)
> ⚠️ **v1.5 更新**:本节中"由 L2 几何(TSDF/OctoMap)反推生成 L3 节点内容"的旧写法已被 [§ 2.7b L2-L3 数据流:路由 vs 内容(v1.5 新原则)](#27b-l2-l3-数据流路由-vs-内容v15-新原则) 取代。L3 节点的 `clip_embedding` / `polygon` / 锚点描述等**内容**改为从 L1 高质量 keyframe 直接获取;L2 仅决定"写哪个节点"。旧描述保留以备历史追溯。
### 2.5.1 定位
"房间和走廊怎么连"——机器人**长距离导航**的骨架。
### 2.5.2 存什么
```python
@dataclass
class L3Node:
uid: str # "room_301"
label: str # "Bedroom" / "Hallway" / "Lobby"
center: np.ndarray # (3,) 房间几何中心
polygon: np.ndarray # (N,2) 房间地面多边形
anchors: List[str] # 引用 L4 中的家具 uid（如 'bed_301'）
clip_embedding: np.ndarray # (512,) 房间整体视觉指纹
@dataclass
class L3Edge:
src: str # "room_301"
dst: str # "hallway_3F"
via: str # 连接介质：'door_301a' / 'open_passage'
cost: float # 通行成本（距离 + 难度）
bidirectional: bool = True
class L3Memory:
nodes: Dict[str, L3Node]
edges: List[L3Edge]
anchor_index: Dict[str, str] # furniture_uid -> room_uid 反向索引
```
### 2.5.3 写者 / 读者
| 角色 | 何时 | 操作 |
|------|------|------|
| **写**：iPhone 解析器 | 离线一次 | 按 RoomPlan 的 room 切分自动生成节点；门窗作为边 |
| **写**：ZED 2i + 巡逻 | 机器人实际穿过门时 | 确认/新增边，更新 `cost` |
| **读**：高层规划器 | 任务下发时 | 在 L3 图上跑 A* / Dijkstra 得"房间序列" |
| **读**：重定位器 | 上电 / 跟丢时 | 用 `clip_embedding` 做粗匹配 |
### 2.5.4 过期规则
- 房间节点：除非装修，否则永久保留
- 边：连续 N 次（默认 3 次）巡逻都走不通 → 标记 `deprecated`，路径规划器跳过
- 锚点：被 ZED 检测到原家具消失 → 从 `anchors` 中移除，但节点不删
### 2.5.5 与 L4 的区别（容易混淆！）
- **L3 是"地理"**：节点是**地方**（房间、走廊），边是**通行关系**
- **L4 是"物品"**：节点是**东西**（床、灯、遥控器），边是**支撑/包含/邻接关系**
- L3 节点 `room_301` 通过 `anchors` 字段指向 L4 节点 `bed_301`, `tv_301`...
---
## 2.6 L4 — 语义记忆 (Semantic Memory)
### 2.6.1 定位
"哪个东西在哪儿、长啥样、跟谁挨着"——机器人**理解任务**和**与人对话**的底座。
### 2.6.2 存什么
```python
@dataclass
class L4Node:
uid: str # 'bed_301', 'lamp_301_a'
label: str # 'bed' / 'lamp' / 'remote_control'
category: str # 'furniture' / 'appliance' / 'small_item'
pose: Pose
bbox_3d: np.ndarray # (8,3) OBB
mesh_uri: Optional[str]
clip_embedding: np.ndarray # (512,)
attributes: Dict # {color, material, state(on/off/open/closed),
# mobile: bool, fragile: bool, ...}
parent_room: str # L3 房间 uid
source: Literal['iphone','zed2i','vlm','fused']
first_seen: float
last_seen: float
observation_count: int
confidence: float
@dataclass
class L4Edge:
src_uid: str
dst_uid: str
relation: Literal['on','under','in','next_to',
'inside_drawer','plugged_into',...]
confidence: float
class L4Memory:
nodes: Dict[str, L4Node]
edges: List[L4Edge]
spatial_index: Optional[KDTree] # 加速"附近的东西"查询
semantic_index: Optional[Faiss] # CLIP 向量库，加速文本查物体
```
### 2.6.3 写者 / 读者
| 角色 | 频率 | 操作 |
|------|------|------|
| **写**：iPhone 解析器 | 离线一次 | 16 类家具直接落库（高 confidence） |
| **写**：ZED + VLM | 2 Hz | 检出物品，与现有节点匹配或新建 |
| **写**：Consolidator | 充电时 | 确认 `delta` 并固化 |
| **读**：LLM Agent | 按需 | `find("遥控器在哪")` → CLIP 检索 + 关系遍历 |
| **读**：抓取规划器 | 任务时 | 取目标 `bbox_3d` + `attributes.fragile` |
| **读**：渲染器 | 可视化 | 取所有 `mesh_uri` |
### 2.6.4 三类节点的不同生命周期
| 类型 | 例子 | mobile | 谁写 | 多久过期 |
|------|------|--------|------|----------|
| **固定结构** | 墙、门、内嵌衣柜 | false | iPhone | 永不 |
| **大件家具** | 床、沙发、书桌 | half | iPhone 主，ZED 校准 | 季度级 |
| **小物品** | 遥控器、水杯、毛巾 | true | ZED + VLM 主 | 小时级 |
`mobile` 标签直接影响重定位是否能用它当 anchor（见 [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md)）。
### 2.6.5 关系（Edge）的两种来源
```python
# 1. 几何派生：通过 bbox 相对位置自动得出
def derive_edge_from_geometry(a: L4Node, b: L4Node) -> Optional[L4Edge]:
if a.bbox_3d.contains(b.bbox_3d.center):
return L4Edge(b.uid, a.uid, 'in', confidence=0.9)
if abs(b.bbox_min_z - a.bbox_max_z) < 0.05:
return L4Edge(b.uid, a.uid, 'on', confidence=0.85)
if a.distance_to(b) < 0.5:
return L4Edge(a.uid, b.uid, 'next_to', confidence=0.7)
return None
# 2. VLM 派生：让 GPT-4V / Qwen-VL 看图给关系
def derive_edge_from_vlm(rgb_image, detections) -> List[L4Edge]:
prompt = f"Given these detected objects {detections}, " \
f"list spatial relations as (a, relation, b)."
return vlm.parse_relations(rgb_image, prompt)
```
两路边都允许存在，按 `confidence` 加权融合。
---
## 2.7 层间交互规则
### 2.7.1 写入顺序（数据如何流入大脑）
```mermaid
flowchart TB
ZED["ZED 2i<br/>原始数据"]
VLM["VLM<br/>(在线感知)"]
L1["L1 感知缓冲"]
L2["L2 度量"]
DELTA[("delta/<br/>差异目录")]
L4["L4 语义"]
L3["L3 拓扑"]
CON["Consolidator<br/>（充电时跑）"]
IPHONE["iPhone<br/>(离线一次)"]
LTM[("L2 + L3 + L4<br/>永久保留")]
ZED -- "30 Hz 写入" --> L1
ZED -- "5 Hz 关键帧" --> L2
ZED -- "2 Hz" --> VLM
L2 -- "差异检测" --> DELTA
VLM -- "关联" --> L4
DELTA -- "充电时确认" --> CON
CON --> L4
L4 -- "更新 anchors" --> L3
IPHONE -- "离线一次" --> LTM
style L1 fill:#fde2e2,stroke:#a33
style L2 fill:#fff1c1,stroke:#a87a00
style L3 fill:#d4f0d4,stroke:#2e7d32
style L4 fill:#d8e4ff,stroke:#1565c0
style CON fill:#ffe9b3,stroke:#c97a00
style DELTA fill:#f5e1ff,stroke:#7b1fa2
```
### 2.7.2 查询顺序（Agent 怎么读大脑）
最常见的查询是 **"找东西 + 怎么去"**，标准流程：
```python
def query_find_and_navigate(text: str, memory):
# 1. L4 语义搜索：文本 → 物品 uid
obj_uid = memory.l4.semantic_search(text) # CLIP
obj = memory.l4.nodes[obj_uid]
# 2. L3 拓扑规划：当前房间 → 物品所在房间
cur_room = memory.l3.locate(memory.l1.current_pose)
path_rooms = memory.l3.astar(cur_room, obj.parent_room)
# 3. L2 度量规划：物品所在房间内的精确路径
metric_path = memory.l2.plan_path(
start=memory.l1.current_pose,
goal=obj.pose,
room_sequence=path_rooms)
# 4. L1 实时跟随 + 避障
follow_path(metric_path)
return obj
```
→ **从右向左下钻**：抽象 → 具体，是 PRISM 查询的标准范式。
### 2.7.3 写入冲突的仲裁
当 iPhone 和 ZED 对同一区域有不同观测：
| 情况 | 仲裁规则 |
|------|----------|
| iPhone 说有墙，ZED 说没墙 | 单次→忽略 ZED；连续 N 帧→写 delta；Consolidator 确认才删墙 |
| iPhone 没标家具，ZED 检到家具 | 立即写 L4 新节点，但 `confidence` 起始 0.3，多次确认才上升 |
| iPhone 标了 lamp，ZED 检到 lamp 但位置偏 30 cm | ZED 微调 `pose`，confidence 加权平均 |
| iPhone 标了 chair（mobile=true），ZED 没看到 | 不立即删，标记 `state='moved'`，consolidate 决定 |
总原则：**iPhone 写入的内容"假设正确直到证据充分相反"**。
---
## 2.7b L2-L3 数据流:路由 vs 内容(v1.5 新原则)
> v1.5 起 PRISM 借鉴 Lyra 2.0 的核心思想——**"几何只做路由,不做合成"**——重新厘清 L2 与 L3 之间的职责边界。详细动机与 Lyra 2.0 的对照参见 [`18_lyra_inspirations.md`](18_lyra_inspirations.md)(尤其是 §18.2)。在此之前,PRISM 的 L2 既负责几何稠密表示,又承担"把 voxel 内容写入 L3 节点"的合成职责,导致 L3 节点的视觉指纹(CLIP 嵌入)、bounding box、文字描述等都受 L2 voxel 量化精度限制;新原则把这两件事彻底解耦。
### 2.7b.1 路由 vs 内容:两条数据流分工
```mermaid
flowchart LR
L1KF["<b>L1 keyframe</b><br/>full-res RGB-D<br/>+ CLIP embedding<br/>(高保真原始观测)"]
L2GEO["<b>L2 几何</b><br/>TSDF / OctoMap<br/>(低精度,可量化)"]
L3NODE["<b>L3 节点写入</b><br/>clip_embedding<br/>polygon / anchors<br/>(高保真内容)"]
L1KF -- "内容来源<br/>(高保真)" --> L3NODE
L2GEO -. "仅作路由信号<br/>(决定写哪个节点)" .-> L3NODE
style L1KF fill:#fde2e2,stroke:#a33
style L2GEO fill:#fff1c1,stroke:#a87a00
style L3NODE fill:#d4f0d4,stroke:#2e7d32
```
- **L2 几何 = 路由信号**:仅用来判断"当前观测的 3D 位置应该归属到 L3 图里的哪个节点"。即使 L2 的 voxel 是 2 cm 量化、含噪声、甚至局部缺失,只要它能**指向正确的 L3 节点 uid**就够了。
- **L1 keyframe = 内容源**:节点的 `clip_embedding`、视觉证据、bounding box、属性描述等**实际内容**必须来自 L1 缓存里那一张未经量化的高分辨率 RGB-D keyframe(或它的特征),而非从 L2 voxel 反投影回来。
一句话总结:**几何精度只需要够"指向哪个节点",不需要"决定节点内容"**。
### 2.7b.2 路由式写入伪代码
```python
def write_to_l3(observation, l2_geometry, l3_graph):
# ① L2 只参与"路由":根据观测的 3D 位置定位 L3 节点 uid
target_node = route_via_l2(observation.position, l2_geometry)
# ② 内容来源是 L1 高保真 keyframe,而非 L2 voxel
keyframe_evidence = get_best_keyframe(observation, l3_graph[target_node])
# ③ 用 keyframe 的原始 CLIP / bbox / patch 更新节点内容
l3_graph[target_node].update(keyframe_evidence)
```
四行核心逻辑里,L2 只出现在第一步(`route_via_l2`)且只读;真正塑造 L3 节点内容的是第二步从 L1 取回的 `keyframe_evidence`。这一拆分让 L2 即使在 OctoMap 5 cm + TSDF 噪声的精度下仍然足以胜任,而 L3 的语义保真度由 L1 决定。
### 2.7b.3 对原有章节的影响一览
| 原章节 | 旧写法 | v1.5 新原则下的对应 |
|--------|--------|----------------------|
| §2.4 L2 度量 | L2 既存几何又"派生"L3 内容 | L2 仅做几何 + 路由;不再向 L3 派生内容 |
| §2.5 L3 拓扑 | `clip_embedding` 由 voxel 颜色聚合 | 由 L1 keyframe 直接编码 |
| §2.7.1 写入顺序图 | `L2 --差异检测--> DELTA` 仍然成立 | 但 L3 节点**内容更新**的箭头改为来自 L1,L2 只贡献节点 uid 路由 |
| `06_pipeline_C` §6.6 | TSDF 体素直接重投影生成 patch | 见 [§ 6.6b keyframe-based 内容更新](06_pipeline_C_online_perception.md#66b-v15-新写法keyframe-based-内容更新代替-l2-几何反推) |
---
## 2.8 整体存储预算
以一个 10 房间酒店楼层为例：
| 层 | 数据 | 大小 |
|----|------|------|
| L1 | 10 s 环形缓冲（深度+RGB） | ~ 2 GB 内存 |
| L2 | 10 房间 × (OctoMap 50 MB + TSDF 100 MB + 3DGS 200 MB) | ~ 3.5 GB 磁盘 |
| L3 | 10 节点 + 边 + CLIP 向量 | ~ 100 KB |
| L4 | ~ 500 nodes (10 房 × 50 件) + 关系 | ~ 50 MB |
| **总** | | **~ 4 GB 磁盘 + 2 GB 内存** |
→ 完全可以在 Jetson Orin (32–64 GB) 上跑。
---
## 2.9 本章小结
| 层 | 一句话 | 主写者 | 平均寿命 |
|----|--------|--------|----------|
| **L1** 感知缓冲 | 最近几秒的位姿+深度 | ZED 2i | 10 秒 |
| **L2** 度量 | 3D 几何（导航/渲染用） | iPhone 静态 + ZED 增量 | 月级 |
| **L3** 拓扑 | 房间-走廊连通图 | iPhone | 永久 |
| **L4** 语义 | 物品-关系场景图 | iPhone 大件 + ZED+VLM 小件 | 小时～季度 |
四层不是平行的，而是**沿"频率×抽象"轴展开的连续谱**。
读完本章你应能：
- ✅ 解释每一层"是什么"
- ✅ 解释每一层"由谁主写、谁主读"
- ✅ 理解层间写入冲突的仲裁规则
- ✅ 估算一个 10 房间场景的存储预算
下一章 [`03_data_schema.md`](03_data_schema.md) 给出可直接复制运行的 Python `dataclass` schema + 序列化格式。
---
**章节版本**：v1.0
**估计阅读时间**：18 分钟
**关键收获**：四层结构的"存什么/谁写谁读/何时过期"完整规则
plans/PRISM/03_data_schema.md
+654
View File
@@ -0,0 +1,654 @@
# Chapter 03 — 统一数据模型 (SpatialMemory Schema)
> 本章目标：给出 **可直接复制运行** 的 Python schema、磁盘目录布局、JSON/USD 序列化规范，作为 PRISM 全系统的"数据宪法"。
---
## 3.1 设计原则
| 原则 | 含义 |
|------|------|
| **单一真理源** | 所有层共享 `SpatialMemory` 一个根对象；不允许"iPhone 数据库 + ZED 数据库"并立 |
| **自描述** | 每个节点带 `source`/`level`/`confidence`/`timestamps`，断网恢复后能自解释 |
| **可演化** | Schema 加字段不破坏旧数据；用 `schema_version` 标记 |
| **可序列化** | JSON 用于人读 + Git diff；HDF5/PLY/GLB 用于大数据；USD 用于与 Omniverse/Isaac 对接 |
| **可索引** | 关键查询（"附近的""相似的""最近见到的"）都有 O(log n) 索引支持 |
---
## 3.2 完整 Python Schema
```python
# spatial_memory/schema.py
"""PRISM Spatial Memory — canonical schema v1.0"""
from __future__ import annotations
from dataclasses import dataclass, field, asdict
from typing import Dict, List, Optional, Literal, Any, Tuple
from enum import Enum
import numpy as np
import time
import uuid
SCHEMA_VERSION = "1.0.0"
# ────────────────────── 基础类型 ──────────────────────
@dataclass
class Pose:
"""SE(3) 位姿；统一在 `map` 帧；右手系，z 向上"""
position: np.ndarray # (3,) float32 [x,y,z] 单位 m
quaternion: np.ndarray # (4,) float32 [w,x,y,z] 单位四元数
frame_id: str = "map"
def to_matrix(self) -> np.ndarray:
# 返回 4×4 齐次矩阵
from scipy.spatial.transform import Rotation as R
T = np.eye(4)
T[:3, :3] = R.from_quat(
[self.quaternion[1], self.quaternion[2],
self.quaternion[3], self.quaternion[0]]).as_matrix()
T[:3, 3] = self.position
return T
class MemoryLevel(str, Enum):
L1 = "L1" # 感知缓冲
L2 = "L2" # 度量
L3 = "L3" # 拓扑
L4 = "L4" # 语义
class Source(str, Enum):
IPHONE = "iphone"
ZED2I = "zed2i"
VLM = "vlm"
FUSED = "fused"
HUMAN = "human" # 人工标注/纠正
# ────────────────────── 节点 ──────────────────────
@dataclass
class SpatialNode:
"""统一节点：可表示房间(L3)/家具(L4)/物品(L4)/路点(L2,可选)
> ⚠️ v1.5 更新：本类新增 `keyframe_evidence: List[KeyframeEvidence]` 字段，
> 用于保留 per-frame 独立"翻案证据"。完整动机、字段定义、存储预算与
> retention 策略详见 § 3.2.1（紧随本 Schema 块之后的新增小节）。
> 旧字段语义不变；本注解仅作为升级提示，原 v1.0 字段全部保留。
"""
# ── 标识 ──
uid: str
label: str
level: MemoryLevel
source: Source
confidence: float = 1.0 # 0~1
# ── 几何 ──
pose: Optional[Pose] = None
bbox_3d: Optional[np.ndarray] = None # (8,3) OBB 顶点
polygon_2d: Optional[np.ndarray] = None # (N,2) 仅 L3 房间地面用
mesh_uri: Optional[str] = None # 相对 robot_memory/ 的路径
gaussians_uri: Optional[str] = None
# ── 语义 ──
category: Optional[str] = None # 'wall','furniture','small_item','room'
attributes: Dict[str, Any] = field(default_factory=dict)
# attributes 常用键：
# color, material, state(on/off/open/closed),
# mobile: bool, fragile: bool,
# is_anchor: bool, no_update_zone: bool
clip_embedding: Optional[np.ndarray] = None # (512,) float16
# ── 时序 ──
first_seen: float = field(default_factory=time.time)
last_seen: float = field(default_factory=time.time)
observation_count: int = 1
# ── 关联 ──
parent_uid: Optional[str] = None # L4 物品挂在哪件家具上
parent_room: Optional[str] = None # 反向：所属 L3 房间
children: List[str] = field(default_factory=list)
@staticmethod
def new_uid(prefix: str = "n") -> str:
return f"{prefix}_{uuid.uuid4().hex[:8]}"
# ────────────────────── 边 ──────────────────────
RelationT = Literal[
"contains", "in", # 容纳
"on", "under", # 支撑
"next_to", "front_of", "behind", # 邻接
"connects_to", "reachable_from", # L3 通行
"plugged_into", "inside_drawer", # 特殊
]
@dataclass
class SpatialEdge:
src_uid: str
dst_uid: str
relation: RelationT
weight: float = 1.0 # 距离/通行成本/置信度
source: Source = Source.FUSED
timestamp: float = field(default_factory=time.time)
# ────────────────────── 稠密层（不入节点）──────────────────────
@dataclass
class DenseLayerRefs:
"""L2 稠密表示：用文件 URI 引用，不放内存"""
occupancy_grid_uri: Optional[str] = None # .bt (OctoMap)
tsdf_uri: Optional[str] = None # .vbg / .npz
global_mesh_uri: Optional[str] = None # .glb
global_3dgs_uri: Optional[str] = None # .ply (gsplat 格式)
prior_mask_uri: Optional[str] = None # .npy (1=iPhone 保护)
no_update_zone_uri: Optional[str] = None # .npy (1=镜面)
# ────────────────────── 锚点 ──────────────────────
@dataclass
class Anchor:
"""用于 ZED 上线时与先验地图配准的锚点"""
anchor_uid: str # 引用 SpatialNode.uid
node_label: str # 'bed','tv',...
is_mobile: bool # mobile=True 则不作 anchor
clip_embedding: np.ndarray
geometric_signature: Dict # FPFH 直方图等几何特征
last_validated: float
# ────────────────────── Delta（差异记忆）──────────────────────
@dataclass
class DeltaEvent:
"""ZED 在线发现的、与 LTM 不一致的事件"""
event_id: str
event_type: Literal["object_moved", "object_removed",
"object_added", "geometry_changed"]
target_uid: Optional[str] # 涉及的 LTM 节点（若有）
new_pose: Optional[Pose] = None
new_bbox: Optional[np.ndarray] = None
evidence: List[str] = field(default_factory=list) # 关键帧 ID 列表
observation_count: int = 1
first_observed: float = field(default_factory=time.time)
last_observed: float = field(default_factory=time.time)
status: Literal["pending","confirmed","rejected","applied"] = "pending"
# ────────────────────── 根对象 ──────────────────────
@dataclass
class SpatialMemory:
schema_version: str = SCHEMA_VERSION
world_frame: str = "map"
origin_description: str = "scan start point of iPhone session 1"
gravity: np.ndarray = field(default_factory=lambda: np.array([0,0,-9.81]))
nodes: Dict[str, SpatialNode] = field(default_factory=dict)
edges: List[SpatialEdge] = field(default_factory=list)
dense: DenseLayerRefs = field(default_factory=DenseLayerRefs)
anchors: List[Anchor] = field(default_factory=list)
deltas: List[DeltaEvent] = field(default_factory=list)
# —— 索引（运行时构建，不序列化）——
_kdtree: Any = None # 空间近邻
_faiss: Any = None # CLIP 向量
# —— 便捷方法 ——
def add_node(self, node: SpatialNode) -> None:
assert node.uid not in self.nodes, f"duplicate uid {node.uid}"
self.nodes[node.uid] = node
def add_edge(self, edge: SpatialEdge) -> None:
assert edge.src_uid in self.nodes
assert edge.dst_uid in self.nodes
self.edges.append(edge)
def nodes_of_level(self, lvl: MemoryLevel) -> List[SpatialNode]:
return [n for n in self.nodes.values() if n.level == lvl]
def nodes_in_room(self, room_uid: str) -> List[SpatialNode]:
return [n for n in self.nodes.values() if n.parent_room == room_uid]
```
---
## 3.2.1 v1.5 新增：KeyframeEvidence（per-frame 独立证据）
### 动机
v1.4 之前 PRISM 的 L2 是**融合表示**——TSDF 把每帧深度加权累计到体素、OctoMap 把每条 ray 更新到八叉树占据概率。融合的代价是**累计误差不可回溯**：一旦 L2 在长走廊里漂了 30 cm，"第 t=12.3 s 那一帧 ZED 看到桌子在哪"就再也拿不回来——它已经被融进了几百万个体素的加权平均里。
借鉴 Lyra 2.0 § 3.2(a)"**3D 缓存绝不融合（never fuse）**"的设计，v1.5 让每个 L3 节点额外保留 ≤ 5 个独立 keyframe 作为"翻案证据"：当下游 L4 发现 bbox 估计与节点对不齐、或巩固期需要回溯某个语义改变是否真实发生时，可以**绕过已融合的 L2，直接从原始 keyframe 重新估计**。完整原则陈述与风险分析见 [`18_lyra_inspirations.md` § 18.3](18_lyra_inspirations.md)。
注意：这不是把 TSDF 扔掉，而是让 TSDF（路由 + 避障）与 keyframe 列表（翻案证据）**并存**——二者承担不同任务、对几何精度有不同容忍度。
### 完整 `KeyframeEvidence` Dataclass
```python
# spatial_memory/schema.py (v1.5 新增,接在原 SpatialNode 之后)
@dataclass
class KeyframeEvidence:
"""
每个 L3 节点保留的 per-frame 独立证据。
一律 append-only,绝不被 L2 融合操作覆盖(immutable=True)。
设计要点:
- lazy load: depth_path / rgb_path 只存路径,原始张量留磁盘
- 视觉特征预先算好 CLIP 嵌入并 inline(只有 1.5 KB,索引快)
- visibility_score 是 retention 决策的唯一依据(见下文 retention 策略)
"""
# ── 标识 ──
kf_id: str # 全局唯一,匹配 stm/keyframes/{kf_id}/
ts: float # 采集时间戳 (epoch sec)
source_pipeline: Literal["C_online", "A_offline", "D_consolidation"]
# 该证据由哪条管线产生
# ── 几何 ──
T_cam_world: np.ndarray # (4,4) float32 相机→世界 SE(3)
pose_uncert: Optional[np.ndarray] = None # (6,6) 协方差,可空
# ── 内容引用(lazy load) ──
rgb_path: str = "" # 相对 robot_memory/ 的 JPEG 路径
depth_path: str = "" # 相对路径,uint16 PNG(mm 单位)
intrinsics: Optional[np.ndarray] = None # (3,3) 该帧内参 K
# ── 内联特征(小,索引友好) ──
clip_emb: Optional[np.ndarray] = None # (768,) float16, ~1.5 KB
# ── 路由元数据 ──
routed_l3: Optional[str] = None # 当时路由到的 L3 节点 uid
routed_l4: List[str] = field(default_factory=list) # 关联 L4 uid 列表
visibility_score: float = 0.0 # [0,1] 该帧对节点的可见度/信息量
# 用于 retention 时挑选最差帧替换
# ── 不变性保护 ──
fused_into_l2: bool = False # 是否曾参与 L2 TSDF 融合
immutable: bool = True # 写入后禁止修改;违反者 raise
```
字段从 [`18_lyra_inspirations.md` § 18.3](18_lyra_inspirations.md) 的 `KeyframeEvidence`
扩展而来：保留 Lyra 风格的 append-only 与 lazy-load 语义，并补齐了 PRISM
专用的 `rgb_path / intrinsics / visibility_score / source_pipeline` 字段以贴合
"L3 节点级别保留"这一更具体的工程目标。
### `SpatialNode` 字段扩展示例
在原 `SpatialNode` 定义（§ 3.2 中段，第 72 行起）的末尾新增**一个字段**，其他字段全部保留：
```python
@dataclass
class SpatialNode:
# ... 原 v1.0 所有字段保持不变(uid/label/level/source/confidence/
# pose/bbox_3d/.../parent_room/children) ...
# ── v1.5 新增 ──
keyframe_evidence: List[KeyframeEvidence] = field(default_factory=list)
# 最多保留 5 条,append-only;满后用 visibility_score 替换最差(见下)
```
序列化时 `keyframe_evidence` 走与 `clip_embedding` 同样的"大向量外链"策略：
JSON 中仅留 `kf_id + rgb_path + depth_path + visibility_score + ts` 等元数据，
`clip_emb` 落 `embeddings/{kf_id}.clip.npy`。
### 存储预算估算
按典型酒店楼层（500 L3+L4 节点、每节点保留 5 帧）做单帧 + 总量两级估算：
| 项 | 编码 | 单帧大小 | 备注 |
|----|------|----------|------|
| RGB | 320×240 JPEG q=85 | ~30 KB | ZED/iPhone 下采样后足够做事后翻案 |
| Depth | 320×240 uint16 PNG | ~50 KB | mm 单位,压缩率 ~3× |
| CLIP 嵌入 | 768D float16 | 1.5 KB | inline 进 JSON 索引 |
| 内参 + 位姿 + meta | JSON | ~0.5 KB | T_cam_world + intrinsics + pose_uncert |
| **单帧合计** | — | **~82 KB** | 取整 ≈ 80 KB |
| 维度 | 数值 | 总量 |
|------|------|------|
| 每节点保留帧数 | 5 | 5 × 80 KB ≈ **400 KB / 节点** |
| 节点数（典型一层） | 500 | 500 × 400 KB ≈ **200 MB / 楼层** |
| 节点数（大型场馆 5 层） | 2500 | 2500 × 400 KB ≈ **1.0 GB / 整馆** |
| 旁通存储（NVMe SSD） | — | 1 GB 完全可接受（snapshots/ 同盘） |
结论：**单层 ~200 MB，整馆 ~1 GB**，相比 `dense/octomap.bt`（典型 50–200 MB）
与 `dense/3dgs.ply`（典型 100 MB–1 GB / 房间）属于同量级，**没有引入新数量
级的存储瓶颈**。若磁盘吃紧，可把 `rgb_path` 进一步压到 160×120 JPEG
（~12 KB / 帧）把整馆压到 ~250 MB。
### Retention 策略（满 5 帧后如何替换）
`keyframe_evidence` 容量上限默认 **5 帧/节点**。新证据写入时若已满，按以下规则替换：
```python
def admit_evidence(node: SpatialNode, new_kf: KeyframeEvidence,
capacity: int = 5) -> None:
"""append-only 语义下的 admission control:
capacity 未满则直接 append;满则用 visibility_score 替换最差帧。
注意:被替换的 KeyframeEvidence 在 Pipeline D 巩固期归档到
robot_memory/snapshots/ 而非硬删,保留可追溯性。
"""
if len(node.keyframe_evidence) < capacity:
node.keyframe_evidence.append(new_kf)
return
# 已满:挑当前 visibility_score 最低的一帧
worst_idx = min(range(capacity),
key=lambda i: node.keyframe_evidence[i].visibility_score)
if new_kf.visibility_score > node.keyframe_evidence[worst_idx].visibility_score:
archive_to_snapshot(node.keyframe_evidence[worst_idx]) # 归档,不硬删
node.keyframe_evidence[worst_idx] = new_kf
# 否则新帧也不如最差帧好,直接丢弃(由 Pipeline D 决定是否进 snapshots/)
```
`visibility_score ∈ [0, 1]` 的计算遵循 **geometry-aware retrieval**
（Lyra 2.0 § 3.2(b)）：综合 (a) 该帧对节点 OBB 的覆盖面积比例、
(b) 视角与已有保留帧的角度差异、(c) 深度有效像素占比，三者加权平均。
直觉是"保留信息互补、视角多样、深度可信的 5 帧"，而不是"最近 5 帧"。
回链：本小节落实了 [`18_lyra_inspirations.md` § 18.3](18_lyra_inspirations.md)
"原则二：Per-frame 独立 keyframe 证据"在数据 schema 上的承接。
[`13_evaluation.md`](13_evaluation.md) 后续需新增"翻案命中率 / 翻案误报率"
两项指标来闭环验证本字段的工程价值。
---
## 3.3 序列化规范
### 3.3.1 JSON（人读 + Git 友好）
```python
# spatial_memory/io_json.py
import json
import numpy as np
from .schema import SpatialMemory, SpatialNode, SpatialEdge
def _np_encoder(o):
if isinstance(o, np.ndarray):
return {"__ndarray__": True, "dtype": str(o.dtype),
"shape": list(o.shape), "data": o.flatten().tolist()}
if isinstance(o, (np.float32, np.float16)):
return float(o)
raise TypeError(f"non-serializable: {type(o)}")
def _np_decoder(d):
if d.get("__ndarray__"):
return np.array(d["data"], dtype=d["dtype"]).reshape(d["shape"])
return d
def save(mem: SpatialMemory, path: str) -> None:
from dataclasses import asdict
blob = asdict(mem)
# 大向量不入 JSON：替换为 .npy 外链
for uid, node in blob["nodes"].items():
if node["clip_embedding"] is not None:
np.save(f"{path}.{uid}.clip.npy", node["clip_embedding"])
node["clip_embedding"] = {"__npy__": f"{uid}.clip.npy"}
with open(path, "w") as f:
json.dump(blob, f, indent=2, default=_np_encoder)
def load(path: str) -> SpatialMemory:
with open(path) as f:
blob = json.load(f, object_hook=_np_decoder)
# 反向恢复 dataclass
...
return SpatialMemory(**blob)
```
### 3.3.2 USD（与 Omniverse / Isaac Sim / Polycam 互通）
每个 L4 节点 → 一个 USD `Xform` prim；几何挂在子 prim：
```
/World/Hotel
/Room_301 (Xform, custom attr: room_label="Bedroom")
/Bed_301 (Xform, ref=bed.usd)
/TV_301 (Xform, ref=tv.usd)
/Hallway_3F
...
```
转换器：
```python
# spatial_memory/io_usd.py
from pxr import Usd, UsdGeom, Gf
def memory_to_usd(mem: SpatialMemory, usd_path: str):
stage = Usd.Stage.CreateNew(usd_path)
world = UsdGeom.Xform.Define(stage, "/World")
# L3 房间作为父 Xform
for room in mem.nodes_of_level(MemoryLevel.L3):
room_xform = UsdGeom.Xform.Define(stage, f"/World/{room.uid}")
room_xform.GetPrim().CreateAttribute(
"prism:label", Sdf.ValueTypeNames.String).Set(room.label)
for obj in mem.nodes_in_room(room.uid):
obj_xform = UsdGeom.Xform.Define(
stage, f"/World/{room.uid}/{obj.uid}")
T = obj.pose.to_matrix()
obj_xform.AddTransformOp().Set(Gf.Matrix4d(T.tolist()))
if obj.mesh_uri:
obj_xform.GetPrim().GetReferences().AddReference(obj.mesh_uri)
stage.GetRootLayer().Save()
```
### 3.3.3 二进制（生产部署）
为了在 Jetson 上加载快，关键张量走二进制：
| 数据 | 格式 | 工具 |
|------|------|------|
| 点云 | `.ply` / `.las` | Open3D |
| TSDF | `.vbg` | Open3D VoxelBlockGrid |
| 3DGS | `.ply` (gsplat 标准布局) | gsplat / nerfstudio |
| OctoMap | `.bt` | octomap-cpp |
| Mesh | `.glb` (gltf 2.0) | trimesh |
| CLIP 向量库 | `.faiss` | Faiss |
JSON 仅存**索引和元数据**，数据走文件引用。
---
## 3.4 磁盘目录布局（落盘规范）
```
robot_memory/
├── manifest.json # 总入口，含 schema_version & 各文件 SHA256
├── ltm/ # 长期记忆（iPhone 主导，写入后近乎只读）
│ ├── spatial_memory.json # SpatialMemory 主体（不含大向量）
│ ├── embeddings/
│ │ └── {uid}.clip.npy # 每节点 CLIP 向量
│ ├── meshes/
│ │ ├── room_301.glb
│ │ └── bed_301.glb
│ ├── pointcloud/
│ │ └── room_301.ply
│ ├── dense/
│ │ ├── octomap.bt
│ │ ├── tsdf.vbg
│ │ ├── 3dgs.ply
│ │ ├── prior_mask.npy
│ │ └── no_update_zone.npy
│ ├── anchors.json
│ └── usd/
│ └── hotel.usdz # 给 Omniverse / 仿真用
│
├── stm/ # 短期/工作记忆（运行时，环形覆盖）
│ ├── current_pose.txt # 单行最新位姿
│ ├── trajectory.tum # TUM 格式轨迹（追加写）
│ ├── keyframes/
│ │ └── {ts}_{idx}/
│ │ ├── rgb.jpg
│ │ ├── depth.png
│ │ └── meta.json
│ ├── live_octomap.bt # 实时局部
│ └── live_tsdf.vbg
│
├── delta/ # 差异记忆（待巩固）
│ ├── pending.jsonl # 一行一个 DeltaEvent
│ ├── confirmed.jsonl
│ └── rejected.jsonl
│
├── snapshots/ # 版本化历史 LTM
│ ├── 2026-05-16_v1.tar.zst
│ └── 2026-06-01_v2.tar.zst
│
└── logs/
├── relocalize.log
├── consolidation.log
└── metrics.parquet # 评测指标时间序列
```
`manifest.json` 示例：
```json
{
"schema_version": "1.0.0",
"world_frame": "map",
"created_at": "2026-05-16T10:00:00+08:00",
"scene_name": "Hotel-Demo-Floor3",
"ltm_version": 5,
"files": {
"ltm/spatial_memory.json": {
"sha256": "ab12...",
"size_bytes": 1048576
},
"ltm/dense/octomap.bt": { "sha256": "...", "size_bytes": 52428800 }
}
}
```
---
## 3.5 数据库映射（生产部署用 Neo4j）
原型期可用 JSON + NetworkX，生产期建议把 L3+L4 进 Neo4j（节点 + 关系），稠密数据仍走文件：
```cypher
// 创建房间节点
MERGE (r:Room {uid: 'room_301'})
SET r.label='Bedroom',
r.center=point({x:1.2, y:3.4, z:0.0}),
r.confidence=0.95;
// 创建家具节点 + 关系
MERGE (b:Furniture {uid: 'bed_301'})
SET b.label='bed', b.mobile=false,
b.pose_x=2.0, b.pose_y=3.5, b.pose_z=0.3;
MERGE (r)-[:CONTAINS]->(b);
// 小物品 on 家具
MERGE (rc:Item {uid: 'remote_xyz'})
SET rc.label='remote_control', rc.mobile=true;
MERGE (rc)-[:ON {confidence:0.85}]->(b);
```
LLM Agent 查询时可以直接发 Cypher：
```cypher
MATCH (i:Item {label:'remote_control'})-[:ON]->(f)<-[:CONTAINS]-(r:Room)
RETURN i, f, r;
```
---
## 3.6 版本化与迁移
```python
# spatial_memory/migrate.py
def migrate(blob: dict) -> dict:
v = blob.get("schema_version", "0.0.0")
if v == "1.0.0":
return blob
if v == "0.9.0":
# 0.9 → 1.0：把 'class' 字段重命名为 'category'
for node in blob["nodes"].values():
node["category"] = node.pop("class", None)
blob["schema_version"] = "1.0.0"
return migrate(blob)
raise ValueError(f"Unsupported schema version: {v}")
```
每次 schema 升级 → 写一个迁移函数 + 在 `snapshots/` 留备份。
---
## 3.7 一致性校验
写完任何节点都跑：
```python
def validate(mem: SpatialMemory) -> List[str]:
errs = []
for uid, node in mem.nodes.items():
if uid != node.uid:
errs.append(f"uid mismatch: {uid} vs {node.uid}")
if node.level == MemoryLevel.L4 and node.parent_room is None:
errs.append(f"L4 node {uid} missing parent_room")
if node.confidence < 0 or node.confidence > 1:
errs.append(f"{uid} confidence out of [0,1]")
for edge in mem.edges:
if edge.src_uid not in mem.nodes:
errs.append(f"dangling edge src: {edge.src_uid}")
if edge.dst_uid not in mem.nodes:
errs.append(f"dangling edge dst: {edge.dst_uid}")
return errs
```
CI 里跑 `validate()` 防止 schema 退化。
---
## 3.8 API 设计原则（给上层 Agent）
不要让 Agent 直接访问 `mem.nodes`；提供高层方法：
```python
class SpatialMemoryAPI:
def find(self, text: str, top_k: int = 5) -> List[SpatialNode]: ...
def locate(self, query: SpatialNode) -> Pose: ...
def neighbors(self, uid: str, radius: float = 1.0) -> List[SpatialNode]: ...
def path_rooms(self, src_room: str, dst_room: str) -> List[str]: ...
def changes_since(self, t: float) -> List[DeltaEvent]: ...
def describe(self, uid: str) -> str:
"""生成自然语言描述供 LLM 消化"""
node = self.nodes[uid]
room = self.nodes[node.parent_room]
return f"A {node.attributes.get('color','')} {node.label} " \
f"in {room.label}, last seen {ago(node.last_seen)}."
```
→ Agent 看不见底层格式变更，只用 API。
---
## 3.9 本章小结
| 关键约定 | 一句话 |
|----------|--------|
| **根对象** | `SpatialMemory`（含 nodes/edges/dense/anchors/deltas） |
| **节点** | `SpatialNode`（带 level + source + confidence + 时间戳） |
| **边** | `SpatialEdge`（含 relation + weight） |
| **稠密数据** | 不入节点，走文件 URI |
| **磁盘** | 三主目录：`ltm/` `stm/` `delta/` + `snapshots/` |
| **数据库** | 原型 NetworkX/JSON，生产 Neo4j |
| **序列化** | JSON 主，二进制大数据走外链；USD 用于仿真桥接 |
| **版本** | `schema_version` + 迁移函数 |
读完本章你应能：
- ✅ 把 `schema.py` 复制进项目就开始写代码
- ✅ 知道一份 LTM 在磁盘上长什么样
- ✅ 给 LLM Agent 提供安全的 API
下一章 [`04_pipeline_A_iphone_offline.md`](04_pipeline_A_iphone_offline.md) 用这套 schema 实现"iPhone 扫描 → SpatialMemory 灌入"的完整管线。
---
**章节版本**：v1.0
**估计阅读时间**：15 分钟
**关键收获**：拿到可立即使用的 dataclass + 磁盘布局
plans/PRISM/04_pipeline_A_iphone_offline.md
+663
View File
@@ -0,0 +1,663 @@
# Chapter 04 — 管线 A：iPhone 离线建图
> 本章目标：把 iPhone Pro 扫描得到的 **USDZ / RoomPlan JSON / 原始 ARKit 数据**，转换为 PRISM 的 **`SpatialMemory` 长期记忆 LTM**，包括 L2 度量 + L3 拓扑 + L4 语义。
---
## 4.1 管线总览
```mermaid
flowchart LR
A["📱 iPhone Pro<br/>RoomPlan + ARKit RAW<br/><b>1. 采集</b>"]
B["💻 Mac / Linux<br/>上传 / 下载<br/><b>2. 传输</b>"]
C["🐍 Parser & Glue<br/>(Python)<br/><b>3. 解析 + 几何处理</b>"]
D[("🧠 PRISM LTM<br/>写入<br/><b>4. 落盘</b>")]
A --> B --> C --> D
style A fill:#e3f2fd,stroke:#1565c0
style D fill:#fff7d6,stroke:#c97a00
```
5 个阶段：
| 阶段 | 工具 | 输入 | 输出 |
|------|------|------|------|
| A1 采集 | Swift App (RoomPlan + ARKit) | 人手持 iPhone | `Hotel.usdz` + `roomplan.json` + ARKit raw |
| A2 传输 | scp / iCloud / WebDAV | iPhone → Mac/PC | 同上 |
| A3 坐标对齐 | Python + ArUco | iPhone session 坐标 | `T_iphone→map` |
| A4 解析 | `parser_iphone.py` | RoomPlan JSON | `SpatialNode/Edge` 列表 |
| A5 几何派生 | Open3D / nvblox | mesh + 点云 | OctoMap + TSDF + 3DGS |
---
## 4.2 阶段 A1：iPhone 端采集
### 4.2.1 推荐的 Swift App 骨架
```swift
// PRISMScanner/RoomScannerView.swift
import RoomPlan
import ARKit
class RoomScannerCoordinator: NSObject, RoomCaptureSessionDelegate {
let captureSession = RoomCaptureSession()
let arSession = ARSession()
var capturedRoom: CapturedRoom?
var capturedFrames: [ARFrame] = [] // 同步保留原始帧
func start() {
// 1) RoomPlan 高层结构
captureSession.delegate = self
captureSession.run(configuration: .init())
// 2) ARKit 原始数据（深度 + RGB + 位姿）—— 单独存
let cfg = ARWorldTrackingConfiguration()
cfg.frameSemantics.insert(.sceneDepth)
cfg.frameSemantics.insert(.smoothedSceneDepth)
arSession.delegate = self
arSession.run(cfg)
}
func captureSession(_ session: RoomCaptureSession,
didEndWith data: CapturedRoomData,
error: Error?) {
Task {
let room = try await RoomBuilder().capturedRoom(from: data)
try export(room, frames: capturedFrames)
}
}
private func export(_ room: CapturedRoom, frames: [ARFrame]) throws {
// (a) RoomPlan 结构化输出
try room.export(to: docsURL.appendingPathComponent("Hotel.usdz"))
let json = try JSONEncoder().encode(room)
try json.write(to: docsURL.appendingPathComponent("roomplan.json"))
// (b) ARKit 原始 → 给 Python 端的稠密重建用
try ARKitDumper.dump(frames, to: docsURL.appendingPathComponent("arkit/"))
}
}
```
### 4.2.2 采集 SOP（标准作业流程）
| 步骤 | 时间 | 关键动作 |
|------|------|----------|
| 1. 环境准备 | 2 min | 开灯、移除人/宠物、关闭电视 |
| 2. **放置 ArUco 标识** | 1 min | 在地面放 1 个 30 cm × 30 cm ArUco/AprilTag（用于后续 `map` 原点对齐，见 4.4） |
| 3. App 启动 | 30 s | 检查 LiDAR 工作正常（预览有点云） |
| 4. 房间扫描 | 10–15 min | 沿墙慢走，距墙 1–1.5 m，速度 < 0.3 m/s |
| 5. 重点区域回扫 | 5 min | 床、桌、衣柜（开门）、卫生间门口 |
| 6. 走廊连接 | 3 min/段 | 同一 ARKit session 内穿过门，保证多房间共享坐标系 |
| 7. 结束 + 命名 | 1 min | 输出 `Hotel-3F.usdz` 等 |
### 4.2.3 多房间扫描的两种策略
| 策略 | 适用 | 优点 | 缺点 |
|------|------|------|------|
| **单 Session 连扫** | < 5 房间，路径连续 | 自动共享坐标系，无需配准 | 长时漂移大 |
| **多 Session 分扫 + ArUco 拼接** | ≥ 5 房间或不连通 | 每房间独立精度高 | 需手工拼接（见 4.4） |
> 经验：酒店一层楼建议 **每 3 个房间 + 中间走廊** 作为 1 个 session，最多扫 4–5 个 session，最后用走廊的公共 ArUco 串起来。
---
## 4.3 阶段 A2：数据传输
```mermaid
flowchart TB
subgraph IOS["iPhone Files App<br/><i>On My iPhone/PRISMScanner/</i>"]
F1["Hotel-3F.usdz<br/><i>(4–20 MB)</i>"]
F2["roomplan.json<br/><i>(~50 KB)</i>"]
subgraph ARK["arkit/"]
A1["frames.h5<br/><i>深度图序列, 200 MB ~ 2 GB</i>"]
A2["rgb/*.jpg"]
A3["poses.tum"]
end
end
TRANS["iCloud / scp over WiFi (Mac) / WebDAV"]
REPO[("Project repo:<br/>data/scans/2026-05-16_3F/")]
IOS --> TRANS --> REPO
style IOS fill:#e3f2fd,stroke:#1565c0
style REPO fill:#fff7d6,stroke:#c97a00
```
**传输脚本**（在 Mac/Linux 跑）：
```bash
# scripts/fetch_scan.sh
NAME=$1 # 2026-05-16_3F
mkdir -p data/scans/$NAME
# 通过 SSH 文件传输（需在 iPhone 上装 a-Shell 或类似 SSH 服务）
scp -r mobile:Documents/PRISMScanner/$NAME/ data/scans/$NAME/
ls data/scans/$NAME/
```
---
## 4.4 阶段 A3：坐标系对齐 — 把 iPhone 锚到 `map` 帧
iPhone 每个 ARKit session 的原点是**第一帧时设备所在位置**，机器人却需要一个**稳定不变的世界原点 `map`**。
### 4.4.1 公共原点策略（推荐）
在场景里放 1 个 **30 cm × 30 cm ArUco DICT_5X5_100 id=42** 标识，约定其**左上角**为 `map` 原点，**长边指 +x，短边指 +y，z 向上**。
iPhone 扫描时只要拍到这个标识就能算出 `T_iphone→map`：
```python
# spatial_memory/align_to_map.py
import cv2
import numpy as np
from scipy.spatial.transform import Rotation as R
ARUCO_SIZE_M = 0.30
ARUCO_DICT = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_5X5_100)
TARGET_ID = 42
def find_T_iphone_to_map(rgb_jpg, intrinsics, arkit_pose) -> np.ndarray:
"""rgb_jpg: 某帧 RGB；arkit_pose: 该帧的 ARKit 位姿 T_cam→iphone_origin"""
img = cv2.imread(rgb_jpg)
corners, ids, _ = cv2.aruco.detectMarkers(img, ARUCO_DICT)
if ids is None or TARGET_ID not in ids:
return None
idx = list(ids.flatten()).index(TARGET_ID)
obj_pts = np.array([[0,0,0],[ARUCO_SIZE_M,0,0],
[ARUCO_SIZE_M,ARUCO_SIZE_M,0],
[0,ARUCO_SIZE_M,0]], dtype=np.float32)
ok, rvec, tvec = cv2.solvePnP(obj_pts, corners[idx][0], intrinsics, None)
T_cam_to_map = np.eye(4)
T_cam_to_map[:3,:3] = cv2.Rodrigues(rvec)[0]
T_cam_to_map[:3, 3] = tvec.flatten()
T_cam_to_iphone = arkit_pose
# T_iphone_to_map = T_cam_to_map @ inv(T_cam_to_iphone)
return T_cam_to_map @ np.linalg.inv(T_cam_to_iphone)
```
把 `T_iphone→map` 存进 `manifest.json`，后续所有几何都左乘这个变换。
### 4.4.2 退化方案：无 ArUco 时
按约定：**第一次扫描的"门口正中、面朝房间内"**作为 `map` 原点；以后所有 session 用 ICP 拼到第一次。精度略低（±5 cm），但简单。
---
## 4.5 阶段 A4：解析 RoomPlan → SpatialNode/Edge
### 4.5.1 RoomPlan JSON 结构（核心字段）
```json
{
"version": "iOS17",
"story": {
"floors": [{"identifier": "F3", "z_height": 0.0}],
"walls": [
{"id": "w_001", "category": "Wall",
"transform": [16 floats], "dimensions": [3.5, 2.7, 0.10]},
...
],
"openings": [
{"id": "o_001", "category": "Door",
"transform": [...], "dimensions": [0.9, 2.1, 0.05],
"connects": ["room_301","hallway_3F"]}
],
"objects": [
{"id": "f_001", "category": "Bed",
"transform": [...], "dimensions": [2.0, 1.8, 0.6],
"confidence": "high"}
],
"rooms": [
{"id": "room_301", "label": "Bedroom",
"polygon": [[x,y],...]}
]
}
}
```
### 4.5.2 解析器实现
```python
# spatial_memory/parser_iphone.py
import json, numpy as np
from .schema import (SpatialMemory, SpatialNode, SpatialEdge,
Pose, MemoryLevel, Source)
CATEGORY_MAP = {
"Bed": ("bed", False), "Sofa": ("sofa", False),
"Chair": ("chair", True), "Table": ("table", False),
"Storage": ("storage", False), "TV": ("tv", False),
"Toilet": ("toilet", False), "Sink": ("sink", False),
"Bathtub": ("bathtub", False), "Refrigerator": ("fridge", False),
"Stove": ("stove", False), "Dishwasher": ("dishwasher", False),
"Oven": ("oven", False), "Washer": ("washer", False),
"Fireplace": ("fireplace", False), "Stairs": ("stairs", False),
}
def parse_roomplan(json_path: str, T_iphone_to_map: np.ndarray,
mesh_dir: str) -> SpatialMemory:
mem = SpatialMemory()
rp = json.load(open(json_path))
story = rp["story"]
# ── 1. 房间节点（L3）──
for room in story["rooms"]:
polygon = np.array(room["polygon"], dtype=np.float32)
center = polygon.mean(axis=0)
node = SpatialNode(
uid=f"room_{room['id']}",
label=room.get("label", "Room"),
level=MemoryLevel.L3, source=Source.IPHONE,
confidence=0.95,
pose=Pose(position=np.array([center[0], center[1], 0.0]),
quaternion=np.array([1,0,0,0])),
polygon_2d=polygon,
category="room",
attributes={"is_anchor": False})
mem.add_node(node)
# ── 2. 墙节点（L2，不进场景图主查询，但保留供渲染/规划）──
for wall in story["walls"]:
T = transform_to_matrix(wall["transform"])
T_map = T_iphone_to_map @ T # 关键：左乘对齐
pos = T_map[:3, 3]
quat = matrix_to_quat(T_map[:3, :3])
bbox = obb_from_transform_and_dims(T_map, wall["dimensions"])
node = SpatialNode(
uid=f"wall_{wall['id']}",
label="wall", category="structure",
level=MemoryLevel.L2, source=Source.IPHONE,
confidence=0.95,
pose=Pose(position=pos, quaternion=quat),
bbox_3d=bbox,
attributes={"mobile": False, "no_update_zone": False})
mem.add_node(node)
# ── 3. 门窗作为 L3 边的载体 ──
for op in story["openings"]:
if op["category"] not in ("Door", "Opening"):
continue
connects = op.get("connects", [])
if len(connects) == 2:
r1, r2 = f"room_{connects[0]}", f"room_{connects[1]}"
if r1 in mem.nodes and r2 in mem.nodes:
T = transform_to_matrix(op["transform"])
T_map = T_iphone_to_map @ T
center_xyz = T_map[:3, 3]
cost = 1.0 # 门，可通行
mem.add_edge(SpatialEdge(
src_uid=r1, dst_uid=r2, relation="connects_to",
weight=cost, source=Source.IPHONE))
# 也存为节点本身（可视化、关门状态等）
mem.add_node(SpatialNode(
uid=f"door_{op['id']}",
label="door", category="opening",
level=MemoryLevel.L2, source=Source.IPHONE,
pose=Pose(position=center_xyz,
quaternion=matrix_to_quat(T_map[:3,:3])),
bbox_3d=obb_from_transform_and_dims(T_map, op["dimensions"]),
confidence=0.9,
attributes={"state":"closed", "mobile": False}))
# ── 4. 家具节点（L4）──
for obj in story["objects"]:
cat = obj["category"]
if cat not in CATEGORY_MAP:
label, mobile = cat.lower(), True
else:
label, mobile = CATEGORY_MAP[cat]
T = transform_to_matrix(obj["transform"])
T_map = T_iphone_to_map @ T
# 找它所在房间（点-多边形）
parent_room = find_room_for_point(T_map[:2, 3], mem)
node = SpatialNode(
uid=f"{label}_{obj['id']}",
label=label, category="furniture",
level=MemoryLevel.L4, source=Source.IPHONE,
confidence=0.9 if obj.get("confidence")=="high" else 0.6,
pose=Pose(position=T_map[:3,3], quaternion=matrix_to_quat(T_map[:3,:3])),
bbox_3d=obb_from_transform_and_dims(T_map, obj["dimensions"]),
parent_room=parent_room,
mesh_uri=f"meshes/{label}_{obj['id']}.glb", # 4.6 会生成
attributes={"mobile": mobile,
"is_anchor": (not mobile and label in
("bed","sofa","tv","toilet","bathtub","sink"))})
mem.add_node(node)
if parent_room:
mem.add_edge(SpatialEdge(
src_uid=parent_room, dst_uid=node.uid,
relation="contains", source=Source.IPHONE))
return mem
```
辅助函数 `transform_to_matrix`、`matrix_to_quat`、`obb_from_transform_and_dims`、`find_room_for_point` 是标准几何工具，略。
---
## 4.6 阶段 A5：几何派生（mesh → OctoMap/TSDF/3DGS + 切件 mesh）
### 4.6.1 从 USDZ 拆出每件家具的 mesh
```python
# spatial_memory/extract_furniture_mesh.py
from pxr import Usd, UsdGeom
import trimesh
import numpy as np
def split_usdz_per_object(usdz_path: str, out_dir: str,
mem: SpatialMemory) -> None:
stage = Usd.Stage.Open(usdz_path)
for prim in stage.Traverse():
if not UsdGeom.Mesh(prim):
continue
name = str(prim.GetPath())
# 根据 prim 名匹配到 SpatialNode
uid = match_prim_to_uid(name, mem)
if uid is None:
continue
# 提取顶点/面 → trimesh → 导出 .glb
verts, faces = read_usd_mesh(prim)
mesh = trimesh.Trimesh(vertices=verts, faces=faces)
# 用 SpatialNode 的 inverse pose 把 mesh 移到局部坐标
T_map = mem.nodes[uid].pose.to_matrix()
mesh.apply_transform(np.linalg.inv(T_map))
mesh.export(f"{out_dir}/{uid}.glb")
```
### 4.6.2 烘焙 OctoMap（用于 2D 导航）
```python
# spatial_memory/bake_octomap.py
import open3d as o3d
import numpy as np
def mesh_to_octomap(global_mesh_path: str, out_bt: str, resolution: float = 0.05):
mesh = o3d.io.read_triangle_mesh(global_mesh_path)
pc = mesh.sample_points_uniformly(number_of_points=2_000_000)
pts = np.asarray(pc.points)
# 调外部 octomap 工具（pip install octomap-python 或 octovis）
import octomap
tree = octomap.OcTree(resolution)
for p in pts:
tree.updateNode(p, True, lazy_eval=True)
tree.updateInnerOccupancy()
tree.writeBinary(out_bt.encode())
```
### 4.6.3 烘焙 TSDF（用于差异检测）
```python
# spatial_memory/bake_tsdf.py
import open3d as o3d
import numpy as np
def mesh_to_tsdf(mesh_path: str, out_vbg: str, voxel: float = 0.02):
mesh = o3d.io.read_triangle_mesh(mesh_path)
# Open3D 0.18+ 的 VoxelBlockGrid
vbg = o3d.t.geometry.VoxelBlockGrid(
attr_names=('tsdf', 'weight'),
attr_dtypes=(o3d.core.float32, o3d.core.float32),
attr_channels=((1,), (1,)),
voxel_size=voxel, block_resolution=16, block_count=50000)
# 用 mesh 上采样的虚拟"深度图"灌入
# （或直接用 ARKit 留下的真实深度图，质量更好）
...
o3d.t.io.write_voxel_block_grid(out_vbg, vbg)
```
> **强烈推荐**用 ARKit 留下的**真实深度帧**而非 mesh 重采样来填 TSDF——保留噪声分布特性，差异检测才公平。
### 4.6.4 训练 3DGS（可选）
```bash
# 用 nerfstudio splatfacto，输入是 ARKit 留的 RGB+pose+depth
ns-process-data record3d \
--data data/scans/2026-05-16_3F/arkit/ \
--output-dir data/processed/3F/
ns-train splatfacto \
--data data/processed/3F/ \
--pipeline.model.use-depth-loss True \
--max-num-iterations 15000
ns-export gaussian-splat \
--load-config outputs/3F/splatfacto/config.yml \
--output-dir robot_memory/ltm/dense/
```
输出 `ltm/dense/3dgs.ply`。
### 4.6.5 生成 `prior_mask` 与 `no_update_zone`
```python
def build_masks(mem: SpatialMemory, voxel_grid_shape, voxel_size) -> Tuple:
prior_mask = np.zeros(voxel_grid_shape, dtype=np.uint8)
no_update = np.zeros(voxel_grid_shape, dtype=np.uint8)
for n in mem.nodes.values():
if n.bbox_3d is None: continue
if not n.attributes.get("mobile", True):
mark_obb_in_grid(prior_mask, n.bbox_3d, voxel_size, val=1)
if n.label in ("mirror","window","glass_wall") or \
n.attributes.get("reflective", False):
mark_obb_in_grid(no_update, n.bbox_3d, voxel_size, val=1)
return prior_mask, no_update
```
镜面识别可以在 iPhone 端就让人工标，或后期跑一遍 ZED 的反射检测——这里允许后补。
---
## 4.7 计算 CLIP embedding（给 L3 房间 + L4 家具）
```python
import open_clip, torch
from PIL import Image
model, _, preprocess = open_clip.create_model_and_transforms("ViT-B-32")
model.eval().cuda()
def compute_room_clip(room_uid: str, rgb_dir: str) -> np.ndarray:
"""房间用 5–10 张代表性 RGB 平均"""
imgs = sample_keyframes_for_room(room_uid, rgb_dir, k=8)
feats = []
with torch.no_grad():
for img in imgs:
x = preprocess(Image.open(img)).unsqueeze(0).cuda()
feats.append(model.encode_image(x).cpu().numpy()[0])
return np.mean(feats, axis=0).astype(np.float16)
def compute_object_clip(node: SpatialNode, mesh_dir: str,
rgb_dir: str) -> np.ndarray:
"""家具：优先用 mesh 渲染的多视角图；退化为 ARKit 帧裁剪"""
# 方法 1：trimesh + pyrender 多视角离线渲染
views = render_mesh_views(f"{mesh_dir}/{node.uid}.glb", num_views=6)
feats = []
with torch.no_grad():
for v in views:
x = preprocess(v).unsqueeze(0).cuda()
feats.append(model.encode_image(x).cpu().numpy()[0])
return np.mean(feats, axis=0).astype(np.float16)
```
> 渲染时背景设为白色 / 透明，避免环境干扰主体语义。
为每个节点存一份 `.npy` 到 `ltm/embeddings/{uid}.clip.npy`，便于在重定位时按需加载。
---
## 4.8 生成 Anchors（重定位锚点候选集）
```python
# spatial_memory/build_anchors.py
from .schema import Anchor
ANCHOR_LABELS = {"bed","sofa","tv","toilet","bathtub","sink","fridge",
"stove","door","wardrobe","fireplace"}
def build_anchors(mem: SpatialMemory) -> List[Anchor]:
anchors = []
for node in mem.nodes.values():
if node.label not in ANCHOR_LABELS: continue
if node.attributes.get("mobile", True): continue # 可移动的不能当锚点
if node.confidence < 0.7: continue
# FPFH 几何签名（用 OBB 表面采样点）
fpfh = compute_fpfh_signature(node.bbox_3d, node.mesh_uri)
anchors.append(Anchor(
anchor_uid=node.uid,
node_label=node.label,
is_mobile=False,
clip_embedding=node.clip_embedding,
geometric_signature=fpfh,
last_validated=time.time()))
return anchors
```
每个房间建议至少有 **2 个不同类的 anchor**（如床+电视），方便 ICP 收敛。
---
## 4.9 完整 CLI：`prism-ingest-iphone`
把上述阶段串成一个命令：
```python
# tools/prism_ingest_iphone.py
import click, json, numpy as np
from spatial_memory.schema import SpatialMemory
from spatial_memory.parser_iphone import parse_roomplan
from spatial_memory.align_to_map import find_T_iphone_to_map
from spatial_memory.extract_furniture_mesh import split_usdz_per_object
from spatial_memory.bake_octomap import mesh_to_octomap
from spatial_memory.bake_tsdf import mesh_to_tsdf
from spatial_memory.build_anchors import build_anchors
from spatial_memory.io_json import save
@click.command()
@click.option("--scan", required=True, help="data/scans/<NAME>/")
@click.option("--out", required=True, help="robot_memory/ltm/")
@click.option("--aruco-id", default=42, type=int)
def main(scan, out, aruco_id):
# 1. 找一帧能看到 ArUco 的图，算 T_iphone→map
T = find_T_iphone_to_map_from_dir(f"{scan}/arkit/", aruco_id)
if T is None:
print("⚠️ no ArUco found, fallback to first-frame origin")
T = np.eye(4)
# 2. 解析 RoomPlan JSON
mem = parse_roomplan(f"{scan}/roomplan.json", T, mesh_dir=f"{out}/meshes/")
# 3. 拆 USDZ → 每件家具一份 .glb
split_usdz_per_object(f"{scan}/Hotel.usdz", f"{out}/meshes/", mem)
# 4. 全局 mesh 也烘焙一份给渲染
global_mesh = bake_global_mesh(f"{scan}/Hotel.usdz", T,
f"{out}/dense/global_mesh.glb")
# 5. OctoMap & TSDF
mesh_to_octomap(f"{out}/dense/global_mesh.glb",
f"{out}/dense/octomap.bt", resolution=0.05)
mesh_to_tsdf(f"{out}/dense/global_mesh.glb",
f"{out}/dense/tsdf.vbg", voxel=0.02)
# 6. CLIP 向量
for uid, node in mem.nodes.items():
if node.level in ("L3","L4"):
emb = compute_room_clip(uid, f"{scan}/arkit/rgb/") \
if node.level == "L3" \
else compute_object_clip(node, f"{out}/meshes/",
f"{scan}/arkit/rgb/")
np.save(f"{out}/embeddings/{uid}.clip.npy", emb)
node.clip_embedding = emb
# 7. Anchors
mem.anchors = build_anchors(mem)
# 8. dense URI 落到 schema
mem.dense.occupancy_grid_uri = "dense/octomap.bt"
mem.dense.tsdf_uri = "dense/tsdf.vbg"
mem.dense.global_mesh_uri = "dense/global_mesh.glb"
mem.dense.global_3dgs_uri = "dense/3dgs.ply" # 若已训练
# 9. masks
prior_mask, no_update = build_masks(mem, voxel_grid_shape=(...),
voxel_size=0.02)
np.save(f"{out}/dense/prior_mask.npy", prior_mask)
np.save(f"{out}/dense/no_update_zone.npy", no_update)
mem.dense.prior_mask_uri = "dense/prior_mask.npy"
mem.dense.no_update_zone_uri = "dense/no_update_zone.npy"
# 10. 序列化 + 校验
save(mem, f"{out}/spatial_memory.json")
errs = validate(mem)
if errs:
print("❌ validation errors:")
for e in errs: print(" ", e)
else:
print(f"✅ LTM written to {out} (nodes={len(mem.nodes)}, "
f"edges={len(mem.edges)}, anchors={len(mem.anchors)})")
if __name__ == "__main__":
main()
```
运行：
```bash
python -m tools.prism_ingest_iphone \
--scan data/scans/2026-05-16_3F/ \
--out robot_memory/ltm/
```
---
## 4.10 质检清单（每次扫描后必查）
| 检查项 | 命令 | 通过标准 |
|--------|------|----------|
| 节点总数合理 | `jq '.nodes \| length' spatial_memory.json` | 10 房 ≈ 150–500 |
| 每房间至少 1 个 anchor | `python -m tools.check_anchors` | ✅ |
| 所有 L4 节点都有 `parent_room` | `validate()` | 无 dangling |
| 镜面区已标 `no_update_zone` | 可视化 mask | 卫生间镜面 100% 覆盖 |
| OctoMap 在客房中央可通行 | `python -m tools.viz_octomap` | 人眼检 |
| CLIP 检索"床"返回正确 bed 节点 | `python -m tools.search "床"` | Top-1 命中率 100% |
| 重定位测试（同帧重投影） | `python -m tools.test_relocalize` | RMSE < 3 cm |
---
## 4.11 重扫触发条件
iPhone 扫描不是"一次到永远"。下列情况必须重扫：
| 触发 | 检测者 | 动作 |
|------|--------|------|
| 装修 / 大改 | 人工 | 全场景重扫 |
| > 30% 大件家具被搬动 | Consolidator 统计 | 自动提示运维 |
| Anchor 总数 < 阈值（10 房 < 15 个） | 自检 | 提示补扫 |
| LTM 已超 90 天 | 定时任务 | 提示走查（不强制） |
重扫策略：用 `snapshots/` 保留旧版本，新版用 `ltm_version: N+1` 写入，回滚友好。
---
## 4.12 本章小结
| 阶段 | 输入 | 输出 |
|------|------|------|
| A1 采集 | 人 + iPhone Pro + ArUco | USDZ + JSON + ARKit raw |
| A2 传输 | iPhone | Mac/Linux 上 data/scans/ |
| A3 对齐 | ArUco + ARKit pose | `T_iphone→map` |
| A4 解析 | RoomPlan JSON | L3+L4 节点与边 |
| A5 几何派生 | USDZ + ARKit 深度 | OctoMap + TSDF + 3DGS + 每件 mesh + masks |
| A6 CLIP | RGB + mesh 渲染 | 每节点的 (512,) embedding |
| A7 Anchors | 节点子集 | `anchors.json` |
| A8 落盘 | 所有上述 | `robot_memory/ltm/` |
完成本管线后，**LTM 已就绪**，等待 ZED 2i 上机器人后做 [Chapter 05](05_pipeline_B_relocalization.md) 的握手重定位。
---
**章节版本**：v1.0
**估计阅读时间**：20 分钟
**关键收获**：可立即跑通的 `prism-ingest-iphone` 完整管线
plans/PRISM/05_pipeline_B_relocalization.md
+368
View File
@@ -0,0 +1,368 @@
# Chapter 05 — 管线 B：ZED 2i 重定位握手
> 本章目标：机器人上电后，**用 ZED 2i 当前观测把自己"安放"到 iPhone 先验地图的 `map` 坐标系里**——这是两个方案能合体的"握手时刻"。
---
## 5.1 为什么必须重定位
ZED 2i 自己能跑 VIO，但 VIO 给出的位姿是相对于"**ZED 开机时所在的某点**"，与 iPhone `map` 帧无关。如果不做重定位：
- ❌ 机器人查 LTM 时，「床」的全局坐标对它毫无意义
- ❌ 多次开机后，每次的"原点"都不同
- ❌ 长距离漂移无法用先验校正
**重定位 = 计算一个一次性的 `T_zed→map`，把后续所有 ZED VIO 输出都左乘这个矩阵。**
---
## 5.2 触发条件
| 触发场景 | 模式 |
|----------|------|
| 上电首次 | **冷启动**：无任何先验位姿 |
| 长时间漂移（> 30 s 未匹配 anchor） | **温启动**：有粗略先验 |
| 跟丢 / 绑架（被人抱起放下） | **绑架恢复**：可能瞬移到任意房间 |
| 跨房间穿门 | **过门校正**（轻量）：仅刷新位姿不重头 |
| 周期性（每 5 min） | **健康检查**：核对漂移 |
冷启动与绑架恢复需要全局检索，温启动只在小范围验证。
---
## 5.3 两段式重定位策略：粗 → 精
```mermaid
flowchart TB
S1["<b>Stage 1: 粗匹配</b>（CLIP / DINO 视觉指纹）<br/>──────<br/>输入：当前 ZED RGB 帧<br/>输出：Top-K 候选房间 (L3 nodes)<br/>耗时：50–150 ms<br/>召回率目标：Top-3 > 95%"]
S2["<b>Stage 2: 精配准</b>（点云 ICP / TEASER++）<br/>──────<br/>输入：ZED 当前点云 vs 候选房间 mesh/anchor<br/>输出：T_zed→map 及其 fitness 分数<br/>耗时：200–800 ms<br/>位置误差目标：< 10 cm"]
Q{"fitness > 0.7 ?"}
OK(["接受，发布 TF"])
FB(["退到下一候选 / 走 Stage 1.b 兜底"])
S1 --> S2 --> Q
Q -- yes --> OK
Q -- no --> FB
style S1 fill:#e3f2fd,stroke:#1565c0
style S2 fill:#fff7d6,stroke:#c97a00
style OK fill:#d4f0d4,stroke:#2e7d32
style FB fill:#fde2e2,stroke:#a33
```
---
## 5.4 Stage 1 — 视觉粗匹配
### 5.4.1 CLIP 房间检索（默认方案）
```python
# spatial_memory/relocalize_coarse.py
import open_clip, torch, numpy as np
class CoarseRelocalizer:
def __init__(self, mem: SpatialMemory):
self.mem = mem
self.model, _, self.preprocess = open_clip.create_model_and_transforms("ViT-B-32")
self.model.eval().cuda()
# 预加载 L3 房间向量库
self.room_uids, self.room_feats = [], []
for uid, node in mem.nodes.items():
if node.level == "L3" and node.clip_embedding is not None:
self.room_uids.append(uid)
self.room_feats.append(node.clip_embedding)
self.room_feats = np.stack(self.room_feats).astype(np.float32)
# 归一化
self.room_feats /= np.linalg.norm(self.room_feats, axis=1, keepdims=True)
@torch.no_grad()
def __call__(self, zed_rgb: np.ndarray, top_k: int = 3) -> List[Tuple[str, float]]:
from PIL import Image
img = Image.fromarray(zed_rgb)
x = self.preprocess(img).unsqueeze(0).cuda()
feat = self.model.encode_image(x).cpu().numpy()[0]
feat /= np.linalg.norm(feat)
sims = self.room_feats @ feat # (N,)
idx = np.argsort(-sims)[:top_k]
return [(self.room_uids[i], float(sims[i])) for i in idx]
```
### 5.4.2 增强方案：DINOv2 / NetVLAD
CLIP 在"语义相似但几何不同"（如所有酒店客房都长得差不多）时会混。增强做法：
```python
# 用 DINOv2 给出更强的"场景几何指纹"
import dinov2
class HybridCoarse:
def __init__(self, mem):
self.clip_r = CoarseRelocalizer(mem)
self.dino = dinov2.load("dinov2_vitb14")
# 建议在 ingest 阶段同时离线提 DINO 特征存进 node.attributes['dino_emb']
...
def __call__(self, rgb):
clip_top = self.clip_r(rgb, top_k=5)
# 再用 DINO 重排
dino_feat = extract_dino(self.dino, rgb)
reranked = []
for uid, _ in clip_top:
d = cosine(dino_feat, self.mem.nodes[uid].attributes['dino_emb'])
reranked.append((uid, d))
reranked.sort(key=lambda x: -x[1])
return reranked[:3]
```
### 5.4.3 投票（取连续多帧增强稳定性）
```python
def coarse_with_voting(coarse_fn, zed_stream, window=5) -> str:
votes = {}
for _ in range(window):
frame = zed_stream.get_rgb()
for uid, sim in coarse_fn(frame, top_k=3):
votes[uid] = votes.get(uid, 0) + sim
return max(votes, key=votes.get)
```
---
## 5.5 Stage 2 — 几何精配准
### 5.5.1 流程
```python
# spatial_memory/relocalize_fine.py
import open3d as o3d
import numpy as np
def fine_register(zed_depth, zed_intrinsics, candidate_room_uid,
mem: SpatialMemory,
voxel: float = 0.05) -> Tuple[np.ndarray, float]:
# 1) ZED 帧 → 点云
src = depth_to_pointcloud(zed_depth, zed_intrinsics)
src = src.voxel_down_sample(voxel)
src.estimate_normals()
# 2) 房间 mesh 采样
room_mesh_path = f"robot_memory/ltm/meshes/{candidate_room_uid}.glb" # 房间 mesh
# 若按房间没单独 mesh，则用 global mesh + 房间多边形裁剪
tgt = mesh_to_pointcloud(room_mesh_path, n=200_000)
tgt = tgt.voxel_down_sample(voxel)
tgt.estimate_normals()
# 3) 全局粗配准 (TEASER++ 或 RANSAC+FPFH)
src_fpfh = o3d.pipelines.registration.compute_fpfh_feature(
src, o3d.geometry.KDTreeSearchParamHybrid(voxel*5, 100))
tgt_fpfh = o3d.pipelines.registration.compute_fpfh_feature(
tgt, o3d.geometry.KDTreeSearchParamHybrid(voxel*5, 100))
result_ransac = o3d.pipelines.registration.registration_ransac_based_on_feature_matching(
src, tgt, src_fpfh, tgt_fpfh, mutual_filter=True,
max_correspondence_distance=voxel*1.5,
estimation_method=o3d.pipelines.registration.TransformationEstimationPointToPoint(False),
ransac_n=4,
checkers=[o3d.pipelines.registration.CorrespondenceCheckerBasedOnEdgeLength(0.9),
o3d.pipelines.registration.CorrespondenceCheckerBasedOnDistance(voxel*1.5)],
criteria=o3d.pipelines.registration.RANSACConvergenceCriteria(100000, 0.999))
# 4) ICP 精化
result_icp = o3d.pipelines.registration.registration_icp(
src, tgt, voxel*0.5, result_ransac.transformation,
o3d.pipelines.registration.TransformationEstimationPointToPlane())
return result_icp.transformation, result_icp.fitness
```
### 5.5.2 候选轮询
```python
def relocalize(zed_frame, mem) -> RelocalizeResult:
coarse = CoarseRelocalizer(mem)
candidates = coarse(zed_frame.rgb, top_k=3)
best = None
for room_uid, _ in candidates:
T, fit = fine_register(zed_frame.depth, zed_frame.intrinsics,
room_uid, mem)
if best is None or fit > best.fitness:
best = RelocalizeResult(T_zed_to_map=T, fitness=fit,
room_uid=room_uid)
if fit > 0.85: # 高质量提前退出
break
return best
```
### 5.5.3 接受 / 拒绝阈值
| fitness | 行动 |
|---------|------|
| > 0.85 | 接受，直接发布 |
| 0.70–0.85 | 接受但标记 `confidence=medium`，触发 5 s 内复检 |
| 0.50–0.70 | 拒绝，换 anchor 级精配（5.6）再试 |
| < 0.50 | 失败，进入人工兜底 |
---
## 5.6 Anchor 级超精配（对付 fitness 偏低）
当房间级配准 fitness < 0.7，可能是因为房间太大、视场只看到局部。退化为 **anchor 级**：
```python
def fine_register_anchor(zed_pc, anchor: Anchor, mem) -> Tuple[np.ndarray, float]:
node = mem.nodes[anchor.anchor_uid]
# 取该 anchor 的局部 mesh（如这张床）
tgt_pc = mesh_to_pointcloud(f"ltm/meshes/{node.uid}.glb", n=20_000)
# 把 tgt 变换到 map 帧
T_map = node.pose.to_matrix()
tgt_pc = tgt_pc.transform(T_map)
# 在 src 里**先用 YOLO 切出对应物体**，再 ICP
src_pc = crop_pc_by_yolo_detection(zed_pc, label=node.label)
if len(src_pc.points) < 500:
return None, 0.0
T_init = best_guess_init(node, zed_pose_estimate)
T, fit = open3d_icp(src_pc, tgt_pc, T_init)
return T, fit
```
→ 这种做法只用"床这个物体"对齐，比整个房间快也更鲁棒。
---
## 5.7 在线维护：连续 anchor 校正
冷启动成功后，机器人开始巡逻。VIO 会慢慢漂，需要持续校正：
```python
class OnlineRelocalizer:
def __init__(self, mem, vio):
self.mem = mem
self.vio = vio
self.T_zed_to_map = np.eye(4) # 上次重定位结果
self.last_correction_t = 0
self.drift_estimate = 0.0 # 累计漂移估计
def step(self, zed_frame):
# 1. 取 ZED VIO 当前位姿（zed 帧）
T_robot_zed = self.vio.current_pose()
# 2. 全局位姿
T_robot_map = self.T_zed_to_map @ T_robot_zed
# 3. 用 YOLO 看当前帧有没有 anchor 类家具
detections = yolo_world(zed_frame.rgb, classes=ANCHOR_LABELS)
for det in detections:
anchor = match_to_anchor(det, T_robot_map, self.mem)
if anchor is None: continue
# 4. 局部 ICP 校正
T_new, fit = fine_register_anchor(zed_frame.pc, anchor, self.mem)
if fit > 0.8:
# 用新 T 替换全局
self.T_zed_to_map = T_new @ np.linalg.inv(T_robot_zed)
self.last_correction_t = time.time()
self.drift_estimate = 0
return
# 没看到 anchor：用 IMU 估计漂移
self.drift_estimate += self.vio.expected_drift_per_sec * dt
if self.drift_estimate > 0.5: # 50 cm 漂移触发主动重定位
self.trigger_full_relocalize()
```
---
## 5.8 输出：ROS 2 TF 发布
把握手结果发布到 TF 树，让导航栈/Agent 用：
```python
# nodes/relocalizer_node.py
import rclpy
from rclpy.node import Node
from geometry_msgs.msg import TransformStamped
from tf2_ros import StaticTransformBroadcaster
class RelocalizerNode(Node):
def __init__(self):
super().__init__("prism_relocalizer")
self.br = StaticTransformBroadcaster(self)
def publish(self, T_zed_to_map: np.ndarray, stamp):
msg = TransformStamped()
msg.header.stamp = stamp
msg.header.frame_id = "map"
msg.child_frame_id = "zed2i_init" # 锚定 ZED 起点
msg.transform.translation.x = float(T_zed_to_map[0,3])
msg.transform.translation.y = float(T_zed_to_map[1,3])
msg.transform.translation.z = float(T_zed_to_map[2,3])
q = matrix_to_quat(T_zed_to_map[:3,:3])
msg.transform.rotation.w, msg.transform.rotation.x, \
msg.transform.rotation.y, msg.transform.rotation.z = q
self.br.sendTransform(msg)
```
TF 树：
```mermaid
flowchart LR
MAP["map"] --> INIT["zed2i_init<br/><i>(相对固定，<br/>重定位时刷新)</i>"] --> VIO["zed2i_camera (VIO)<br/><i>(30 Hz VIO 输出)</i>"] --> BASE["base_link"]
style MAP fill:#fff7d6,stroke:#c97a00
style INIT fill:#ffe9b3,stroke:#c97a00
```
---
## 5.9 失败兜底（人工 / 半自动）
如果 fitness 一直 < 0.5：
| 兜底方式 | 操作 | 适用 |
|----------|------|------|
| **二维码兜底** | 在每个房间门口贴 1 个 ArUco（与 iPhone 扫描时的一致） | 永久可靠，建议默认配置 |
| **遥控引导** | 人手柄遥控机器人到某 anchor 前，按"我在这"按钮 | 应急 |
| **iPhone 联动** | 用员工的 iPhone 走到机器人旁，App 计算两者相对位姿 | 黑科技兜底 |
---
## 5.10 评测协议
每次部署后跑一遍：
| 测试 | 方法 | 通过标准 |
|------|------|----------|
| 冷启动重定位 | 机器人随机放在 10 个位置开机 | 9/10 成功，位置误差 < 15 cm |
| 跨房间一致性 | 跑同一条路径 3 次，比较返回时位姿 | 漂移 < 20 cm |
| 绑架恢复 | 机器人运行中被抱到另一房间放下 | 30 s 内恢复 |
| 高反光区 | 把机器人放在卫生间镜前 | 应自动跳过镜面区做 ICP，不挂 |
| 黑暗 | 关灯只留小夜灯 | CLIP 降级，仍能由几何 ICP 兜住 |
---
## 5.11 性能预算（Jetson Orin AGX）
| 阶段 | 耗时 | GPU 占用 |
|------|------|----------|
| Stage 1 CLIP | 80 ms | 1.2 GB |
| Stage 1 DINO 重排（可选） | +60 ms | 2 GB |
| Stage 2 RANSAC+FPFH | 300 ms | 0 (CPU) |
| Stage 2 ICP refine | 100 ms | 0 |
| 总冷启动 | < 1 s | < 3 GB |
| 在线 anchor 校正 | 200 ms / 触发 | 1 GB |
---
## 5.12 本章小结
| 关键点 | 一句话 |
|--------|--------|
| **目的** | 算一个 `T_zed→map`，让 ZED 的所有后续输出与 iPhone LTM 同框 |
| **策略** | 两段式：CLIP 粗匹配 → ICP 精配准 |
| **效率** | 冷启动 < 1 s，在线 anchor 校正 < 200 ms |
| **鲁棒** | 多帧投票 + 多 anchor 轮询 + ArUco 兜底 |
| **维护** | 巡逻中持续看到 anchor 就刷新一次，对抗 VIO 长漂 |
读完本章你应能：
- ✅ 实现一个 ROS 2 节点，3 s 内告诉机器人"我在 305 房间床前"
- ✅ 设计鲁棒性测试用例
- ✅ 在 fitness 低时知道如何回退
下一章 [`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md) 讲机器人已经定位后，怎么把在线观测**写回**记忆。
---
**章节版本**：v1.0
**估计阅读时间**：18 分钟
**关键收获**：从"我在哪都不知道"到"我精确在 map 帧 (1.23, 4.56, 0.0)"的完整握手流程
plans/PRISM/06_pipeline_C_online_perception.md
+550
View File
@@ -0,0 +1,550 @@
# Chapter 06 — 管线 C：在线感知与差异检测
> 本章目标：机器人已定位（Chapter 05 完成）后，**把 ZED 2i 30 Hz 流式数据持续写入 L1/L2，并在与 LTM 不一致时记录到 `delta/`**，但**不直接修改 LTM**（修改交给 Chapter 07 的巩固阶段）。
---
## 6.1 设计原则：四条铁律
1. **不阻塞**：感知主循环必须能跑满 30 Hz，慢操作（CLIP / VLM）放异步队列
2. **不破坏**：LTM 永远只读；ZED 的所有"修改意图"都写到 `delta/`
3. **不耗内存**：L1 是环形缓冲，老数据自动覆盖；L2 局部 TSDF 与全局合并是后台任务
4. **可追溯**：每条 delta 都带 `evidence`（关键帧 ID 列表），方便巩固时复核
---
## 6.2 主循环架构
```mermaid
flowchart TB
SDK["📷 ZED SDK<br/><i>30 Hz: pose, rgb_left, depth, imu</i>"]
FP["Frame Producer (30 Hz)<br/>写入 L1 环形缓冲"]
AV["Avoidance<br/>(30 Hz)"]
TSDF["TSDF Worker (5 Hz)<br/>depth → L2"]
DET["Detector Worker (2 Hz)<br/>rgb → YOLO / VLM"]
CD["Change Detector<br/>(TSDF vs LTM)"]
OA["Object Associator<br/>(det ↔ L4 node)"]
DELTA[("delta/pending.jsonl")]
SDK --> FP
FP --> AV
FP --> TSDF
FP --> DET
TSDF --> CD
DET --> OA
CD --> DELTA
OA --> DELTA
style SDK fill:#e3f2fd,stroke:#1565c0
style FP fill:#fff7d6,stroke:#c97a00
style DELTA fill:#f5e1ff,stroke:#7b1fa2
style AV fill:#fde2e2,stroke:#a33
style TSDF fill:#fff1c1,stroke:#a87a00
style DET fill:#d4f0d4,stroke:#2e7d32
```
三个 worker **独立频率、独立队列**，通过 Python `multiprocessing.Queue` 或 ROS 2 topic 通信。
---
## 6.3 Frame Producer（L1 写入器）
```python
# perception/frame_producer.py
import pyzed.sl as sl
from collections import deque
import numpy as np, time
class FrameProducer:
def __init__(self, capacity_sec=10.0, fps=30):
cam = sl.Camera()
init = sl.InitParameters()
init.camera_resolution = sl.RESOLUTION.HD720
init.camera_fps = fps
init.depth_mode = sl.DEPTH_MODE.QUALITY
init.coordinate_units = sl.UNIT.METER
init.coordinate_system = sl.COORDINATE_SYSTEM.RIGHT_HANDED_Z_UP
cam.open(init)
cam.enable_positional_tracking(sl.PositionalTrackingParameters())
self.cam = cam
self.buf = deque(maxlen=int(capacity_sec * fps))
self.keyframes = deque(maxlen=200)
self.last_kf_t = 0
self.kf_interval = 0.5 # 关键帧每 0.5 s 一张
def step(self) -> PerceptualFrame:
rt = sl.RuntimeParameters()
if self.cam.grab(rt) != sl.ERROR_CODE.SUCCESS:
return None
# 取数据
rgb = sl.Mat(); self.cam.retrieve_image(rgb, sl.VIEW.LEFT)
dpth = sl.Mat(); self.cam.retrieve_measure(dpth, sl.MEASURE.DEPTH)
pose = sl.Pose(); self.cam.get_position(pose, sl.REFERENCE_FRAME.WORLD)
frame = PerceptualFrame(
timestamp=time.time(),
pose=zed_pose_to_Pose(pose),
rgb_left=rgb.get_data()[:, :, :3].copy(),
depth=dpth.get_data().copy(),
imu_packet=self.cam.get_sensors_data(...))
self.buf.append(frame)
if frame.timestamp - self.last_kf_t > self.kf_interval:
self.keyframes.append(frame)
self.last_kf_t = frame.timestamp
return frame
```
**注意**：ZED VIO 输出的位姿是 `T_camera→world(zed)`；需要左乘 Chapter 05 的 `T_zed→map` 得到 `T_camera→map`。这一步在 `zed_pose_to_Pose` 里做。
---
## 6.4 TSDF Worker（L2 写入器）
### 6.4.1 局部 TSDF 持续融合
```python
# perception/tsdf_worker.py
import open3d as o3d
import numpy as np
class TSDFWorker:
def __init__(self, l2_memory, voxel=0.02, hop=0.2):
self.l2 = l2_memory
self.voxel = voxel
# 实时维护的"局部 TSDF"：跟随机器人，半径 5 m 范围
self.local = o3d.t.geometry.VoxelBlockGrid(
attr_names=('tsdf','weight','color'),
attr_dtypes=(o3d.core.float32,)*3,
attr_channels=((1,),(1,),(3,)),
voxel_size=voxel, block_resolution=16, block_count=10000,
device=o3d.core.Device("CUDA:0"))
self.last_pos = None
def integrate(self, kf: PerceptualFrame, intrinsics):
depth_o3d = o3d.t.geometry.Image(kf.depth).to("CUDA:0")
T = kf.pose.to_matrix()
# 跳过镜面区与 mobile 家具区（参考 prior_mask / no_update_zone）
mask = self.l2.compute_skip_mask(kf.pose, kf.depth.shape)
depth_o3d = depth_o3d * (1.0 - mask) # 在 GPU 上掩膜
frustum_blocks = self.local.compute_unique_block_coordinates(
depth_o3d, intrinsics, T, depth_scale=1.0, depth_max=5.0)
self.local.integrate(frustum_blocks, depth_o3d, intrinsics, T,
depth_scale=1.0, depth_max=5.0)
def export_local_pc(self) -> o3d.t.geometry.PointCloud:
return self.local.extract_point_cloud()
```
### 6.4.2 局部 → 全局合并（后台 1 Hz）
```python
def merge_local_to_global(self):
"""每秒一次把局部 TSDF 的稳定块合并到全局"""
stable_blocks = self.local.get_blocks_with_weight_above(threshold=8)
for blk in stable_blocks:
self.l2.tsdf.merge_block(blk, weight_prior=self.l2.prior_weight(blk.coord))
# 老的局部块过期淘汰（机器人已离开 > 5 m）
self.local.prune_blocks_far_from(self.current_pose, max_dist=6.0)
```
### 6.4.3 prior_mask 起作用的地方
```python
def compute_skip_mask(self, pose, depth_shape) -> np.ndarray:
"""根据当前视锥决定哪些像素的写入应被降权或跳过"""
H, W = depth_shape
mask = np.zeros((H, W), dtype=np.float32)
# 1) 镜面区（投影 LTM 的 no_update_zone 到当前像素）
proj_mirror = project_voxels_to_image(self.l2.no_update_zone, pose, intrinsics)
mask[proj_mirror] = 1.0
# 2) 先验墙区（不是不写，而是低权重；让 integrate 收到 weight=0.1）
# 在 integrate 里另做
return mask
```
---
## 6.5 Detector Worker（L4 增量写入器）
### 6.5.1 检测 + 关联
```python
# perception/detector_worker.py
from ultralytics import YOLO
import torch
class DetectorWorker:
def __init__(self, mem, vlm=None):
self.mem = mem
self.yolo = YOLO("yolov8x-worldv2.pt")
# 用 LTM 里出现过的标签作为开放词表
all_labels = set(n.label for n in mem.nodes.values()
if n.level == "L4")
# 额外加常见小物品
all_labels |= {"remote","cup","bottle","phone","book","towel",
"luggage","backpack","slipper"}
self.yolo.set_classes(list(all_labels))
self.vlm = vlm # optional GPT-4V / Qwen-VL
def step(self, frame: PerceptualFrame):
results = self.yolo.predict(frame.rgb_left, conf=0.3, verbose=False)
detections = parse_yolo(results)
events = []
for det in detections:
# 1) 把 2D bbox + depth → 3D bbox（map 帧）
bbox3d = lift_2d_to_3d(det.bbox_2d, frame.depth,
frame.intrinsics, frame.pose)
# 2) 与 LTM 中已有节点关联
match = associate(bbox3d, det.label, self.mem)
if match is not None:
events.append(self.update_existing(match, det, bbox3d, frame))
else:
events.append(self.propose_new(det, bbox3d, frame))
return events
```
### 6.5.2 关联算法（detection ↔ LTM node）
```python
def associate(bbox3d_obs, label, mem, dist_thresh=0.5):
"""简单贪心：找同 label、距离最近的节点"""
cand = [n for n in mem.nodes.values()
if n.level == "L4" and n.label == label]
if not cand:
return None
center_obs = bbox3d_obs.mean(axis=0)
best, best_d = None, float("inf")
for n in cand:
center_n = n.pose.position
d = np.linalg.norm(center_obs - center_n)
if d < best_d:
best, best_d = n, d
return best if best_d < dist_thresh else None
```
更稳健的做法是用 **CLIP embedding 相似度 + 几何距离的加权**：
```python
def associate_hybrid(rgb_crop, bbox3d, label, mem,
w_geo=0.5, w_clip=0.5):
geo_scores = []
clip_scores = []
crop_feat = clip_encode_image(rgb_crop)
for n in mem.nodes_of_label(label):
d = np.linalg.norm(bbox3d.mean(0) - n.pose.position)
s_g = np.exp(-d / 0.5)
s_c = cosine(crop_feat, n.clip_embedding) if n.clip_embedding is not None else 0
geo_scores.append(s_g); clip_scores.append(s_c)
total = w_geo*np.array(geo_scores) + w_clip*np.array(clip_scores)
idx = int(np.argmax(total))
return list(mem.nodes_of_label(label))[idx] if total[idx] > 0.5 else None
```
### 6.5.3 更新已有节点
```python
def update_existing(self, node, det, bbox3d, frame) -> DeltaEvent:
node.last_seen = frame.timestamp
node.observation_count += 1
# 位姿用 EMA 平滑更新
alpha = 0.1 # iPhone 来的节点更"硬"
new_center = bbox3d.mean(axis=0)
old_center = node.pose.position
drift = np.linalg.norm(new_center - old_center)
if drift > 0.20: # 偏离 > 20 cm，写 delta
return DeltaEvent(
event_id=uuid(),
event_type="object_moved",
target_uid=node.uid,
new_pose=Pose(new_center, est_quat(bbox3d)),
new_bbox=bbox3d,
evidence=[frame.keyframe_id])
else:
node.pose.position = alpha*new_center + (1-alpha)*old_center
return None # 微调，不写 delta
```
### 6.5.4 新增节点提案
```python
def propose_new(self, det, bbox3d, frame) -> DeltaEvent:
"""ZED 看到 LTM 没记录的物品 → 候选节点"""
new_node = SpatialNode(
uid=SpatialNode.new_uid("zed"),
label=det.label,
category="small_item" if det.label in SMALL_ITEMS else "furniture",
level=MemoryLevel.L4,
source=Source.ZED2I,
confidence=0.3, # 初始低置信
pose=Pose(bbox3d.mean(0), est_quat(bbox3d)),
bbox_3d=bbox3d,
parent_room=find_room_for_point(bbox3d.mean(0)[:2], self.mem),
attributes={"mobile": True},
observation_count=1)
return DeltaEvent(
event_id=uuid(),
event_type="object_added",
target_uid=None,
new_pose=new_node.pose,
new_bbox=bbox3d,
evidence=[frame.keyframe_id],
# 嵌入候选节点作为 payload（巩固时直接 add_node）
payload=asdict(new_node))
```
---
## 6.6 Change Detector（TSDF vs LTM）
> ⚠️ **v1.5 起本节描述的写法已被 [§ 6.6b v1.5 新写法:keyframe-based 内容更新(代替 L2 几何反推)](#66b-v15-新写法keyframe-based-内容更新代替-l2-几何反推) 升级。** 本节中"从 TSDF voxel 直接重投影生成 L3/L4 节点 patch 与 CLIP 嵌入"的旧写法仍保留以备追溯,但实际数据流应改为:**TSDF 只贡献"哪个节点被触发"的路由信号,节点内容(patch / CLIP / bbox)从 L1 keyframe 缓存读取**。详见 §6.6b 与 [`02_architecture.md` §2.7b](02_architecture.md#27b-l2-l3-数据流路由-vs-内容v15-新原则)。
家具被搬走的检测来源不是物体检测的"没看到"（视野限制太多），而是 **TSDF 的几何变化**：
```python
# perception/change_detector.py
def detect_geometric_change(local_tsdf, ltm_tsdf, ltm_mesh,
voxel=0.02, threshold=0.05):
"""对比局部实时 TSDF 与 LTM TSDF；返回变化体素"""
overlap_blocks = local_tsdf.overlap_with(ltm_tsdf)
moved_voxels, added_voxels = [], []
for blk in overlap_blocks:
local_sdf = local_tsdf.get_sdf(blk)
prior_sdf = ltm_tsdf.get_sdf(blk)
weight = local_tsdf.get_weight(blk)
# 只在 weight 足够时下判断
diff = (prior_sdf - local_sdf) * (weight > 5.0)
moved_voxels.extend(blk.coords_where(diff > threshold))
added_voxels.extend(blk.coords_where(diff < -threshold))
return moved_voxels, added_voxels
def cluster_to_event(voxels, mem, kind: str):
"""把散乱的变化体素聚类成"对象级"事件"""
pts = np.array(voxels)
if len(pts) < 50: # 太小忽略
return []
labels = dbscan_cluster(pts, eps=0.15, min_samples=20)
events = []
for cid in set(labels):
if cid == -1: continue
cluster = pts[labels==cid]
bbox = aabb_from_points(cluster)
center = cluster.mean(axis=0)
# 找该位置对应的 LTM 节点
target = nearest_node_in_bbox(self.mem, bbox)
events.append(DeltaEvent(
event_id=uuid(),
event_type="object_removed" if kind=="moved"
else "object_added",
target_uid=target.uid if target else None,
new_bbox=bbox if kind=="added" else None,
evidence=current_keyframe_ids()))
return events
```
调用频率：1 Hz 即可。
---
## 6.6b v1.5 新写法:keyframe-based 内容更新(代替 L2 几何反推)
### 6.6b.1 动机
v1.4 之前,差异检测在发现 TSDF voxel 变化后,会直接从那些变化的 voxel **重投影**回当前视图,把 voxel 颜色/法线聚合成一块 patch,再喂给 CLIP 生成 L3/L4 节点的视觉指纹。这种"从几何反推内容"的做法有两个固有缺陷:
1. **量化损失**:TSDF 默认 2 cm 体素,远小于 CLIP encoder 期望的 224×224 patch 细节;voxel 颜色已经过加权平均,CLIP 嵌入精度受 voxel 噪声放大。
2. **角度损失**:voxel 重投影出来的"虚拟视图"未必与机器人当初看到该物体的最佳视角一致,导致同一物体在不同时刻的嵌入漂移。
v1.5 起,我们采纳 [`02_architecture.md` §2.7b](02_architecture.md#27b-l2-l3-数据流路由-vs-内容v15-新原则) 的新原则:**TSDF 体素只用来"路由"——告诉系统"L3 图里哪个节点该被更新";真正的节点内容(RGB patch / CLIP / 文字描述)直接从 L1 缓存中那一张原始 keyframe 上取**。这样 CLIP 嵌入面对的是未经量化的高分辨率像素与机器人当初真实采到的视角,语义保真度显著上升。
### 6.6b.2 旧 vs 新伪代码并列对比
**v1.4 旧写法(从 TSDF voxel 重投影获取颜色)**:
```python
# v1.4: 内容来自 L2 voxel 反推,受量化限制
def update_l3_node_v1_4(changed_voxels, l2_tsdf, l3_graph):
target_uid = nearest_l3_node(changed_voxels.mean(axis=0), l3_graph)
# ① 把 voxel 颜色聚合 → 虚拟 patch(几何反推,信息损失)
voxel_colors = l2_tsdf.get_colors(changed_voxels) # 2 cm 量化
virtual_patch = reproject_voxels_to_image(
changed_voxels, voxel_colors,
fake_camera=synth_view(changed_voxels)) # 视角是合成的
# ② 用合成 patch 算 CLIP(精度天然受限)
clip_emb = clip_encode_image(virtual_patch)
l3_graph[target_uid].clip_embedding = clip_emb
l3_graph[target_uid].bbox_3d = aabb_from_voxels(changed_voxels)
```
**v1.5 新写法(从 L1 keyframe 原图取 patch)**:
```python
# v1.5: 路由来自 L2,内容来自 L1 keyframe 原图
def update_l3_node_v1_5(changed_voxels, l2_tsdf, l1_buffer, l3_graph):
# ① L2 只做路由:决定写哪个 L3 节点
target_uid = nearest_l3_node(changed_voxels.mean(axis=0), l3_graph)
# ② 从 L1 keyframe 池里挑"看该区域最清楚的那一帧"
region_center = changed_voxels.mean(axis=0)
best_kf = l1_buffer.pick_best_keyframe(
target_point=region_center,
criteria=("nearest_pose", "max_pixel_coverage", "min_blur"))
# ③ 在原始 RGB 上裁出 patch(无量化,真实视角)
real_patch = crop_keyframe_to_region(best_kf, region_center)
clip_emb = clip_encode_image(real_patch) # 高保真
l3_graph[target_uid].clip_embedding = clip_emb
l3_graph[target_uid].bbox_3d = aabb_from_voxels(changed_voxels)
l3_graph[target_uid].evidence_keyframes.append(best_kf.id)
```
关键差异:旧版第 ① 步就把"路由"与"取内容"绑死在 L2 voxel 上;新版第 ① 步把路由留在 L2(`nearest_l3_node` 只看 voxel 位置,不看颜色),第 ② 步明确转身向 L1 索取原始观测,第 ③ 步在未经量化的 RGB 上做 CLIP 编码。
### 6.6b.3 验证策略与评测预期
本改动主要影响 [`13_evaluation.md`](13_evaluation.md) 中"语义一致性"相关指标:同一物体在不同时段被重观测时,其 CLIP 嵌入的余弦相似度应当上升。基于 patch 分辨率从 ~2 cm voxel 重投影提升到 RGB 原图(≥ 720p)、视角从合成视角恢复为真实采集视角这两个因素,**工程预估**新写法相对 v1.4:
- 同物体跨时段 CLIP 嵌入余弦相似度:**+0.05 ~ +0.10** 个点(注:**工程预估而非实测**,需在 §13 评测里跑 A/B 验证)
- 对 L3 节点 `clip_embedding` 做的"文本→物体"检索 top-1 命中率应同步上升
落地后请在 [`13_evaluation.md`](13_evaluation.md) 的"语义一致性"小节补一组 v1.4 vs v1.5 的对照实验,把上述预估替换为真实数字。
---
## 6.7 Delta 写入与去重
`delta/pending.jsonl` 是 append-only 的事件日志，但需要去重 + 累计观测数：
```python
class DeltaLog:
def __init__(self, path="robot_memory/delta/pending.jsonl"):
self.path = path
self.events_by_signature: Dict[str, DeltaEvent] = {}
self._load_existing()
def append(self, ev: DeltaEvent):
sig = self._signature(ev)
if sig in self.events_by_signature:
old = self.events_by_signature[sig]
old.observation_count += 1
old.last_observed = ev.last_observed
old.evidence.extend(ev.evidence)
self._rewrite()
else:
self.events_by_signature[sig] = ev
with open(self.path, "a") as f:
f.write(json.dumps(asdict(ev)) + "\n")
def _signature(self, ev):
# 同一 target + 同事件类型 + 中心点接近 → 视为同一事件
center = ev.new_pose.position if ev.new_pose else (0,0,0)
cell = tuple(np.round(np.array(center)/0.3).astype(int))
return f"{ev.event_type}|{ev.target_uid}|{cell}"
```
---
## 6.8 完整 ROS 2 节点编排
```yaml
# launch/prism_online.launch.yaml
nodes:
- name: zed_node
package: zed_wrapper
type: zed_camera
- name: prism_relocalizer
package: prism
type: relocalizer_node # Chapter 05
parameters:
ltm_dir: /robot_memory/ltm
- name: prism_frame_producer
package: prism
type: frame_producer
parameters:
keyframe_interval: 0.5
- name: prism_tsdf_worker
package: prism
type: tsdf_worker
parameters:
voxel: 0.02
max_dist: 5.0
- name: prism_detector_worker
package: prism
type: detector_worker
parameters:
detection_rate_hz: 2.0
- name: prism_change_detector
package: prism
type: change_detector
parameters:
rate_hz: 1.0
diff_threshold_m: 0.05
- name: prism_delta_log
package: prism
type: delta_log
parameters:
path: /robot_memory/delta/pending.jsonl
```
---
## 6.9 性能预算（Jetson Orin AGX 64 GB）
| 模块 | 频率 | CPU | GPU | 内存 |
|------|------|-----|-----|------|
| Frame Producer | 30 Hz | 1 core | 2 GB（ZED 自带） | 500 MB |
| TSDF Worker | 5 Hz | 1 core | 1.5 GB | 1 GB |
| Detector Worker (YOLO-World) | 2 Hz | 1 core | 3 GB | 800 MB |
| Change Detector | 1 Hz | 1 core | 0 | 200 MB |
| Relocalizer (待命) | 按需 | <1 core | 1 GB peak | 500 MB |
| **合计** | | 5 cores | < 8 GB | < 4 GB |
Orin AGX 12 核 + 64 GB，**留 50% 余量**给上层 Agent。
---
## 6.10 调试与可视化
提供两个 dashboard：
| 工具 | 用途 |
|------|------|
| `prism viz live` | RViz2 显示：当前位姿 + 局部 TSDF + 检测 bbox + delta 红框 |
| `prism viz delta` | Open3D 窗口：LTM mesh + delta 事件标点，鼠标点击可看 evidence keyframe |
---
## 6.11 异常处理
| 异常 | 表现 | 处理 |
|------|------|------|
| ZED 掉线 | grab 失败 | Frame Producer 重试 5 次，仍失败则降级模式（仅 IMU 推算） |
| TSDF OOM | GPU 内存爆 | 自动减小 `block_count`，丢弃最旧块 |
| YOLO 类别太多 | 推理慢 | 动态裁剪：只保留当前房间可能出现的类别 |
| delta 太多（异常情况） | 文件爆 | 触发紧急 Consolidation 或人工介入 |
---
## 6.12 本章小结
| 关键点 | 一句话 |
|--------|--------|
| **L1 写入** | Frame Producer 30 Hz |
| **L2 写入** | TSDF Worker 5 Hz，受 prior_mask / no_update_zone 保护 |
| **L4 写入** | Detector Worker 2 Hz；存在节点 → EMA 更新；新物 → 候选 |
| **差异感知** | 几何变化由 TSDF 对比 + DBSCAN 聚类得出 |
| **铁律** | 永远不直接改 LTM，所有"想改"都先入 `delta/pending.jsonl` |
| **性能** | Jetson Orin AGX 上 < 50% 资源占用 |
读完本章你应能：
- ✅ 在 ROS 2 启动整套在线感知节点
- ✅ 解释为什么必须有 prior_mask 与 no_update_zone
- ✅ 实现一个最小的 DeltaLog 并跑通去重
下一章 [`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md) 讲机器人"睡觉"时如何把 `delta/` 中的内容真正巩固进 LTM。
---
**章节版本**：v1.0
**估计阅读时间**：22 分钟
**关键收获**：可运行的实时感知三 worker 设计 + delta 写入机制
plans/PRISM/07_pipeline_D_consolidation.md
+551
View File
@@ -0,0 +1,551 @@
# Chapter 07 — 管线 D：记忆巩固 (Memory Consolidation)
> 本章目标：机器人**充电 / 空闲**时跑一次"睡眠"，把 `delta/pending.jsonl` 中**反复确认**的变化真正写入 LTM，并淘汰过时锚点、重训 3DGS、版本化备份。
---
## 7.1 为什么需要"睡眠"
如果 ZED 看到一次"沙发挪了"就立刻改 LTM，机器人就会变得**易骗**：
- 客人挪一下沙发拍照？被记成永久挪动
- ZED 单帧检测错把椅子识成桌子？长期记忆被污染
- 镜面区噪声偶尔产生"虚影"？写进去再也清不掉
仿照人脑：**白天积累短期记忆 → 睡眠时筛选 → 仅把多次确认的信号转入长期记忆**。
PRISM 也一样：
```mermaid
flowchart LR
subgraph DAY["白天（Online）"]
D1[("delta/pending.jsonl<br/><i>各种 DeltaEvent 累积，可能很乱</i>")]
end
subgraph NIGHT["夜里 / 充电时（Consolidation）"]
direction LR
N1["pending"] --> N2["筛选"] --> N3["confirmed"] --> N4["应用到 LTM"] --> N5["版本化"]
end
DAY -. 触发 .-> NIGHT
style DAY fill:#fff7d6,stroke:#c97a00
style NIGHT fill:#d8e4ff,stroke:#1565c0
```
---
## 7.2 触发条件
| 触发 | 频率 | 模式 |
|------|------|------|
| 充电桩 + 静止 > 10 min | 通常每天 1 次 | **完整巩固** |
| 手动命令 `prism consolidate` | 按需 | 完整巩固 |
| `pending.jsonl` > 5000 条 | 紧急 | **轻量巩固**（只筛选不重训） |
| LTM 版本年龄 > 30 天 | 月级 | **深度巩固**（含重算 anchors + CLIP） |
---
## 7.3 巩固总流程
> ⚠️ **v1.5 升级**：本节描述的"筛选 → 仲裁 → 应用 → 重训"线性流程及其
> 内部 deduplicate / merge 决策器（Step 1–3）的**训练范式**已被
> [§ 7.13.1](#7131-v15-升级self-augmentation-巩固训练) 增强为
> self-augmentation 监督学习；本节流程本身仍可作为 baseline 使用，
> 决策器若由简单规则（`PROMOTION_RULES` + `arbitrate()`）实现则完全不受影响。
> 仅当决策器升级为可训练策略网络（v1.5 → v2.0 路线图）时，§ 7.13.1
> 的 self-aug 范式才生效。
```mermaid
flowchart TB
P[("pending.jsonl")]
S1["<b>Step 1: 事件分类与筛选</b><br/>observation_count > N<br/>时间跨度 > T<br/>多 keyframe 多视角"]
S2["<b>Step 2: 冲突仲裁</b><br/>iPhone 标 vs ZED 改"]
S3["<b>Step 3: 应用到 LTM</b><br/>add / update / remove node"]
S4["<b>Step 4: 锚点 / 索引 / CLIP</b><br/>重新计算受影响项"]
S5["<b>Step 5: 增量 3DGS 重训</b><br/>只重训受影响房间"]
S6["<b>Step 6: 版本化 + 健康检查</b><br/>snapshots/, validate()"]
P --> S1 --> S2 --> S3 --> S4 --> S5 --> S6
style P fill:#f5e1ff,stroke:#7b1fa2
style S6 fill:#d4f0d4,stroke:#2e7d32
```
---
## 7.4 Step 1：事件筛选
```python
# consolidation/filter.py
from datetime import timedelta
# 不同事件类型有不同"晋升门槛"
PROMOTION_RULES = {
"object_moved": dict(min_obs=5, min_span_s=300, min_views=2),
"object_removed": dict(min_obs=10, min_span_s=600, min_views=3),
"object_added": dict(min_obs=8, min_span_s=300, min_views=2),
"geometry_changed":dict(min_obs=15, min_span_s=900, min_views=4),
}
def classify(ev: DeltaEvent, mem: SpatialMemory) -> str:
rule = PROMOTION_RULES[ev.event_type]
span = ev.last_observed - ev.first_observed
unique_views = count_unique_viewpoints(ev.evidence, mem)
# 若是 iPhone 的高 confidence 节点，门槛 1.5x
target = mem.nodes.get(ev.target_uid) if ev.target_uid else None
if target and target.source == "iphone" and target.confidence > 0.85:
rule = {k: v*1.5 for k, v in rule.items()}
if ev.observation_count >= rule["min_obs"] and \
span >= rule["min_span_s"] and \
unique_views >= rule["min_views"]:
return "confirm"
if span > 7*86400 and ev.observation_count < rule["min_obs"]//2:
return "reject" # 7 天还没攒够观测 → 拒绝
return "keep" # 继续等待
```
**关键直觉**：
- `min_obs`：要"看过很多次"
- `min_span_s`：必须**跨越足够长时间**（防止瞬时假象）
- `min_views`：必须**来自多个视点**（防止单点死磕）
---
## 7.5 Step 2：冲突仲裁
不同 delta 之间，或 delta 与 LTM 之间可能冲突：
```python
# consolidation/arbiter.py
def arbitrate(events: List[DeltaEvent], mem: SpatialMemory) -> List[DeltaEvent]:
"""对同一 target 的冲突事件做仲裁"""
by_target = group_by_target(events)
final = []
for uid, evs in by_target.items():
if len(evs) == 1:
final.append(evs[0]); continue
# 多个事件涉及同一节点
sorted_evs = sorted(evs, key=lambda e: e.observation_count, reverse=True)
primary = sorted_evs[0]
# "搬走"+"挪到新位置" → 合并为一个 moved 事件
if {e.event_type for e in evs} == {"object_removed","object_added"}:
primary = merge_remove_add(evs)
final.append(primary)
return final
def merge_remove_add(evs):
rem = next(e for e in evs if e.event_type == "object_removed")
add = next(e for e in evs if e.event_type == "object_added")
return DeltaEvent(
event_id=uuid(),
event_type="object_moved",
target_uid=rem.target_uid,
new_pose=add.new_pose,
new_bbox=add.new_bbox,
evidence=rem.evidence + add.evidence,
observation_count=rem.observation_count + add.observation_count,
first_observed=min(rem.first_observed, add.first_observed),
last_observed=max(rem.last_observed, add.last_observed))
```
---
## 7.6 Step 3：应用到 LTM
```python
# consolidation/apply.py
import shutil
from copy import deepcopy
def apply_events(events: List[DeltaEvent], mem: SpatialMemory) -> SpatialMemory:
"""返回应用后的新 SpatialMemory；不修改原对象"""
new = deepcopy(mem)
for ev in events:
if ev.event_type == "object_moved":
n = new.nodes[ev.target_uid]
n.pose = ev.new_pose
n.bbox_3d = ev.new_bbox
n.last_seen = ev.last_observed
n.confidence = min(1.0, n.confidence * 0.95) # 轻微降低（被动过）
# 重新计算 parent_room
n.parent_room = find_room_for_point(n.pose.position[:2], new)
elif ev.event_type == "object_removed":
uid = ev.target_uid
n = new.nodes[uid]
# 不立刻硬删除，标记 deprecated 一段时间
n.attributes["state"] = "removed"
n.confidence *= 0.3
# 切断关系
new.edges = [e for e in new.edges
if e.src_uid != uid and e.dst_uid != uid]
elif ev.event_type == "object_added":
node = SpatialNode(**ev.payload)
node.source = Source.FUSED # 经过 consolidation
node.confidence = 0.7 # 提升
new.add_node(node)
new.add_edge(SpatialEdge(node.parent_room, node.uid,
"contains", source=Source.FUSED))
elif ev.event_type == "geometry_changed":
# 在 L2 TSDF / OctoMap 上打孔重建，不改场景图
update_dense_layer_at_bbox(new, ev.new_bbox)
return new
```
→ 注意 `deepcopy + 整体替换` 保证**原子性**：要么全部成功，要么回滚。
---
## 7.7 Step 4：锚点 / 索引 / CLIP 更新
```python
# consolidation/refresh.py
def refresh_anchors_and_index(new_mem: SpatialMemory):
# 1) 锚点重算
new_mem.anchors = build_anchors(new_mem) # 同 Chapter 04
# 2) 受影响节点的 CLIP 重算
for ev in events_just_applied:
if ev.event_type in ("object_moved","object_added"):
uid = ev.target_uid or ev.payload["uid"]
node = new_mem.nodes[uid]
# 渲染或从近期 keyframe 裁剪
crops = collect_recent_crops(uid)
node.clip_embedding = mean_clip_embedding(crops)
# 3) Faiss 索引重建（L4 全量）
build_faiss_index(new_mem)
# 4) L3 房间 CLIP 受影响时也重算
affected_rooms = {ev.target_uid_room for ev in events_just_applied}
for r_uid in affected_rooms:
new_mem.nodes[r_uid].clip_embedding = recompute_room_clip(r_uid)
```
---
## 7.8 Step 5：增量 3DGS / TSDF 重训
完全重训 3DGS 太贵（一房间 15 k iter × 5 min），所以**只重训受影响房间** + **复用未变区域**：
```python
def incremental_retrain_3dgs(new_mem: SpatialMemory, events):
affected_rooms = set(find_room_for_point(ev.new_pose.position[:2], new_mem)
for ev in events if ev.new_pose)
for r_uid in affected_rooms:
# 收集该房间最近 24h 的关键帧
kfs = list_keyframes_in_room(r_uid, since=time.time()-86400)
# 从老 ckpt warmstart
cmd = f"""ns-train splatfacto-bigtraining \
--pipeline.model.warmstart-ckpt outputs/{r_uid}/last.ckpt \
--max-num-iterations 3000 \
--data {kfs_dir}"""
subprocess.run(cmd, shell=True)
# 替换 LTM 中的 .ply
shutil.move(f"outputs/{r_uid}/final.ply",
f"robot_memory/ltm/dense/3dgs/{r_uid}.ply")
def incremental_update_tsdf(new_mem, events):
"""TSDF 更便宜，直接在受影响 bbox 内重新融合最近关键帧"""
for ev in events:
bbox = ev.new_bbox if ev.new_bbox is not None else \
new_mem.nodes[ev.target_uid].bbox_3d
# 把 LTM TSDF 在该 bbox 内"清零"
new_mem.dense.tsdf.reset_in_bbox(bbox)
# 用最近 keyframes 重融合
for kf in keyframes_observing_bbox(bbox, since_h=24):
new_mem.dense.tsdf.integrate(kf)
```
---
## 7.9 Step 6：版本化 + 健康检查 + 切换
巩固结果不直接覆盖在线版本，先写到一个 staging，校验通过才切换：
```python
# consolidation/commit.py
def commit(new_mem, old_dir, new_dir):
# 1) 写 staging
save(new_mem, f"{new_dir}/spatial_memory.json")
copy_dense_dir(f"{old_dir}/dense", f"{new_dir}/dense")
apply_dense_updates(new_dir, new_mem)
# 2) 校验
errs = validate(new_mem)
sanity = run_sanity_relocalize(new_dir) # 用 10 张历史关键帧重定位
if errs or sanity.success_rate < 0.9:
log.error(f"consolidation rejected: {errs}, success={sanity}")
return False
# 3) 原子切换
timestamp = datetime.now().strftime("%Y%m%d_%H%M")
snapshot = f"robot_memory/snapshots/{timestamp}.tar.zst"
archive(old_dir, snapshot)
os.rename(old_dir, f"{old_dir}.prev")
os.rename(new_dir, old_dir)
return True
```
---
## 7.10 锚点淘汰策略
```python
def prune_stale_anchors(mem, max_age_days=60, min_recent_validations=2):
keep = []
for anc in mem.anchors:
node = mem.nodes[anc.anchor_uid]
age_days = (time.time() - anc.last_validated) / 86400
# 仍是不可移动 + 最近 60 天被巩固确认过 → 保留
if not anc.is_mobile and \
node.attributes.get("state","") != "removed" and \
age_days < max_age_days:
keep.append(anc)
elif node.observation_count >= min_recent_validations:
keep.append(anc)
mem.anchors = keep
```
→ 每次巩固跑一次。
---
## 7.11 完整 CLI：`prism consolidate`
```python
# tools/prism_consolidate.py
import click
@click.command()
@click.option("--mode", default="full",
type=click.Choice(["full","light","deep"]))
@click.option("--ltm", default="robot_memory/ltm")
@click.option("--delta", default="robot_memory/delta/pending.jsonl")
def main(mode, ltm, delta):
mem = load(f"{ltm}/spatial_memory.json")
raw_events = load_pending(delta)
print(f"[consolidate-{mode}] start, ltm_nodes={len(mem.nodes)}, "
f"pending={len(raw_events)}")
# Step 1: filter
decisions = [(ev, classify(ev, mem)) for ev in raw_events]
confirmed = [ev for ev, d in decisions if d == "confirm"]
rejected = [ev for ev, d in decisions if d == "reject"]
kept = [ev for ev, d in decisions if d == "keep"]
# Step 2: arbitrate
final = arbitrate(confirmed, mem)
# Step 3-4: apply + refresh
staging = f"{ltm}.staging"
new_mem = apply_events(final, mem)
refresh_anchors_and_index(new_mem)
# Step 5: dense（mode=full/deep 时才跑）
if mode in ("full","deep"):
incremental_update_tsdf(new_mem, final)
if mode == "deep":
incremental_retrain_3dgs(new_mem, final)
# Step 6: commit
ok = commit(new_mem, ltm, staging)
if not ok:
print("[consolidate] rolled back; pending.jsonl unchanged")
return
# 重写 pending：把 confirmed 移走，rejected 归档，kept 留下
write_jsonl(f"robot_memory/delta/confirmed.jsonl", final, append=True)
write_jsonl(f"robot_memory/delta/rejected.jsonl", rejected, append=True)
write_jsonl(delta, kept) # 覆盖
print(f"[consolidate-{mode}] applied={len(final)} "
f"rejected={len(rejected)} kept={len(kept)}")
```
运行：
```bash
# 充电时自动触发
systemd-timer --on=20:00 --weekly --command "prism consolidate --mode full"
# 月度深度巩固
systemd-timer --on=monthly --command "prism consolidate --mode deep"
```
---
## 7.12 安全机制
| 风险 | 缓解 |
|------|------|
| 巩固后机器人 reloc 失败 | Step 6 的 sanity check 不通过 → 自动回滚 |
| 巩固过程中断电 | staging 写完之前不替换 `ltm/`；半成品 staging 启动时清理 |
| 误删 iPhone 高 conf 节点 | 应用 remove 前再次检查 `source==iphone && conf>0.85`，若是则需 ≥ 10 倍证据 |
| 数据竞争（在线感知与巩固同时写） | 巩固开始前发 `set_readonly(true)`，在线感知此期间只允许写 `delta/`，不能切换 LTM 句柄 |
---
## 7.13 巩固后的指标记录
```python
def log_metrics(events, mem_before, mem_after):
with open("robot_memory/logs/consolidation.log", "a") as f:
f.write(json.dumps({
"ts": datetime.now().isoformat(),
"applied": len(events),
"nodes_before": len(mem_before.nodes),
"nodes_after": len(mem_after.nodes),
"anchors_before": len(mem_before.anchors),
"anchors_after": len(mem_after.anchors),
"ltm_version_new": mem_after.schema_version,
}) + "\n")
```
便于后续画"机器人记忆演化"曲线。
---
## 7.13.1 v1.5 升级：Self-Augmentation 巩固训练
### 背景：train-test discrepancy
§ 7.4–7.7 的 dedup / merge / arbitrate 决策当前由**简单规则**
（`PROMOTION_RULES` 阈值 + `arbitrate()` 启发式）做。v1.5 → v2.0 路线
图中，我们计划把这些规则升级为可训练的**策略网络**
（`policy_net: SceneRepr → {confirm, reject, merge, ...}`）。一旦走到
策略网络，就立刻撞上 Lyra 2.0 § 3.3 描述的 train-test discrepancy：
- **训练时**：策略网络看到的是"干净的当前 L3 snapshot"——离线流水线
bundle adjustment 完毕、节点位姿无累计漂移、CLIP 嵌入由完整观测重算。
- **推理时**：策略网络面对的是"有累积误差的 snapshot"——白天 Pipeline C
在线写入、L2 已漂、CLIP 由匆忙的 ZED crop 算出、还混着第 50 次巩固
之后才逐渐显形的系统性偏差。
这种分布偏移导致策略网络在第 1–10 次巩固时表现良好、到第 50 次开始
误删高 confidence 节点或把两个真实独立的家具误合为一个。Lyra 2.0
§ 3.3(b) 给出的解法是 **self-augmentation**：训练时**主动把模型自己
之前的不完美输出当作输入**，但**监督信号仍用干净 ground-truth**——模型
反复见到"自己会犯的错"，因此学会自我纠错。完整原则陈述与 $p_{\text{aug}}$
取值讨论见 [`18_lyra_inspirations.md` § 18.4](18_lyra_inspirations.md)。
### 算法：`consolidate_with_self_aug()`
```python
# consolidation/self_aug.py (v1.5 新增,配合 policy_net 升级使用)
import numpy as np
def consolidate_with_self_aug(
l3_clean: SceneRepr, # 当前离线巩固出的"干净"L3 快照
history_l3s: List[SceneRepr], # 过去 N 次巩固留下的不完美快照
policy_net, # 待训练的 dedup/merge 决策器
p_aug: float = 0.7,
t_max: float = 0.5,
) -> torch.Tensor:
"""
单步训练:以 p_aug 概率把"历史不完美 L3"当输入,
监督信号始终是从 l3_clean 推得的 dedup/merge 目标。
"""
# ---- 1. 决定本步是否使用 self-aug ----
if np.random.rand() < p_aug and len(history_l3s) > 0:
# 从历史中采样一份"曾经的不完美 snapshot"做基底
t = np.random.uniform(0.0, t_max) # 噪声强度,见下 t 的物理意义
base = sample_history(history_l3s)
corrupt_input = inject_noise_from_history(
l3_clean, base, t=t) # 见下"噪声注入策略"
else:
# 1 - p_aug = 0.3 概率用真实的"干净"输入做兜底
corrupt_input = l3_clean
# ---- 2. 前向 + 统一向"干净目标"对齐 ----
pred = policy_net(corrupt_input) # logits over actions
targets = dedup_targets_from(l3_clean) # 监督一律取 clean
loss = cross_entropy(pred, targets)
return loss # 调用方负责 backward + step;典型 1000 iter / 场景(见参数表)
```
### 噪声注入策略：把历史 L3 当 "corruption source"
`inject_noise_from_history(l3_clean, base, t)` 不是凭空加 Gaussian，而是
**让 `l3_clean` 退化成"看起来像 `base` 那个时点的中间状态"**。具体扰动
按 `t ∈ [0, 0.5]` 线性加权应用以下三类：
| # | 扰动名 | 实现 | 模拟的真实失败模式 |
|---|--------|------|-------------------|
| 1 | **节点位置抖动** | 对每个节点 `pose.position` 加 `N(0, σ_xyz)`，`σ_xyz = t · 0.10 m` | L2 长走廊漂移导致 L3 节点中心偏移 ±10 cm |
| 2 | **CLIP 嵌入扰动** | 对 `clip_embedding` 加 `N(0, σ_clip)` 后重新 L2 归一化，`σ_clip = t · 0.05` | ZED 暗光 / 模糊 crop 导致嵌入轻微偏离 |
| 3 | **节点随机丢弃** | 以概率 `p_drop = t · 0.10`（即 t=0.5 时丢 5%）随机移除非锚点节点 | 在线管线漏检小物品 / 被遮挡未上报 |
实现时三种扰动**独立采样、叠加施加**，并对应 `history_l3s` 中真实出现过
的偏差量级做了线性归一（最大扰动幅度对齐"第 50 次巩固"时的统计观测）。
### 参数表
| 参数 | 默认值 | 物理意义 / 来源 |
|------|--------|----------------|
| `p_aug` | **0.7** | Lyra 2.0 § 3.3(b) 报告的最佳点；> 0.5 让模型主要见自己的错，< 1.0 保留 30% 真实兜底，避免沉迷自生失败模式 |
| `t_max` | **0.5** | $t \in [0, 0.5]$ 表示"最多让历史看起来像**巩固到一半时**的中间状态"；t=1.0 会让样本退化到几乎全噪声，监督信号失效 |
| `σ_xyz` | t · 0.10 m | 与典型 L2 长走廊漂移上限对齐 |
| `σ_clip` | t · 0.05 | 与 ZED 暗光 crop 实测 CLIP 偏移分位数对齐 |
| `p_drop` | t · 0.10 | 与 Pipeline C 漏检率（v1.4 实测 ~5%）对齐 |
| 收敛迭代数 | **~1000 iter / 场景** | Lyra 用 7000 iter（视频扩散数据规模大）；PRISM 单场景规模约小 5–10×，1000 iter 经验上够 |
| 训练批大小 | 1 场景 / step | 场景图本身就是一个 graph batch，无需 mini-batch |
### 评测建议（回链 [`13_evaluation.md`](13_evaluation.md)）
在 [`13_evaluation.md`](13_evaluation.md) 的"**长时一致性**"指标族
（典型项：第 N 次巩固后的节点误删率、误合率、ghost-node 残留率）下，
**工程预估**：使用 self-aug 训练的策略网络在第 50 次巩固之后，
- **误删高 confidence 节点率**：下降 **30–50%**（baseline 无 self-aug 训练时通常 ~8% → 预期 4–5.5%）
- **误合相邻独立家具率**：下降 **30–40%**
- 第 1–10 次巩固指标基本持平（self-aug 主要解决长尾分布偏移，不解决初期能力问题）
ablation 计划：固定其他条件，对 `p_aug ∈ {0.3, 0.5, 0.7, 0.9}` 与
`t_max ∈ {0.25, 0.5, 0.75}` 跑 2D 扫描，验证 (0.7, 0.5) 是否对 PRISM
真实数据也最优——如果最优点漂到 (0.5, 0.5)，说明 Pipeline C 已足够稳定，
self-aug 信号可以降权。
### 与原巩固管线的关系
本小节**不替换** § 7.4–7.7 的任何步骤，只升级其中 dedup/merge 决策器的
**训练范式**——决策器接口、`apply_events()` 的下游调用、commit / 回滚
机制全部不变。旧版规则式决策器（`PROMOTION_RULES` + `arbitrate()`）仍
可作为 baseline 与 self-aug 训练出的策略网络做 A/B。回链
[`18_lyra_inspirations.md` § 18.4](18_lyra_inspirations.md)
"原则三：巩固期 Self-Augmentation"。
---
## 7.14 与 Chapter 06 的接口
| 项 | Chapter 06 | Chapter 07 |
|----|-----------|------------|
| 写 LTM | ❌ 严禁 | ✅ 唯一允许的写者 |
| 写 delta | ✅ append | 读取 + 重写 confirmed/rejected/kept |
| 读 LTM | 只读 | 只读 + 写 staging |
| 频率 | 实时 | 离线（充电时） |
两者通过 **文件系统** 解耦：在线感知不必知道巩固何时跑，巩固也不必停下感知。
---
## 7.15 本章小结
| 关键点 | 一句话 |
|--------|--------|
| **何时跑** | 充电 / 空闲；典型每天 1 次 |
| **筛选规则** | observation_count + 时间跨度 + 视点多样性 |
| **冲突仲裁** | 同 target 多事件合并；iPhone 高 conf 项有更高门槛 |
| **应用** | deepcopy + 整体替换，保证原子性 |
| **重训** | 仅受影响房间 + warmstart |
| **安全** | staging + sanity check + 回滚机制 |
| **比喻** | 像人脑睡眠——白天乱记，夜里整理 |
读完本章你应能：
- ✅ 编写一个能跑通的 `prism consolidate` 脚本
- ✅ 解释为什么要"延迟修改 LTM"
- ✅ 设计安全的回滚机制
下一章 [`08_runtime_timeline.md`](08_runtime_timeline.md) 把 Chapter 04–07 的所有模块串成一个**完整的端到端时序剧本**，从 T0 到 T4。
---
**章节版本**：v1.0
**估计阅读时间**：18 分钟
**关键收获**：把"短期 delta"变成"长期 LTM"的完整安全流程
plans/PRISM/08_runtime_timeline.md
+483
View File
@@ -0,0 +1,483 @@
# Chapter 08 — 端到端运行时序
> 本章目标：把 Chapter 04–07 的四个管线**串成一条完整的时间线**，用一个具体酒店场景的"剧本"展示 PRISM 如何在 T0 到 T4 的全生命周期工作。
---
## 8.1 场景设定
> 某商务酒店 3 楼，共 8 间客房 + 走廊 + 电梯厅，约 300 m²。
> 部署一台轮式服务机器人「Robo-Concierge」，搭载 ZED 2i，需要：
> - 自主巡逻
> - 接受客人语音指令"去 308 取一下遥控器"
> - 早晨给打扫清单上的房间送备品
> - 检测家具异常移动（防盗 / 维护提醒）
---
## 8.2 时间线全景
```mermaid
timeline
title PRISM 端到端运行时序 T0 -- T6
section 部署前
T0 D-7 : iPhone 离线扫描 → 灌入 LTM
section 开机当天
T1 9点 : 机器人首次开机<br/>重定位握手
T2 全天 : 巡逻/任务<br/>delta 累积
T3 14点30 : 取遥控器任务
T4 22点 : 充电桩<br/>记忆巩固
section 长期维护
T5 D+30 : 月度深度巩固
T6 D+90 : 装修后部分重扫
```
---
## 8.3 T0：iPhone 离线扫描（D-7，运维人员）
### 物理动作
```
09:00 运维进入 3 楼电梯厅
09:01 放置 ArUco 标识在地面（id=42, 30 cm×30 cm）
09:03 打开 PRISMScanner App
09:05 ── 单 session 开始 ──
1) 在电梯厅扫一圈，看到 ArUco ✓
2) 沿走廊缓慢推进
3) 进 301，扫描 12 分钟
4) 退出 301，沿走廊
5) 进 302...... 重复直到 304
09:55 暂停 session（4 个房间）
10:00 换电池 + 第二 session（305–308）
10:50 全部完成
```
### 数据落地（D-6，工作站）
```
data/scans/
├── 2026-05-09_3F_session1/ # 301-304
│ ├── Hotel.usdz
│ ├── roomplan.json
│ └── arkit/ (rgb + depth + pose)
└── 2026-05-09_3F_session2/ # 305-308
└── ...
```
### Ingest（D-5，工程师）
```bash
# Session 1
python -m tools.prism_ingest_iphone \
--scan data/scans/2026-05-09_3F_session1/ \
--out robot_memory/ltm/ \
--aruco-id 42
# Session 2（追加进同一 LTM）
python -m tools.prism_ingest_iphone \
--scan data/scans/2026-05-09_3F_session2/ \
--out robot_memory/ltm/ \
--aruco-id 42 \
--merge
# 检查
python -m tools.check_anchors # 期望 ≥ 16 个 anchor（每房 2 个）
python -m tools.viz_octomap # 人眼检查走廊可通行
```
输出：
```yaml
nodes: 289 (8 房 + 8 走廊 + ~273 家具)
edges: 312
anchors: 21
LTM size: 3.4 GB
```
### 部署
```
D-3: 把 robot_memory/ 拷贝到机器人 SSD
D-2: 机器人在测试间预演重定位（不到现场）
D-1: 机器人运到 3 楼电梯厅，关机待命
```
---
## 8.4 T1：机器人首次开机（D 早上 9:00）
### 时序（精确到毫秒）
```mermaid
sequenceDiagram
autonumber
participant U as 用户 电源
participant R as ROS 2 PRISM
participant Z as ZED 2i SDK
participant FP as Frame Producer
participant RL as Relocalizer
participant NAV as Avoidance Nav
participant FSM as 状态机
U->>R: 09:00:00.000 Power on
R-->>R: 09:00:02.500 节点 launch 完成
R->>Z: 09:00:02.600 启动 ZED 2i SDK
Z->>FP: 09:00:03.100 第一帧到达
FP->>RL: 09:00:03.150 触发 cold_start
RL-->>RL: 09:00:03.230 CLIP 编码当前帧
RL-->>RL: 09:00:03.310 L3 Top-3 lobby_3F 0.82 room_308 0.41 hallway_3F 0.39
RL-->>RL: 09:00:03.320 几何配准 FPFH+RANSAC 0.74 ICP refine 0.91 通过
RL->>R: 09:00:03.510 发布 TF map to zed2i_init
R->>NAV: 09:00:03.550 Avoidance Nav 上线
R->>FSM: 09:00:03.600 状态机 to IDLE
FSM-->>U: 09:00:04.000 开始默认任务 巡逻
Note over U,FSM: 总冷启动约 3.6 秒 其中 PRISM 握手约 400 ms
```
**总冷启动时间：~ 3.6 秒**（其中 PRISM 的握手 ~ 400 ms）
### 内部状态变化
| 时刻 | L1 | L2 局部 | L3 当前位置 | L4 缓存 | TF |
|------|----|---------|-------------|---------|----|
| 09:00:00 | 空 | 空 | 未知 | 全部从盘加载 | base_link→? |
| 09:00:03.1 | 1 帧 | 空 | 未知 | 已加载 | 仍未对齐 |
| 09:00:03.5 | 多帧 | 开始融合 | lobby_3F | — | **map→zed2i_init 发布✓** |
---
## 8.5 T2：日间巡逻（持续 ~ 8 小时）
### 高频循环（30 Hz 主循环）
```
每帧 33.3 ms:
ZED grab → FrameProducer.append → 局部避障 → 30Hz 控制循环
```
### 中频循环（5 Hz TSDF）
```
每 200 ms:
取一个关键帧 → TSDFWorker.integrate
- 检查 prior_mask: 卫生间镜面区被跳过
- 检查 prior_weight: 走廊墙面 weight=0.1（轻微校准）
- 走廊地面新发现的"水迹"区域：正常融合，weight=1.0
```
### 低频循环（2 Hz 检测）
```
每 500 ms:
YOLO-World on 当前 RGB
→ 检出: bed_room_303, lamp_room_303, person, slipper
Person → 隐私过滤，丢弃这帧的语义写入（只保留几何）
bed_303 → 命中 LTM，更新 last_seen + EMA pose（微调）
lamp_303 → 命中 LTM，OK
slipper → 未在 LTM，propose_new → DeltaEvent(object_added,
payload=...)，写 delta/pending.jsonl
```
### 异常事件（11:34）
```
机器人巡逻到 305：
- TSDF 在沙发区域显示"空"
- LTM mesh 显示"有沙发"
- ChangeDetector 1 Hz 跑：
diff > 5 cm → 聚类 → DeltaEvent(
event_type="object_removed",
target_uid="sofa_305",
evidence=[kf_305_1134_01, kf_305_1134_07])
- 写入 delta/pending.jsonl
但机器人 *不立刻*认为沙发没了：
- LTM 中 sofa_305 仍在
- 后续路径规划仍把它当障碍（保守）
- 等下次累计观测足够 + 充电时巩固才真删
```
### Anchor 校正（11:42）
```
机器人路过 304 床前：
- YOLO 检到 'bed' (high conf)
- 与 LTM bed_304 关联成功
- 在线触发 fine_register_anchor:
fitness=0.88 → 更新 T_zed→map
drift 估计归零
- 自此重新校准过；与上次校准间隔 38 分钟（< 1 h 健康）
```
### 任务级中断（13:00）
```
语音："去 306 把客人遗忘的伞拿到前台"
LLM Agent 查 PRISM API:
find("伞", room="room_306")
→ L4 检索: umbrella_xxx (source=zed2i, conf=0.5,
last_seen=今早 10:21, parent=nightstand_306)
→ L3 路径: cur(hallway) → 306
→ L2 度量路径
→ 执行
```
---
## 8.6 T3：完整任务剧本——"取遥控器"
### 13:00:00 用户输入
```
客人在 308 房间说："请帮我去 308 桌上拿遥控器"
（机器人当前位于 hallway 走廊中段）
```
### 13:00:01 LLM Agent 查询
```python
# Agent 不直接访问数据库；用 PRISM API
api = SpatialMemoryAPI("robot_memory/ltm/")
# 1) find 物品
result = api.find("remote control", filters={"parent_room": "room_308"})
# → [SpatialNode(uid='remote_xx33', label='remote_control',
# parent_uid='tv_stand_308', confidence=0.6,
# last_seen=2026-05-16T08:14:22)]
# 2) describe（给 LLM 看的自然语言）
print(api.describe(result[0].uid))
# "A black remote_control on tv_stand_308 in room_308 (Bedroom),
# last seen 4h 45min ago, confidence 0.6."
# 3) 检查最近 delta（是否被搬过）
recent = api.changes_since(time.time() - 3*3600)
# 关于这个遥控器的 delta? → 没有
# OK，假定还在
# 4) 路径规划
path_rooms = api.path_rooms(src="hallway_3F", dst="room_308")
# → ["hallway_3F", "room_308"]
metric_path = api.l2.plan(start=cur_pose, goal=result[0].pose)
```
### 13:00:02 执行
```
13:00:02.300 开始驱动，沿走廊向东
13:00:14.200 到达 308 门前
13:00:14.300 ChangeDetector 注意到 308 的门 LTM 标 closed 但实际开着
→ 写 delta(object_state_change, door_308)
13:00:14.500 进入 308
13:00:16.800 到 tv_stand_308 前 0.5 m
13:00:17.000 ZED 视野中检出 remote_xx33 ✓（YOLO conf=0.78）
位置与 LTM 偏 12 cm（在桌面被人挪过 12 cm）
EMA 更新 pose，不写 delta（< 20 cm 阈值）
13:00:18.000 机械臂伸出抓取
13:00:20.000 抓握成功，撤回
13:00:25.000 返回走廊
13:00:40.000 抵达前台，交接
```
**任务耗时 38 秒；查询 PRISM 总耗时 < 50 ms。**
---
## 8.7 T4：充电时记忆巩固（D 22:00）
### 触发
```
21:55:00 机器人电量 < 25%
21:55:05 返回充电桩（导航 ~ 8 min）
22:03:00 上桩，开始充电
22:13:00 静止 > 10 min，触发 prism consolidate --mode=full
```
### 巩固内容（当天积累）
```
pending.jsonl 当天总计 1234 条事件：
├── object_added (slippers, towels, ...) 413
├── object_moved (人挪过的小物) 382
├── object_removed 91
├── geometry_changed (走廊积水/物件堆放) 38
└── 重复信号 310
```
### Step 1: 筛选
```
观察累积 > 5 + 时间跨度 > 5 min + 多视点 > 2:
→ 162 条通过 confirm
→ 798 条 reject（< 5 min 的短暂物）
→ 274 条 keep（继续等待第二天）
```
### Step 2-3: 应用
```
其中：
- sofa_305 的 object_removed: 累积 47 次观测，跨越 6 小时
→ 通过 ✓，标记 sofa_305 state=removed，confidence ×= 0.3
- 12 个新增的 small_item（拖鞋、毛巾、水瓶）→ 加入 L4
- 41 个家具位置微调
```
### Step 4: 锚点 / 索引刷新
```
21 → 20 个 anchor（sofa_305 被淘汰）
受影响的 12 个新节点：渲染图像 + CLIP → 入 Faiss 索引
room_305 的房间 CLIP 重算（因为 sofa 不再）
```
### Step 5: 增量重训
```
受影响房间：room_305
3DGS warmstart from outputs/room_305/last.ckpt
3000 iter (~ 4 min on Orin AGX)
→ ply 替换 ltm/dense/3dgs/room_305.ply
TSDF 在 sofa_305 bbox 内 reset → 用今日 keyframes 重融合
```
### Step 6: 验证 + 切换
```
sanity_relocalize on 10 历史关键帧:
10/10 成功，RMSE 4.2 cm ✓
validate(new_mem):
0 errors ✓
snapshot: robot_memory/snapshots/20260516_2230.tar.zst (3.4 GB)
原子切换：ltm.staging → ltm；旧 ltm → ltm.prev（next day 删除）
22:34:00 consolidate 完成，总用时 31 min
22:34:01 机器人继续充电
```
### 第二天早晨
```
07:00:00 机器人启动，加载新 LTM
07:00:03 重定位（在充电桩，看天花板灯）→ Top-1 = hallway_3F ✓
07:00:04 IDLE 等待任务
```
---
## 8.8 T5：月度深度巩固（D+30 凌晨）
```
01:00:00 定时任务触发 prism consolidate --mode=deep
01:00:30 完整巩固 + 全场景 3DGS 重训（每房间 5000 iter）
03:45:00 全部完成 (8 房 × ~ 20 min)
03:46:00 自检：sanity_relocalize 20 张历史帧 → 19/20 通过 ✓
```
---
## 8.9 T6：装修后部分重扫（D+90）
```
酒店在 D+85 改造了 307 房间（新家具）
D+90: 运维带 iPhone 重扫 307 + 走廊端
ingest 时指定 --only-rooms room_307:
- 仅替换 LTM 中 room_307 相关的 nodes/edges
- 其他房间保持不变
- 重新生成 room_307 的 anchor + CLIP
- 重训 room_307 的 3DGS
ltm_version 从 v15 → v16
```
---
## 8.10 全生命周期 KPI（连续运行 90 天后回看）
```yaml
重定位:
冷启动成功率: 98.4% (1230/1250 次)
平均冷启动时长: 670 ms
绑架恢复成功率: 93% (用 ArUco 兜底后 100%)
记忆维护:
pending 平均长度: ~ 800 条/日
每日 confirmed: ~ 150 条
reject 率: ~ 64%（健康，说明门槛起作用）
错误巩固撤销次数: 2 次（手工回滚）
任务表现:
"取物"任务成功率: 94% (n=412)
错误归因:
- 物品已被移走 (delta 未及时巩固): 4%
- 抓取失败: 2%
- 重定位失败: < 1%
资源占用 (Jetson Orin AGX):
CPU 平均: 37%
GPU 平均: 52%
内存: 12.3 GB / 64 GB
磁盘增长: + 0.2 GB / 月（snapshots）
```
---
## 8.11 完整状态图
```mermaid
stateDiagram-v2
[*] --> BOOTING: power
BOOTING --> IDLE: relocalize ok
IDLE --> EXECUTING_TASK: task assigned
EXECUTING_TASK --> IDLE: done
IDLE --> RELOCALIZING: relocalize lost
EXECUTING_TASK --> RELOCALIZING: relocalize lost
RELOCALIZING --> IDLE: recovered
RELOCALIZING --> FALLBACK_HUMAN: failed > N
FALLBACK_HUMAN --> IDLE: human help
IDLE --> CONSOLIDATING: charging + idle > 10min
CONSOLIDATING --> IDLE: done
note right of IDLE
巡逻或待命
end note
note right of CONSOLIDATING
Background (并行):
• FrameProducer (30 Hz, always)
• TSDFWorker (5 Hz when moving)
• DetectorWorker (2 Hz when moving)
• ChangeDetector (1 Hz when stationary)
• HealthMonitor (0.2 Hz)
end note
```
---
## 8.12 本章小结
| 阶段 | 时间 | 主体 | PRISM 角色 |
|------|------|------|-----------|
| **T0** D-7 | 人工 | iPhone 扫描 | LTM 灌入 |
| **T1** D 9:00 | 机器人 | 冷启动 | 重定位握手 |
| **T2** D 全天 | 机器人 | 巡逻 | 在线感知 + delta 累积 |
| **T3** D 13:00 | 机器人 | 任务 | 查询四层记忆 + 路径 |
| **T4** D 22:00 | 机器人 | 充电 | 巩固 |
| **T5** D+30 | 机器人 | 深夜 | 深度巩固 |
| **T6** D+90 | 人工 | 重扫部分 | 增量 LTM 更新 |
读完本章你应能：
- ✅ 在脑中"播放"完整的 PRISM 一天
- ✅ 解释为什么 sofa_305 不会被"瞬间删除"
- ✅ 估算系统的资源占用
- ✅ 设计 PRISM 的状态机
下一章 [`09_roadmap.md`](09_roadmap.md) 给出 8 周内把这一切跑通的具体里程碑。
---
**章节版本**：v1.0
**估计阅读时间**：18 分钟
**关键收获**：把抽象架构变成"看得见的一天"
plans/PRISM/09_roadmap.md
+327
View File
@@ -0,0 +1,327 @@
# Chapter 09 — 实施路线图（8 周原型 + 4 个月生产）
> 本章目标：把 PRISM 从"文档"变成"能跑的代码"。给出**8 周原型计划**（达到 demo 可演示）+ **4 个月生产计划**（达到上线可交付）。
---
## 9.1 总览
```mermaid
gantt
title PRISM 实施路线图（8 周原型 + 4 个月生产）
dateFormat X
axisFormat W%s
section 8 周原型
Phase 0：准备 硬件 + 环境 :p0, 0, 1
Phase 1：单边管线 iPhone & ZED :p1, 1, 3
Phase 2：核心握手 重定位 MVP :p2, 3, 5
Phase 3：在线感知 + delta :p3, 5, 7
Phase 4：巩固机制 :p4, 7, 8
Phase 5：端到端 demo :milestone, m1, 8, 8
section 4 个月生产
Phase 6：鲁棒性 / 多房间 / 评测 :p6, 8, 16
Phase 7：集成 LLM Agent / 任务系统 :p7, 16, 20
Phase 8：生产部署 / 文档 / 开源 :p8, 20, 24
```
---
## 9.2 Phase 0 — 准备（W0 = 部署前）
### 硬件清单
| 类别 | 物品 | 数量 | 价格 | 用途 |
|------|------|------|------|------|
| 扫描 | iPhone 15 Pro Max | 1 | ¥10 k | RoomPlan |
| 扫描 | DJI OM 6 稳定器 | 1 | ¥0.9 k | 减抖 |
| 扫描 | ArUco 标识板（铝） | 3 | ¥0.2 k | 公共原点 |
| 机器人 | ZED 2i | 1 | ¥4 k | 在线感知 |
| 机器人 | Jetson Orin AGX 64 GB | 1 | ¥18 k | 计算 |
| 机器人 | 移动底盘（差速 / 麦轮） | 1 | ¥10–30 k | 载体 |
| 机器人 | 2D LiDAR (RPLIDAR S2) | 1 | ¥2 k | 安全冗余 |
| 工作站 | RTX 4090 + 64 GB | 1 | ¥25 k | 训练 / 烘焙 |
| **合计** | | | **~ ¥73 k** | |
### 软件环境
```bash
# 工作站
Ubuntu 22.04
CUDA 12.2 + cuDNN 8.9
ROS 2 Humble
Python 3.10 + conda env "prism"
- open3d>=0.18
- pyzed>=4.1
- open_clip
- ultralytics (YOLO-World)
- nerfstudio
- octomap-python
- networkx + neo4j (optional)
# 机器人 (Jetson Orin)
JetPack 6.0
ROS 2 Humble
同上 Python 包（Jetson 编译版）
ZED SDK 4.1
```
### 团队配置（最小）
| 角色 | 人数 | 重点 |
|------|------|------|
| Robotics 工程师 | 1 | ZED + ROS 2 + 控制 |
| ML 工程师 | 1 | CLIP / YOLO / 3DGS |
| 系统工程师 | 1 | PRISM 主框架 + Schema |
| iOS 开发（兼） | 0.5 | PRISMScanner App |
---
## 9.3 Phase 1 — 单边管线（W1–W2）
### W1：iPhone 端管线打通
**目标**：跑通 [`04_pipeline_A_iphone_offline.md`](04_pipeline_A_iphone_offline.md) 全流程，得到一个房间的 LTM。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | 写 `PRISMScanner` Swift App（最小版：调 RoomPlan + 存 USDZ + JSON） | iPhone 真机可扫房间 |
| Tue | 实现 `parser_iphone.py` + `align_to_map.py` | 一个房间 → JSON，节点 ≥ 20 |
| Wed | 实现 `extract_furniture_mesh.py` + `bake_octomap.py` | 每件家具一份 `.glb`；octomap.bt 可视化通过 |
| Thu | 实现 `bake_tsdf.py` + CLIP 计算 | tsdf.vbg + 节点 embedding |
| Fri | 实现 `build_anchors.py` + `prism_ingest_iphone` CLI；写单元测试 | `prism ingest` 一行命令完成 |
**周末验收**：扫一个真实卧室 → 跑 ingest → `prism viz ltm` 在 RViz 看到房间 + 床 + 桌（带标签）。
### W2：ZED 2i 端管线打通
**目标**：跑通 ZED 实时数据采集 + VIO + 关键帧保存，与 PRISM Schema 对接。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | ZED SDK 安装 + ROS 2 wrapper；输出 RGB+Depth+Pose | rqt 看到三路 topic |
| Tue | 实现 `FrameProducer`（环形缓冲 + 关键帧）+ `PerceptualFrame` 序列化 | 关键帧落盘 stm/keyframes/ |
| Wed | 实现 TSDF Worker（局部融合，先不接 prior_mask） | RViz 看到实时 mesh |
| Thu | 实现 `DetectorWorker`（YOLO-World，开放词表） | `prism viz detection` 在 RGB 上画框 |
| Fri | TF 树整理 + 时间戳同步检查 | 所有节点 lag < 100 ms |
**周末验收**：手推机器人在一个房间转一圈，得到 stm/ 目录有 100+ 关键帧，TSDF 完整。
---
## 9.4 Phase 2 — 核心握手：重定位 MVP（W3–W4）
### W3：粗匹配 + 精配准
**目标**：在 1 个房间内实现冷启动重定位成功率 > 90%。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | 实现 `CoarseRelocalizer`（CLIP Top-K） | 单次匹配 < 100 ms |
| Tue | 实现 `fine_register`（FPFH+RANSAC → ICP） | 单次配准 < 500 ms |
| Wed | 串成 `relocalize()` + 在 10 个不同位置测试 | 成功率 ≥ 8/10 |
| Thu | 加多帧投票 + DINOv2 重排 | 成功率 ≥ 9/10 |
| Fri | 发布 TF `map → zed2i_init`；写 ROS 2 节点 | 完整 ROS 流水线运行 |
**周末验收**：把 ZED 在房间内随机放 10 个位置，每次 < 1 s 给出位姿，平移误差 < 15 cm。
### W4：跨房间 + 异常处理
**目标**：扩到多房间，加入容错。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | 扩到 3 个房间 LTM；CLIP 检索 Top-3 房间 → 逐个 ICP | 跨房成功率 ≥ 85% |
| Tue | 实现 anchor 级超精配（5.6）作为兜底 | fitness 提升 0.1+ |
| Wed | 实现 `OnlineRelocalizer`（在线 anchor 校正） | 巡逻 30 min 漂移 < 30 cm |
| Thu | 实现 ArUco 兜底 + 绑架恢复 | 绑架到另一房间 30 s 内恢复 |
| Fri | 写自动化重定位评测脚本 | `prism eval reloc` 输出报表 |
**周末验收**：[`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) §5.10 评测协议全部通过。
---
## 9.5 Phase 3 — 在线感知 + delta（W5–W6）
### W5：完整在线感知
**目标**：实现 Chapter 06 的所有 worker，跑稳。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | TSDF Worker 接入 `prior_mask` / `no_update_zone` | 镜面区不被污染 |
| Tue | Detector Worker 接入 `associate_hybrid` + EMA 更新 | 已知家具不漂 |
| Wed | 实现 `ChangeDetector`（TSDF diff + DBSCAN） | 手动搬走一把椅子，10 s 内出 delta |
| Thu | 实现 `DeltaLog`（去重 + JSONL） | 重复事件不重写 |
| Fri | 整体压测：跑 2 小时 | 无内存泄漏，30 Hz 稳定 |
### W6：差异检测与告警
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | 实现"新增物体"提案流程；与现有节点 disambiguate | 新拖鞋出现写 delta，不重复 |
| Tue | 隐私过滤（person → 不写语义） | 人入镜不写入 LTM |
| Wed | dashboards: `prism viz live` + `prism viz delta` | 两个可视化能跑 |
| Thu | 写 ROS 2 launch 文件，整套节点一键启动 | `ros2 launch prism online.launch.yaml` |
| Fri | Bug fix + 性能 profile | CPU < 50%，GPU < 60% |
**周末验收**：连续运行 4 小时，delta 累积 ≥ 500 条，无崩溃。
---
## 9.6 Phase 4 — 巩固机制（W7）
**目标**：跑通 Chapter 07 的 `prism consolidate` 完整流程。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | 实现 `classify` + 晋升规则；单元测试 | 1000 模拟 delta，准确分类率 100% |
| Tue | 实现 `arbitrate`（冲突合并） | remove+add 自动合并为 moved |
| Wed | 实现 `apply_events`（deepcopy + 整体替换） | 应用后 validate() 0 errors |
| Thu | 实现 `incremental_update_tsdf` + warmstart 3DGS | 单房间增量 < 5 min |
| Fri | staging + sanity_relocalize + commit；回滚测试 | 注入故意错误能成功回滚 |
**周末验收**：跑一次完整 `prism consolidate --mode=full`，前后对比 LTM 健康度提升。
---
## 9.7 Phase 5 — 端到端 demo（W8）
**目标**：完整演示一个 "T0–T4 一天" 剧本（参考 [`08_runtime_timeline.md`](08_runtime_timeline.md)）。
| Day | 任务 | 验收 |
|-----|------|------|
| Mon | 选定真实场景（实验室 + 2 房间），完成 iPhone 扫描 | LTM 节点 ≥ 50，anchors ≥ 6 |
| Tue | 部署到机器人 + 完整启动；冷启动 demo | 视频 1：开机 → 重定位 |
| Wed | 巡逻 + delta 累积 demo | 视频 2：搬走椅子，10 s 内提示 |
| Thu | LLM Agent 集成（GPT-4o 调用 PRISM API）；任务 demo | 视频 3："取遥控器" 任务成功 |
| Fri | 充电触发 consolidate + 第二天重启 demo | 视频 4：巩固前后对比 |
**周末交付**：
- 📹 4 段演示视频（每段 < 2 min）
- 📊 性能报告（重定位 / 在线感知 / 巩固指标）
- 📚 README + Quickstart 文档
- 🎤 15 分钟 demo PPT
**此时 PRISM v0.1 原型可对外展示**。
---
## 9.8 Phase 6 — 鲁棒性与多房间（M3–M4，2 个月）
| 周次 | 主题 |
|------|------|
| M3-W1 | 扩到 8 房间 + 走廊（真实酒店楼层） |
| M3-W2 | 重定位在跨多 session 拼接 LTM 上的稳定性 |
| M3-W3 | 镜面 / 玻璃 / 弱纹理的特殊处理（接 [`hotel_scene_implementation_plan_part2.md`](../hotel_scene_implementation_plan_part2.md)） |
| M3-W4 | 长时（72 h 不间断）压测 |
| M4-W1 | 自动化评测套件（13 章指标） |
| M4-W2 | 数据集采集（HotelScene-Bench v0.5） |
| M4-W3 | 性能优化（FP16 / TensorRT / nvblox） |
| M4-W4 | 第一轮内部用户测试 + 修 bug |
---
## 9.9 Phase 7 — LLM Agent 集成（M5）
| 周 | 任务 |
|----|------|
| W1 | 写 `SpatialMemoryAPI`（高层接口）+ Function-Calling schema |
| W2 | GPT-4o / Qwen-VL 接入；任务规划器 |
| W3 | 6 类典型任务实现：取物、引路、检查、补货、问询、迎宾 |
| W4 | 多轮对话 + 上下文管理 + 错误恢复 |
---
## 9.10 Phase 8 — 生产 / 开源（M6）
| 周 | 任务 |
|----|------|
| W1 | 安装脚本 / Docker 镜像 / 一键部署 |
| W2 | 完整文档（API ref + 教程 + FAQ） |
| W3 | 开源准备（GitHub 仓库 + README + LICENSE） |
| W4 | 发布 v1.0 + 论文撰写启动 |
---
## 9.11 关键里程碑（Go/No-Go gates）
| 时刻 | Gate | 通过标准 | 不通过怎么办 |
|------|------|----------|--------------|
| **W2 末** | 单边管线 | 两条管线各自 demo 成功 | 延期 1 周；不能进 W3 |
| **W4 末** | 重定位 | 多房间冷启动 ≥ 85% | 不能进 W5；换 DINOv2 + anchor 兜底 |
| **W6 末** | 在线感知稳定 | 4 h 无崩溃 + 资源占用合理 | 优化或简化 worker |
| **W8 末** | 端到端 demo | 4 段视频 + 报告 | 缩小 demo 范围 |
| **M4 末** | 鲁棒性 | 72 h 压测通过 | 暴露真实问题再迭代 |
| **M6 末** | 生产就绪 | 文档 + Docker + 开源 | 推迟 release |
---
## 9.12 风险驱动的调整规则
每周五开 30 min sync，按下表决定下周计划：
| 信号 | 调整 |
|------|------|
| 重定位 fitness 持续 < 0.7 | 暂停新功能，先解决（可能要补 DINOv2 / SuperPoint） |
| Detector 误检率高 | 增加训练时间或换 Grounding-DINO |
| delta/pending 增长太快 | 提前实现 consolidate 的轻量版 |
| iPhone 扫描人工成本太高 | 简化采集 SOP，或试用 Polycam 替代 RoomPlan |
| Jetson 资源不足 | 砍 3DGS（用 Mesh 代替）或换 Orin AGX 64 GB |
---
## 9.13 团队任务分配建议
| Sprint | Robotics | ML | System | iOS |
|--------|----------|----|----|-----|
| W1 | (env setup) | (env setup) | parser_iphone | PRISMScanner |
| W2 | ZED ROS2 + FrameProducer | YOLO-World | TSDF Worker | — |
| W3 | TF / 控制 | CLIP coarse | fine_register / ICP | — |
| W4 | OnlineReloc / Anchor | DINOv2 rerank | ChangeDetector | (ArUco 兜底) |
| W5 | Avoidance 接入 | EMA 更新 | DeltaLog | — |
| W6 | Launch + 压测 | dashboards | — | — |
| W7 | 集成 | 3DGS warmstart | consolidate full | — |
| W8 | E2E demo | demo 物料 | docs / quickstart | — |
---
## 9.14 预算（8 周原型阶段）
| 类别 | 金额 |
|------|------|
| 硬件（一次性） | ~ ¥73 k |
| 软件订阅（Roboflow / Polycam Pro / OpenAI API） | ¥3 k |
| 云 GPU 备用（恒源云） | ¥2 k |
| 场地租用（一个酒店楼层 1 周） | ¥10 k |
| 人力（3.5 FTE × 2 月） | ¥150 k |
| **合计** | **~ ¥240 k** |
→ 即 8 周原型阶段约需 ¥24 万。生产阶段（M3-M6）再追加约 ¥80 万。
---
## 9.15 本章小结
| 关键里程碑 | 时刻 | 一句话 |
|-----------|------|--------|
| **单边可跑** | W2 末 | iPhone & ZED 分别打通 |
| **握手成功** | W4 末 | 任意位置 1 s 内重定位 |
| **在线稳定** | W6 末 | 4 h 压测无崩溃 |
| **巩固闭环** | W7 末 | 充电 → 巩固 → 重启验证 |
| **端到端 demo** | W8 末 | 4 段视频 + 报告 |
| **生产就绪** | M6 | 开源 + 文档 + Docker |
读完本章你应能：
- ✅ 制定一份 PRISM 的 Gantt 图
- ✅ 在每个 gate 决定 Go/No-Go
- ✅ 估算预算与团队配置
下一章 [`10_tech_stack.md`](10_tech_stack.md) 细化具体技术栈选型与版本依赖。
---
**章节版本**：v1.0
**估计阅读时间**：12 分钟
**关键收获**：从今天起到 8 周后能演示，每天该做什么
plans/PRISM/10_tech_stack.md
+384
View File
@@ -0,0 +1,384 @@
# Chapter 10 — 技术栈选型
> 本章目标：把前面章节涉及的所有"软件 / 算法 / 模型"按层级列出，给出**版本号、依赖关系、为什么选它**，让团队拿到这章就能搭出可复现的环境。
---
## 10.1 整体层级图
```mermaid
flowchart TB
L7["<b>Layer 7 — Agent / Application</b><br/>GPT-4o · Qwen2.5-VL · LangGraph · ROS 2 Action Servers"]
L6["<b>Layer 6 — PRISM Core（本项目自研）</b><br/>spatial_memory.* · relocalizer · workers · consolidator"]
L5["<b>Layer 5 — Perception Models</b><br/>CLIP · DINOv2 · YOLO-World · SAM 2 · Grounding-DINO"]
L4["<b>Layer 4 — Geometry & Mapping</b><br/>Open3D · TEASER++ · gsplat · nerfstudio · OctoMap · nvblox"]
L3["<b>Layer 3 — SLAM / VIO / Sensor Drivers</b><br/>ZED SDK · RoomPlan · ARKit · ORB-SLAM3(备) · rtabmap(备)"]
L2["<b>Layer 2 — Middleware</b><br/>ROS 2 Humble · tf2 · message_filters · rclpy / rclcpp"]
L1["<b>Layer 1 — Storage / Database</b><br/>JSON · HDF5 · Faiss · Neo4j · SQLite (catalog)"]
L0["<b>Layer 0 — System</b><br/>Ubuntu 22.04 · CUDA 12.2 · cuDNN 8.9 · Python 3.10"]
L7 --> L6 --> L5 --> L4 --> L3 --> L2 --> L1 --> L0
style L6 fill:#fff7d6,stroke:#c97a00,stroke-width:2px
style L7 fill:#e3f2fd,stroke:#1565c0
style L0 fill:#eeeeee,stroke:#555
```
---
## 10.2 完整版本矩阵
| 层 | 组件 | 版本 | License | 用途 |
|----|------|------|---------|------|
| L0 | Ubuntu | 22.04 LTS | open | 主操作系统 |
| L0 | CUDA | 12.2 | NVIDIA EULA | GPU runtime |
| L0 | cuDNN | 8.9 | NVIDIA | DL backend |
| L0 | TensorRT | 10.0 | NVIDIA | Jetson 推理优化 |
| L0 | Python | 3.10 | PSF | 主语言 |
| L0 | PyTorch | 2.3 + CUDA12 | BSD | 训练 / 推理 |
| L1 | Faiss | 1.8 (cpu/gpu) | MIT | CLIP 向量库 |
| L1 | Neo4j Community | 5.20 | GPLv3 | 生产期图库 |
| L1 | SQLite | 3.42 | PD | metadata catalog |
| L1 | h5py | 3.11 | BSD | ARKit 帧序列 |
| L2 | ROS 2 | Humble Hawksbill | Apache 2.0 | 中间件 |
| L2 | tf2 | (ROS2 包) | BSD | 坐标变换 |
| L3 | ZED SDK | 4.1 | proprietary | ZED 2i 驱动 + VIO |
| L3 | pyzed | 4.1 | proprietary | Python 绑定 |
| L3 | RoomPlan / ARKit | iOS 17+ | Apple | 扫描 SDK |
| L3 | ORB-SLAM3 | 1.0 (备选) | GPLv3 | 备用 VIO |
| L3 | rtabmap | 0.21 (备选) | BSD | 备用 RGB-D SLAM |
| L4 | Open3D | 0.18 | MIT | 点云 / TSDF / ICP / RANSAC |
| L4 | TEASER++ | 1.0 | MIT | 鲁棒全局配准 |
| L4 | OctoMap | 1.10 + python | BSD | 占据栅格 |
| L4 | nvblox | 0.0.6 (Isaac ROS) | Apache 2.0 | GPU TSDF（可选替代） |
| L4 | gsplat | 1.0 | Apache 2.0 | 3DGS 训练 |
| L4 | nerfstudio | 1.1 | Apache 2.0 | NeRF/3DGS 工具链 |
| L4 | trimesh | 4.4 | MIT | mesh I/O |
| L4 | pxr (USD) | 24.05 | Modified Apache | USDZ 读写 |
| L5 | OpenCLIP | 2.24 | MIT | CLIP ViT-B/32 |
| L5 | DINOv2 | 0.2 | Apache 2.0 | 视觉 fingerprint 重排 |
| L5 | Ultralytics (YOLO) | 8.3 + YOLO-Worldv2 | AGPL | 开放词表检测 |
| L5 | SAM 2 | 1.0 | Apache 2.0 | 实例分割 |
| L5 | Grounding-DINO | 1.5 (备选) | Apache 2.0 | 文本→bbox |
| L6 | PRISM Core | v0.1 (自研) | TBD | 本项目 |
| L7 | LangChain / LangGraph | 0.2 | MIT | Agent 编排 |
| L7 | OpenAI Python | 1.40 | Apache 2.0 | GPT-4o |
| L7 | Qwen-VL | 2.5-VL-7B | Apache 2.0 | 国产备选 VLM |
---
## 10.3 关键选型理由
### 10.3.1 为什么是 Open3D 而不是 PCL？
- ✅ Python API 一等公民（PCL Python 绑定常年烂尾）
- ✅ 0.18 起 `VoxelBlockGrid` 提供 GPU TSDF，与 nvblox 性能接近
- ✅ ICP / RANSAC / FPFH 一体；与 `pyrender` / `trimesh` 互通好
- ❌ 大点云（> 1 亿点）仍不如 PCL+OpenMP——但 PRISM 单次只处理几百万点
### 10.3.2 为什么 ZED SDK 而不是 ORB-SLAM3？
- ✅ 开箱即用：双目 + IMU + VIO + Spatial Mapping 一站式
- ✅ 工业级稳定（生产部署见多）
- ✅ Jetson 上有官方优化版
- ❌ 闭源 → **备选 ORB-SLAM3** 作开源退路
### 10.3.3 为什么 CLIP + DINOv2 双模型？
- CLIP：语义对齐强（文本-图像），用于 "找遥控器"
- DINOv2：纯视觉几何指纹强（同类房间区分），用于消歧重定位
- 二者**互补不替代**：CLIP 召回，DINOv2 重排
### 10.3.4 为什么 YOLO-World 而不是 Grounding-DINO？
- ✅ YOLO-World 在 Jetson Orin 上 2 Hz 实时
- ✅ 开放词表 + COCO 预训练，常见家具/小物覆盖率 > 90%
- ❌ Grounding-DINO 精度更高但 ~ 1 fps，留作离线评测 / 提案兜底
### 10.3.5 为什么 RoomPlan 而不是 Polycam？
- ✅ Apple 官方、免费、有官方 USDZ + JSON 输出（含语义类别）
- ✅ 不依赖云服务，数据隐私可控
- ❌ 仅 iOS；非苹果用户用 Polycam（云端处理）作备选
### 10.3.6 为什么 Neo4j 而生产不用 JSON？
- 原型期 JSON+NetworkX 足够，加载 < 1 s
- 当节点 > 10 k（多楼层场景）时 Cypher 查询比线性扫描快 10–100×
- LLM 直接发 Cypher → 几乎零成本对接
---
## 10.4 计算平台对比
| 平台 | 价格 | 推理性能 | 适用 |
|------|------|----------|------|
| **Jetson Orin AGX 64 GB** | ¥18 k | YOLO-World 8 fps, CLIP 60 fps | ✅ **推荐机器人本体** |
| Jetson Orin AGX 32 GB | ¥14 k | 同上 | 内存吃紧（4 房间以内） |
| Jetson Orin NX 16 GB | ¥6 k | 减半 | 入门尝试 |
| RTX 4090 + i7 | ¥25 k | YOLO 50 fps, 3DGS 训练 | ✅ **推荐工作站** |
| MacBook Pro M3 Max | ¥26 k | MPS 后端，CLIP/CoreML 优秀 | ✅ 替代工作站 |
| 云 GPU (恒源云 4090) | ¥3 / hr | 同 RTX 4090 | 偶发训练 |
---
## 10.5 端到端依赖图（pip / apt 级）
```
Workstation Robot (Jetson Orin)
───────────── ─────────────────────
pip: pip:
pyzed ← ZED SDK 4.1 pyzed (jetson build)
open3d>=0.18 open3d (arm64 wheel)
ultralytics ultralytics-jetson
open_clip-torch open_clip-torch
nerfstudio (不需要训练，可跳过)
trimesh + pyrender (可跳过)
networkx + neo4j networkx
faiss-gpu faiss-cpu
octomap-python octomap-python (arm64)
scipy + numpy + opencv-contrib-python
rclpy (从 ROS 2 deb) rclpy
apt: apt:
ros-humble-desktop ros-humble-ros-base
ros-humble-tf2-tools ros-humble-tf2-ros
cuda-toolkit-12-2 JetPack 6.0 (含 CUDA)
libpcl-dev (备选) 同
```
---
## 10.6 推理优化（机器人端）
| 模型 | 原始 | 优化后 | 工具 |
|------|------|--------|------|
| CLIP ViT-B/32 | 80 ms / img | 25 ms | TensorRT FP16 |
| DINOv2-S | 60 ms | 18 ms | TensorRT FP16 |
| YOLO-World-x | 500 ms | 120 ms (2 Hz OK) | TensorRT INT8 |
| SAM 2 small | 600 ms | 200 ms | TensorRT FP16 |
| open3d ICP | CPU 80 ms | (无需，已快) | — |
部署：通过 `torch2trt` 或官方 `tensorrt-llm` 工具链导出 `.engine` 文件，PRISM 启动时加载。
---
## 10.7 仓库结构建议
```
prism/ # GitHub 仓库根
├── README.md
├── pyproject.toml # poetry / hatch
├── docker/
│ ├── workstation.Dockerfile
│ └── jetson.Dockerfile
├── spatial_memory/ # PRISM core 包（pip 可装）
│ ├── schema.py
│ ├── io_json.py
│ ├── io_usd.py
│ ├── parser_iphone.py
│ ├── align_to_map.py
│ ├── bake_octomap.py
│ ├── bake_tsdf.py
│ ├── relocalize_coarse.py
│ ├── relocalize_fine.py
│ ├── consolidation/
│ │ ├── filter.py
│ │ ├── arbiter.py
│ │ ├── apply.py
│ │ ├── refresh.py
│ │ └── commit.py
│ └── api.py # SpatialMemoryAPI
├── perception/ # 在线 worker
│ ├── frame_producer.py
│ ├── tsdf_worker.py
│ ├── detector_worker.py
│ ├── change_detector.py
│ └── delta_log.py
├── nodes/ # ROS 2 节点
│ ├── relocalizer_node.py
│ ├── frame_producer_node.py
│ ├── tsdf_worker_node.py
│ ├── detector_worker_node.py
│ └── consolidator_node.py
├── tools/ # CLI
│ ├── prism_ingest_iphone.py
│ ├── prism_consolidate.py
│ ├── prism_eval_reloc.py
│ ├── prism_viz.py
│ └── prism_test_e2e.py
├── ios/PRISMScanner/ # Swift App
├── launch/
│ ├── online.launch.yaml
│ └── consolidate.launch.yaml
├── configs/
│ ├── jetson_default.yaml
│ └── workstation_default.yaml
├── tests/
└── docs/ # 本目录就是 docs/PRISM/
```
---
## 10.8 配置文件示例
```yaml
# configs/jetson_default.yaml
hardware:
camera: zed2i
resolution: HD720
fps: 30
imu_rate: 400
depth_mode: QUALITY
max_depth: 5.0
perception:
frame_producer:
keyframe_interval_s: 0.5
buffer_seconds: 10
tsdf:
voxel_size: 0.02
block_count: 10000
rate_hz: 5.0
max_dist_m: 5.0
detector:
model: yolo-worldv2-x-trt-int8
classes: auto # 从 LTM 自动读取
rate_hz: 2.0
conf: 0.3
change_detector:
rate_hz: 1.0
diff_threshold_m: 0.05
cluster_eps: 0.15
cluster_min: 20
relocalize:
coarse:
model: clip-vit-b-32-trt-fp16
top_k: 3
voting_window: 5
fine:
voxel: 0.05
ransac_iters: 100000
icp_threshold: 0.025
accept_fitness: 0.85
fallback_fitness: 0.70
online:
drift_trigger_m: 0.5
health_check_period_s: 300
consolidation:
mode: full
trigger_idle_min: 10
promotion:
object_moved: {min_obs: 5, min_span_s: 300, min_views: 2}
object_removed: {min_obs: 10, min_span_s: 600, min_views: 3}
object_added: {min_obs: 8, min_span_s: 300, min_views: 2}
storage:
ltm_dir: /robot_memory/ltm
stm_dir: /robot_memory/stm
delta_dir: /robot_memory/delta
snapshot_dir: /robot_memory/snapshots
scene_graph_db: json # 切换到 'neo4j' 即生产模式
agent:
vlm: gpt-4o # 或 qwen2.5-vl-7b-trt
prompt_template: default
```
---
## 10.9 Docker 部署
### Workstation 镜像（训练 / ingest）
```dockerfile
# docker/workstation.Dockerfile
FROM nvidia/cuda:12.2.0-cudnn8-devel-ubuntu22.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y \
python3.10 python3-pip python3-venv \
git wget curl build-essential cmake \
libgl1 libegl1 ffmpeg \
&& rm -rf /var/lib/apt/lists/*
# ROS 2 Humble (略)
COPY pyproject.toml /workspace/
WORKDIR /workspace
RUN pip install --no-cache-dir -e .
RUN pip install pyzed open3d ultralytics open_clip_torch nerfstudio \
trimesh pyrender networkx faiss-gpu octomap-python
CMD ["bash"]
```
### Jetson 镜像（运行时）
基于 `nvcr.io/nvidia/l4t-jetpack:r36.3.0`，替换 `pyzed-jetson` `open3d-arm64` 等 wheel。
启动：
```bash
docker run --runtime nvidia --network host \
-v /robot_memory:/robot_memory \
-v /dev:/dev --privileged \
prism:jetson-v0.1 ros2 launch prism online.launch.yaml
```
---
## 10.10 兼容性矩阵
| 软件 A | 软件 B | 已知问题 / 注意事项 |
|--------|--------|--------------------|
| Open3D 0.18 | PyTorch 2.3 + CUDA 12 | 必须用 Open3D 自带 CUDA wheel，与 torch 共存正常 |
| ZED SDK 4.1 | Ubuntu 24 | 不官方支持，坚持 22.04 |
| YOLO-World | TensorRT 10 | 需关闭 dynamic shape；导出脚本见 ultralytics docs |
| nvblox | Open3D TSDF | 不能同时启用：选其一作为 L2 |
| Neo4j | ROS 2 | 默认 7687 端口，与 ROS 无冲突 |
| RoomPlan | iOS 17.5 | 17.4 之前 polygon 字段缺失，必须升 |
---
## 10.11 国产替代方案
| 国外 | 国产替代 | 兼容度 |
|------|----------|--------|
| ZED 2i | Intel RealSense D455i / 奥比中光 Gemini 2 | 接口不同，需自写 driver |
| Jetson Orin | 地平线 RDK X5 / 华为 Atlas 200I | TensorRT → MindSpore/ Bayes |
| GPT-4o | Qwen2.5-VL-7B / GLM-4V | API 调用兼容（OpenAI SDK 可指 base_url） |
| Neo4j | Nebula Graph / TuGraph | Cypher 方言基本一致 |
| OpenAI 嵌入 | BGE-M3 / Conan-embedding | 替换 CLIP text encoder |
---
## 10.12 安全 / License 合规清单
| 风险 | 项 | 处理 |
|------|----|------|
| AGPL 传染 | YOLO (Ultralytics AGPLv3) | 如商用：购买商业 license 或换 PaddleDetection |
| 闭源 | ZED SDK | 可商用但需注意分发限制 |
| Apple 限制 | RoomPlan 仅 iOS | 接受 / 用 Polycam 替代 |
| OpenAI 数据 | GPT-4o 上传 RGB | 切换到本地 Qwen-VL |
| 用户隐私 | 客人入镜 | 6.5.x 已实现 person 过滤 |
| 数据出境 | 国产场景 | 全本地化部署（无云依赖） |
---
## 10.13 本章小结
| 关键决策 | 一句话 |
|----------|--------|
| **VIO** | ZED SDK 自带（备选 ORB-SLAM3） |
| **重定位** | CLIP + DINOv2 + TEASER++/ICP |
| **检测** | YOLO-World（备选 Grounding-DINO） |
| **TSDF** | Open3D VoxelBlockGrid（备选 nvblox） |
| **3DGS** | gsplat + nerfstudio |
| **图库** | 原型 JSON+NetworkX，生产 Neo4j |
| **中间件** | ROS 2 Humble |
| **机器人端推理** | TensorRT FP16/INT8 |
| **国产替代** | 全栈可换，方案不锁死 |
读完本章你应能：
- ✅ 一次性把所有依赖装好
- ✅ 在 license / 国产化等约束下做替换决策
- ✅ 起一个干净的仓库骨架
下一章 [`11_world_model_bridge.md`](11_world_model_bridge.md) 讲 PRISM 如何与 M-JEPA / DreamerV3 等世界模型对接，把"空间记忆"升级为"可想象的世界"。
---
**章节版本**：v1.0
**估计阅读时间**：12 分钟
**关键收获**：拿到完整的版本号、依赖、镜像与仓库结构
plans/PRISM/11_world_model_bridge.md
+267
View File
@@ -0,0 +1,267 @@
# Chapter 11 — 与世界模型 (World Model) 的衔接
> 本章目标：说明 PRISM 输出的"空间记忆"如何被 **M-JEPA / DreamerV3 / Genie / NVIDIA Cosmos** 等世界模型消化，又如何回写到 PRISM——形成 **"感知 → 记忆 → 想象 → 验证"** 的闭环。
---
## 11.1 为什么记忆需要"接"世界模型
PRISM 自己回答了 *what is here*（这里有什么）和 *what was here*（这里曾有过什么）；但还没回答 *what would happen if...*（如果……会怎样）。
后者正是世界模型 (World Model) 的职责：
| 问题 | 谁回答 |
|------|--------|
| "床头柜在哪？" | PRISM L4 |
| "从门口到床走 2 秒会撞到吗？" | PRISM L2 + 简单运动学 |
| "把椅子推到门口，会挡住门吗？" | **需要世界模型 rollout** |
| "如果把水洒在床上，下一秒会发生什么？" | **需要物理世界模型** |
| "客人下一步可能要什么？" | **需要 latent dynamics** |
PRISM 的角色：**给世界模型提供"上下文 + 初始状态 + 反事实约束"**。
---
## 11.2 PRISM 与 World Model 的对接接口
```mermaid
flowchart TB
ACT["Action / Plan"]
subgraph WM["World Model (WM)"]
ENC["Encoder<br/>(obs → z)"]
DYN["Dynamics<br/>(z, a → z')"]
DEC["Decoder<br/>(z → obs / r)"]
ENC --> DYN --> DEC
end
subgraph PRISM["PRISM"]
direction LR
L1["L1"] --- L2["L2"] --- L3["L3"] --- L4["L4"]
DELTA["delta + anchors"]
end
WM --> ACT
PRISM -- "context<br/>(LTM 上下文)" --> WM
WM -- "reality check<br/>(rollout vs delta)" --> PRISM
style WM fill:#e3f2fd,stroke:#1565c0
style PRISM fill:#fff7d6,stroke:#c97a00
style ACT fill:#d4f0d4,stroke:#2e7d32
```
PRISM 与 WM 之间有 **4 条接口**：
| 方向 | 接口 | 含义 |
|------|------|------|
| PRISM → WM | **观测 obs** | 当前 ZED 帧 + 当前 L4 节点的属性 |
| PRISM → WM | **上下文 context** | L3 + L4 的场景图作为 prior |
| WM → PRISM | **想象 rollout** | 假设动作 a 序列下未来 z̃ 的解码图像/状态 |
| WM → PRISM | **物理先验** | 物体的可移动性、稳定性、堆叠规则等 |
---
## 11.3 三类目标世界模型与对接方式
### 11.3.1 M-JEPA（Meta 的预测式 JEPA 系列）
- **特点**：在隐空间预测下一时刻的"嵌入"，不重建像素，物理一致性好
- **对接**：
- 把 PRISM L1 关键帧 + 当前位姿 + 当前局部 TSDF 拼成 `obs`
- 把 L4 场景图序列化为 token 注入到 M-JEPA 的 context 端
- rollout 得到的下一 z 通过 decoder（如轻量 NeRF 渲染头）变回图像，与下一帧真实图像对比
```python
# bridge/mjepa_bridge.py
def build_obs(prism: PRISM, t: float) -> Dict:
return {
"rgb": prism.l1.frames[-1].rgb_left,
"depth": prism.l1.frames[-1].depth,
"pose": prism.l1.frames[-1].pose.to_matrix(),
"occupancy": prism.l2.local_octomap_crop(radius=3.0),
"graph": prism.l4.serialize_neighborhood(radius=3.0),
}
```
### 11.3.2 DreamerV3 / TD-MPC2（基于 RSSM 的循环世界模型）
- **特点**：训练快，适合控制任务（导航、抓取）
- **对接**：用 PRISM 提供的**任务级 reward 信号** —— 例如 "到达目标房间" 由 PRISM L3 判定
- **PRISM 既是 obs source，也是 reward function**：
```python
class PRISMDreamerEnv(gym.Env):
def __init__(self, prism, sim_or_real):
self.prism = prism
self.sim = sim_or_real
def reset(self): return self._obs_from_prism()
def step(self, action):
self.sim.apply(action)
obs = self._obs_from_prism()
reward = self.prism.api.task_progress() # 调 API
return obs, reward, done, info
```
### 11.3.3 视频生成式世界模型（Cosmos / Genie 2 / Sora 类）
- **特点**：像素级 rollout，逼真度高，可生成"如果机器人转左 90° 会看到什么"
- **对接**：
- 用 PRISM L2 的 3DGS 渲染当前帧作为 "first frame"
- 文本指令 + first frame → 视频生成
- 生成的视频被 PRISM detector 复用做"反事实检测"
---
## 11.4 三类核心用法
### 11.4.1 用法 A：Planning by Imagination（用 WM 做规划）
```
1. 任务 = "在 308 把椅子推到角落"
2. PRISM 提供：椅子 bbox、房间 polygon、可通行栅格
3. WM rollout：候选 5 条推送轨迹，模拟每条的结果
4. 评估：每个 rollout 后用 PRISM 验证"椅子是否真在角落"
5. 选最优轨迹，执行
```
### 11.4.2 用法 B：Anomaly Detection（用预测误差发现异常）
```
1. 在线时 WM 持续预测下一帧 z̃
2. 与真实下一帧 z 对比
3. 误差爆增 → 异常事件（玻璃打碎？人摔倒？）
4. PRISM 把异常写入 delta 并触发告警
```
### 11.4.3 用法 C：Self-Supervised Pretraining（用 PRISM 数据训 WM）
```
1. 机器人长期运行积累 stm/keyframes/ 与 pose / action 配对
2. 用这些数据 fine-tune M-JEPA / DreamerV3
3. 每月触发一次，使 WM 越来越懂这个具体场景
```
---
## 11.5 数据格式约定（PRISM-WM Bridge Schema）
```python
@dataclass
class WMObservation:
rgb: np.ndarray # (H,W,3) uint8
depth: np.ndarray # (H,W) float32, meters
pose_world: np.ndarray # (4,4)
local_occ: np.ndarray # (Dx,Dy,Dz) bool, 3m 邻域
graph_tok: List[int] # 序列化场景图 token (≤ 256)
timestamp: float
@dataclass
class WMAction:
type: Literal["move","push","grasp","look_at"]
params: Dict
# e.g. {"type":"move", "params":{"delta_xy":[0.3,0], "delta_yaw":0}}
@dataclass
class WMRollout:
actions: List[WMAction]
predicted_obs: List[WMObservation]
predicted_reward: List[float]
confidence: float
```
→ 这是 PRISM ↔ WM 的"USB 接口"，未来不同 WM 都按这个对接。
---
## 11.6 一个完整闭环示例
```
任务："把垃圾桶从 305 推到电梯厅"
1. PRISM 查 L4 → trash_bin_305, bbox, parent=room_305
2. PRISM 查 L3 → 路径 room_305 → hallway → lift_lobby
3. PRISM 提供 obs_t（当前帧、局部栅格、相关 L4 子图）
4. WM (DreamerV3) 内部 imagine 100 步：
- 候选动作序列 a_{1:100}
- 预测 z_{1:100}, reward_{1:100}
- 选最大累计 reward 的 a 序列
5. 机器人执行 a_1，得到 obs_{t+1}
6. PRISM 用真实 obs_{t+1} 与 WM 预测的 z̃_1 比对：
- 一致 → continue
- 不一致 → 误差注入 WM 再 imagine（重规划）
- 不一致 + 持续大 → 写 delta（环境变了，比如垃圾桶卡住了）
7. 任务结束后：
- 真实轨迹 (obs, a, obs') 累积到 stm/episodes/
- 月度 fine-tune WM
```
---
## 11.7 三层抽象的对应关系
| WM 概念 | PRISM 对应 |
|---------|-----------|
| Observation | L1 关键帧 + L2 局部 occupancy |
| Hidden state z | 不直接对应，但 PRISM 的 L4 节点是其"符号侧倒影" |
| Action | 机器人控制器输出，PRISM 不管 |
| Reward | PRISM API 提供任务级信号 |
| Prior | L3+L4 场景图作为 context token |
| Imagination | 在 WM 内部，不存进 PRISM；只把"决策"和"反例"回写 |
| Reality | PRISM 实时观测，作为 WM 训练 / 校正的 ground-truth |
---
## 11.8 与既有项目对比
| 项目 | 类型 | PRISM 关系 |
|------|------|-----------|
| **Habitat** | 仿真 | PRISM 的 LTM 可导出为 Habitat 场景 |
| **Isaac Sim** | 仿真 | LTM 的 USDZ 直接打开 |
| **NVIDIA Cosmos** | 视频 WM | 可作为 PRISM 的"生成式预测器" |
| **Hydra / Hovsg / ConceptGraphs** | hierarchical SLAM | PRISM 与之同源；区别：PRISM 强调"先验灌入 + 巩固机制" |
| **VoxFormer / NeRF-SLAM** | 单一稠密表示 | PRISM 用作 L2 实现，但加入 L3/L4 |
| **VLMaps / OpenScene** | 开放词表场景图 | 可作为 PRISM 的 L4 backbone |
| **Embodied-CoT / SayCan** | LLM 规划 | 是 PRISM 的下游消费者 |
---
## 11.9 时间表（与 [`09_roadmap.md`](09_roadmap.md) 衔接）
| 阶段 | WM 相关任务 |
|------|-------------|
| 8 周原型 | **不接 WM**，先打地基 |
| M3-M4 | 实现 WMObservation / WMAction schema；DreamerV3 在仿真里跑通 |
| M5 | 接入真实 PRISM，做 Planning by Imagination demo |
| M6 | 异常检测；fine-tune WM 的训练管线 |
| M7+ | 切换到 M-JEPA 或 Cosmos，做语义 / 视频级 rollout |
---
## 11.10 局限与开放问题
| 局限 | 说明 |
|------|------|
| WM 在长时间 rollout 中漂移 | 必须靠 PRISM 不断"reality check" |
| L4 场景图怎么 token 化 | 当前是手写 (label, pos, attr) 三元组 → 可学习的图 encoder 待研究 |
| PRISM 是否要"反向写入"WM 输出 | 暂定**不写**：rollout 是虚拟，写进 LTM 会污染 |
| Reward 设计 | PRISM 提供基本 task progress；复杂 reward 需领域专家 |
| 数据效率 | 单楼层 episode 数有限，跨场景迁移学习是关键 |
---
## 11.11 本章小结
| 关键点 | 一句话 |
|--------|--------|
| **PRISM 之于 WM** | 提供 context + obs + reward + reality check |
| **WM 之于 PRISM** | 提供 imagination + 物理先验，不直接写 LTM |
| **接口** | `WMObservation / WMAction / WMRollout` 三类数据结构 |
| **三类用法** | Planning / Anomaly / Pretraining |
| **闭环关键** | 真实观测和 imagine 偏差大 → 写 delta + 触发 replan |
| **现阶段** | 8 周原型不接 WM；M5 之后逐步引入 |
读完本章你应能：
- ✅ 解释 PRISM 与世界模型的本质区别（记忆 vs 想象）
- ✅ 设计一个最小的 PRISM→DreamerV3 接入
- ✅ 知道为什么 WM 的"幻想"不能写进 LTM
下一章 [`12_risks.md`](12_risks.md) 整理 PRISM 全系统的 6 大风险与对策。
---
**章节版本**：v1.0
**估计阅读时间**：12 分钟
**关键收获**：把 PRISM 升级为"有想象力的大脑"的路径
plans/PRISM/12_risks.md
+358
View File
@@ -0,0 +1,358 @@
# Chapter 12 — 风险与对策
> 本章目标：把 PRISM 全系统在**采集 / 部署 / 运行 / 长期维护**四个阶段可能出问题的地方系统化列出，并给出每项的**缓解措施 / 监控指标 / 应急预案**。
---
## 12.1 风险分类总览
```mermaid
flowchart LR
ROOT["<b>PRISM 风险地图</b>"]
R1["<b>R1 数据风险</b><br/>扫描漏点<br/>坐标系错<br/>镜面区无效"]
R2["<b>R2 算法风险</b><br/>重定位失败<br/>漂移<br/>检测错配"]
R3["<b>R3 工程风险</b><br/>资源不足<br/>依赖断裂<br/>并发竞争"]
R4["<b>R4 物理环境风险</b><br/>高反光<br/>动态人<br/>突发遮挡"]
R5["<b>R5 数据安全风险</b><br/>客人隐私<br/>数据出境<br/>AGPL 传染"]
R6["<b>R6 长期维护风险</b><br/>家具频繁变化<br/>LTM 老化<br/>人员流失"]
ROOT --> R1 & R2 & R3 & R4 & R5 & R6
style ROOT fill:#fff7d6,stroke:#c97a00,stroke-width:2px
style R1 fill:#e3f2fd,stroke:#1565c0
style R2 fill:#fde2e2,stroke:#a33
style R3 fill:#d4f0d4,stroke:#2e7d32
style R4 fill:#fff1c1,stroke:#a87a00
style R5 fill:#f5e1ff,stroke:#7b1fa2
style R6 fill:#d8e4ff,stroke:#1565c0
```
每个风险给出：**概率 × 影响 → 等级 → 对策 → 监控指标**。
---
## 12.2 R1 数据风险
### R1.1 iPhone 扫描漏点 / 几何不闭合
| 项 | 内容 |
|----|------|
| **概率** | 中（每次扫描 30% 概率有漏点） |
| **影响** | 中（机器人可能撞到墙或穿墙规划） |
| **等级** | 🟡 中 |
| **触发条件** | 弱纹理大白墙、强光下、扫描速度过快 |
| **对策** | ① 采集 SOP（[`04_pipeline_A_iphone_offline.md`](04_pipeline_A_iphone_offline.md) §4.2.2）<br>② 多次扫描融合<br>③ 后期人工补点（MeshLab） |
| **监控** | `prism qa` 检查每房间是否闭合 polygon |
| **应急** | 立刻补扫；机器人在该区域加 2D LiDAR 兜底 |
### R1.2 坐标系不一致（iPhone vs ZED `map` 帧）
| 项 | 内容 |
|----|------|
| **概率** | 高（如果不放 ArUco 几乎必失败） |
| **影响** | 致命（机器人位姿完全错） |
| **等级** | 🔴 高 |
| **对策** | ① 强制约定 ArUco 公共原点（[`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) §5.9）<br>② 重力对齐双校验 |
| **监控** | ingest 时若没检到 ArUco 直接报错退出（fail fast） |
| **应急** | 不允许部署没有 ArUco 的 LTM |
### R1.3 镜面 / 玻璃区域几何错乱
| 项 | 内容 |
|----|------|
| **概率** | 高（任何卫生间/玻璃幕墙都有） |
| **影响** | 中（局部 mesh 坏 + 重定位失败） |
| **等级** | 🟡 中 |
| **对策** | ① iPhone 扫描时人工标镜面 → `no_update_zone` mask<br>② 后期跑反射检测（偏振相机或 RGB 反射启发式）补打<br>③ TSDF Worker 在该区域 weight=0 |
| **监控** | 每天巡逻一次镜面区，检查 TSDF noise level |
| **应急** | 卫生间默认整间标 no_update，宁可粗也不污染 |
---
## 12.3 R2 算法风险
### R2.1 重定位失败（特别是冷启动）
| 项 | 内容 |
|----|------|
| **概率** | 低（< 5%，但出现就阻塞所有任务） |
| **影响** | 高（机器人不能工作） |
| **等级** | 🔴 高 |
| **触发** | 强光直射、视野遮挡、相似房间（连号客房） |
| **对策** | ① CLIP + DINOv2 双 backbone 投票<br>② 多帧投票（5 帧）<br>③ ArUco 兜底（房间门口贴）<br>④ 遥控人工"我在这"按钮 |
| **监控** | `prism eval reloc` 每日跑；fitness 滑动均值 < 0.7 报警 |
| **应急** | 启动 ArUco 兜底；通知运维 |
### R2.2 VIO 长时漂移
| 项 | 内容 |
|----|------|
| **概率** | 高（必发生） |
| **影响** | 中（位姿误差累积，但 anchor 校正可吸收） |
| **等级** | 🟡 中 |
| **对策** | ① 每见到 anchor 触发 ICP 校正<br>② 每 5 min 主动重定位健康检查<br>③ 漂移估计 > 50 cm 强制 reloc |
| **监控** | drift_estimate 时间序列；anchor 命中间隔 |
| **应急** | 触发完整重定位 |
### R2.3 检测错配（YOLO 把椅子识成 sofa）
| 项 | 内容 |
|----|------|
| **概率** | 中（开放词表模型常见） |
| **影响** | 低（单次 → confidence 低 → 不会污染 LTM） |
| **等级** | 🟢 低 |
| **对策** | ① CLIP-rerank 二次验证<br>② Consolidate 时的高门槛（min_obs / min_views）<br>③ 跨房间一致性检查 |
| **监控** | delta confirm 率（应在 10–30%） |
| **应急** | 暂停 detector，先查模型版本 |
### R2.4 差异检测假阳性（沙发被坐了不是被搬走）
| 项 | 内容 |
|----|------|
| **概率** | 中（人活动时常触发） |
| **影响** | 低（不立即改 LTM） |
| **等级** | 🟢 低 |
| **对策** | ① 隐私过滤（检到 person 跳过该帧）<br>② min_obs ≥ 10、min_span_s ≥ 600<br>③ iPhone 高 conf 节点门槛 1.5x |
| **监控** | delta reject 率 > 70% 反而正常（说明过滤起效） |
---
## 12.4 R3 工程风险
### R3.1 Jetson 资源不足
| 项 | 内容 |
|----|------|
| **概率** | 中 |
| **影响** | 高（节点掉帧 / 崩溃） |
| **等级** | 🔴 高 |
| **对策** | ① TensorRT FP16/INT8（[`10_tech_stack.md`](10_tech_stack.md) §10.6）<br>② Worker 频率自适应（移动时高，静止时低）<br>③ 选 Orin AGX 64 GB 而非 32 GB |
| **监控** | `tegrastats` 实时；任一 worker 延迟 > 2× 目标即降级 |
| **应急** | 优先停 ChangeDetector 与 3DGS；YOLO 类别动态裁剪 |
### R3.2 依赖版本断裂（CUDA / PyTorch / Open3D）
| 项 | 内容 |
|----|------|
| **概率** | 高（每次升级都可能） |
| **影响** | 中（不能跑） |
| **等级** | 🟡 中 |
| **对策** | ① Docker 固化版本<br>② `pyproject.toml` 钉死 minor 版本<br>③ CI 跑 nightly 跨版本 smoke test |
| **监控** | CI badge 红即拒绝合并 |
### R3.3 并发竞争（在线感知 ↔ Consolidation）
| 项 | 内容 |
|----|------|
| **概率** | 中（每次巩固） |
| **影响** | 中（可能数据丢失） |
| **等级** | 🟡 中 |
| **对策** | ① 巩固开始前调 `set_readonly(true)`；感知此期间只能写 delta<br>② staging 写完才原子切换<br>③ 文件锁 + 进程级互斥 |
| **监控** | 巩固日志中的 lock 等待时间 |
### R3.4 磁盘/IO 瓶颈
| 项 | 内容 |
|----|------|
| **概率** | 低 |
| **影响** | 中（关键帧丢） |
| **等级** | 🟢 低 |
| **对策** | ① SSD 而非 eMMC<br>② keyframes 用环形写（满了覆盖）<br>③ 巩固时压缩老 snapshot |
| **监控** | 磁盘空闲 < 20% 报警 |
---
## 12.5 R4 物理环境风险
### R4.1 高反光（卫生间镜面、玻璃幕墙）
参见 R1.3。再加一条：
| 对策 | 在 LTM 中给整间卫生间标 `category=high_reflection`，机器人路径规划时绕远 |
### R4.2 动态人 / 宠物
| 项 | 内容 |
|----|------|
| **概率** | 高（酒店满人） |
| **影响** | 中 |
| **等级** | 🟡 中 |
| **对策** | ① YOLO 检 `person` → 该帧跳过语义写入（保留几何用于避障）<br>② Avoidance 优先；Nav stack 与 PRISM 解耦<br>③ 跟随人时不写 delta |
| **监控** | 每小时 person-frame 比率，超 50% 触发"高密度模式" |
### R4.3 突发遮挡（清洁车、行李堆）
| 对策 | ① 临时遮挡 < 30 s 不写 delta<br>② Avoidance 短期记忆（local OctoMap 持续 30 s） |
### R4.4 极端光照（开灯/关灯切换）
| 对策 | ① ZED Auto Exposure<br>② 关灯时 fallback 到深度 + 几何重定位（CLIP 退化）<br>③ Anchor ICP 不依赖光照 |
---
## 12.6 R5 数据安全风险
### R5.1 客人隐私（人脸 / 行李 / 身份信息）
| 项 | 内容 |
|----|------|
| **概率** | 必然 |
| **影响** | 高（法律与商业） |
| **等级** | 🔴 高 |
| **对策** | ① YOLO 检 `person/face/luggage` → 像素级模糊或丢弃<br>② RGB 关键帧默认本地存储，不上传云<br>③ Snapshot 加密<br>④ 30 天后自动删除 stm/keyframes |
| **监控** | 每周抽样 100 张关键帧人工检查 |
| **应急** | 客人投诉立即下机检查 |
### R5.2 数据出境（GPT-4o）
| 对策 | 国内场景全栈本地化：Qwen2.5-VL + BGE-M3 + 本地 Cypher |
### R5.3 LICENSE 传染（YOLO AGPL）
| 对策 | ① 商用购 Ultralytics 商业 license；② 或换 PaddleDetection / NanoDet |
---
## 12.7 R6 长期维护风险
### R6.1 家具频繁被移动
| 项 | 内容 |
|----|------|
| **概率** | 高（每天清洁） |
| **影响** | 中 |
| **等级** | 🟡 中 |
| **对策** | ① 给易移动家具打 `mobile=true`，不作 anchor<br>② Consolidate 时这些节点门槛更低<br>③ delta 中关于 mobile 的事件不报警 |
### R6.2 LTM 老化（与现实偏离）
| 项 | 内容 |
|----|------|
| **概率** | 必然（30 天后偏差明显） |
| **影响** | 中 |
| **等级** | 🟡 中 |
| **对策** | ① 月度 deep consolidation<br>② anchor 失活率超 30% 自动提示重扫<br>③ 季度人工重扫一次 |
| **监控** | `prism health` 给出 LTM 健康分（0–100） |
### R6.3 团队人员流失 / 文档失同步
| 对策 | ① 本系列文档；② Schema 版本号 + migrate 函数；③ 每月 release notes |
### R6.4 错误巩固导致 LTM 损坏
| 项 | 内容 |
|----|------|
| **概率** | 低 |
| **影响** | 致命 |
| **等级** | 🔴 高（罕见但毁灭） |
| **对策** | ① 巩固后 sanity_relocalize 自动校验<br>② 失败回滚（保留 `.prev`）<br>③ snapshots 保留 6 份 |
| **监控** | 每次巩固后 health 分变化 ≤ -5 即拒绝 |
| **应急** | 自动回滚 + 人工调查 |
---
## 12.8 风险矩阵（影响 × 概率）
```mermaid
quadrantChart
title 风险矩阵 影响 x 概率
x-axis 低影响 --> 高影响
y-axis 低概率 --> 高概率
quadrant-1 高概率高影响 必须立即处理
quadrant-2 高概率低影响 容忍但要监控
quadrant-3 低概率低影响 可接受
quadrant-4 低概率高影响 建预案
R1-2 ArUco 公共原点: [0.88, 0.85]
R5-1 客人隐私: [0.85, 0.82]
R2-2 漂移: [0.65, 0.82]
R6-1 家具频繁变化: [0.62, 0.80]
R4-4 高反光: [0.20, 0.85]
R3-1 Jetson 资源不足: [0.78, 0.55]
R6-4 巩固回滚: [0.75, 0.52]
R1-1 扫描漏点: [0.55, 0.50]
R1-3 坐标系错: [0.58, 0.48]
R3-3 依赖断裂: [0.55, 0.45]
R4-2 动态人: [0.52, 0.42]
R2-4 检测错配: [0.22, 0.55]
R2-1 重定位失败: [0.82, 0.20]
R5-2 数据出境: [0.78, 0.18]
R6-2 LTM 老化: [0.55, 0.22]
R3-4 并发竞争: [0.20, 0.22]
```
🔴 高风险高影响项必须在 8 周原型期前解决：
- R1.2 ArUco 公共原点（W1 必做）
- R2.1 重定位双 backbone（W3-W4）
- R3.1 Jetson 资源监控（W6）
- R5.1 隐私过滤（W5）
- R6.4 巩固回滚（W7）
---
## 12.9 风险登记表（Risk Register）
| ID | 风险 | 概率 | 影响 | 等级 | Owner | 状态 | 触发指标 |
|----|------|------|------|------|-------|------|----------|
| R1.1 | 漏点 | 中 | 中 | 🟡 | iOS dev | open | polygon 闭合率 |
| R1.2 | 坐标系错 | 高 | 致命 | 🔴 | iOS dev | mitigated by ArUco | ArUco 未检出 |
| R1.3 | 镜面 | 高 | 中 | 🟡 | ML | open | TSDF 噪声 |
| R2.1 | reloc 失败 | 低 | 高 | 🔴 | ML | open | fitness 滑均 |
| R2.2 | VIO 漂 | 高 | 中 | 🟡 | Robotics | mitigated by anchor | drift_estimate |
| R2.3 | 误检 | 中 | 低 | 🟢 | ML | open | delta confirm 率 |
| R3.1 | 资源不足 | 中 | 高 | 🔴 | System | mitigated by Orin64 | tegrastats |
| R3.3 | 竞争 | 中 | 中 | 🟡 | System | mitigated by lock | lock wait |
| R5.1 | 隐私 | 必然 | 高 | 🔴 | System+ML | mitigated | 抽样审查 |
| R6.4 | 错误巩固 | 低 | 致命 | 🔴 | System | mitigated by rollback | health Δ |
| ... | | | | | | | |
→ 用 Notion / 飞书表格维护，每周 review。
---
## 12.10 应急联动手册（Runbook）
| 报警 | 自动动作 | 人工动作 | 升级 |
|------|----------|----------|------|
| reloc 失败 ≥ 3 次 | 启用 ArUco 兜底 | 检查 RGB 是否过曝 | 1 h 内修不好 → 下机 |
| drift > 50 cm | 强制 reloc | 看 anchor 是否被搬走 | 24 h 后重扫 |
| delta confirm < 1% 持续 1 周 | 警告 | 检查检测器 / 模型版本 | 升级 ML 工程师 |
| consolidate 失败回滚 | 自动 + 邮件 | 查 sanity 报告 | 24 h 内查清 |
| 隐私抽检发现人脸未模糊 | 全量下机 | 修复后回归 | 法务介入 |
| LTM 健康分 < 60 | 提示重扫 | 安排现场 | — |
---
## 12.11 与 [`09_roadmap.md`](09_roadmap.md) 的 Gate 联动
每个 Gate 必须扫一遍本章 🔴 高风险项：
| Gate | 必查项 |
|------|--------|
| W2 | R1.2 ArUco 实装 |
| W4 | R2.1 reloc 成功率达标 |
| W6 | R3.1 资源 + R5.1 隐私 |
| W7 | R6.4 回滚机制 |
| W8 | 所有 🔴 通过 |
| M4 | 72 h 压测覆盖所有 R |
---
## 12.12 本章小结
| 关键 | 一句话 |
|------|--------|
| **致命风险** | 坐标系错 + 隐私 + 错误巩固 |
| **常见但可控** | 漂移、误检、家具搬动 |
| **应急核心** | 自动回滚 + ArUco 兜底 + 隐私一键下机 |
| **维护核心** | 月度巩固 + 季度重扫 + Risk Register 每周 review |
读完本章你应能：
- ✅ 给老板列出"PRISM 上线前必须解决的 5 件事"
- ✅ 设计 SLO / SLI 与报警阈值
- ✅ 写出 Runbook 给运维
下一章 [`13_evaluation.md`](13_evaluation.md) 给出系统的客观评测指标和数据集。
---
**章节版本**：v1.0
**估计阅读时间**：15 分钟
**关键收获**：把"可能出问题的事"变成"已被监控、已被准备好回滚的事"
plans/PRISM/13_evaluation.md
+395
View File
@@ -0,0 +1,395 @@
# Chapter 13 — 评测指标
> 本章目标：给出 PRISM **每一层、每一管线**的客观评测指标、测试数据集、自动化脚本，确保"做得对"而不是"看起来对"。
---
## 13.1 评测哲学
```
指标分为四类：
① Layer-wise 每层自身的健康指标（L1/L2/L3/L4 各自达标）
② Pipeline-wise 每条管线 A/B/C/D 的端到端指标
③ System-wise 整体可靠性（72h 压测、健康分）
④ Task-wise 上层任务（取物、巡逻、问答）成功率
测试三种数据：
• Synthetic Isaac Sim 仿真，可控、可重复
• Replay 录制的真实数据回放
• Live 机器人真实运行
```
---
## 13.2 Layer-wise 指标
### L1 感知缓冲
| 指标 | 目标 | 怎么测 |
|------|------|--------|
| 帧到达频率 | 30 ± 1 Hz | 时间戳直方图 |
| 写入延迟 | < 50 ms (P95) | grab→ring.append 计时 |
| 关键帧采样均匀性 | 间隔 STD < 100 ms | 关键帧时间序列 |
| 内存稳定 | < ±5% 漂移/h | RSS 长时间日志 |
### L2 度量
| 指标 | 目标 | 怎么测 |
|------|------|--------|
| 几何精度（Chamfer） | < 3 cm（静态区域） | 与 iPhone Mesh GT 对比 |
| 占据栅格 free/occ 准确 | F1 > 90% | 人工标 GT 随机 100 体素 |
| 局部 → 全局合并延迟 | < 5 s | 时间戳跟踪 |
| 先验区被错误更新 | 0 次 | prior_mask 写入计数 |
| 镜面区噪声 | TSDF noise level < 阈值 | 卫生间专项测 |
### L3 拓扑
| 指标 | 目标 | 怎么测 |
|------|------|--------|
| 房间识别正确 | 100% | 与 RoomPlan label 对比 |
| 门窗连边完整 | Recall > 95% | 与平面图对照 |
| 路径规划成功率 | > 99% | 随机起终点 1000 次 |
### L4 语义
| 指标 | 目标 | 怎么测 |
|------|------|--------|
| 家具识别 mAP@0.5 | > 70% | YOLO 内置 eval |
| 物品-家具关系正确 | > 80% | 人工标 100 关系 |
| CLIP 检索 Recall@1 | > 85%（房间） / 70%（家具） | 人写 100 个 query |
| 隐私过滤召回 | 100%（人不入 LTM） | 录制有人帧专项 |
---
## 13.3 Pipeline-wise 指标
### 管线 A — iPhone ingest
| 指标 | 目标 |
|------|------|
| 扫描 → LTM 时长 | < 30 min / 房间 |
| 节点数 | 单房 30–80 |
| Anchor 数 | 每房 ≥ 2 |
| Schema validate | 0 errors |
| Coverage（mesh 覆盖墙体面积） | > 90% |
### 管线 B — 重定位（[`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) §5.10）
| 指标 | 目标 |
|------|------|
| 冷启动 Recall@1 | > 90% |
| 冷启动延迟 | < 1 s |
| 位置误差 | < 10 cm |
| 朝向误差 | < 5° |
| 绑架恢复时间 | < 30 s |
### 管线 C — 在线感知
| 指标 | 目标 |
|------|------|
| 30 Hz 主循环稳定 | 99% 帧不超时 |
| 检测时延 (frame→delta) | < 500 ms |
| Detector mAP@0.5 | > 65%（开放词表） |
| Change Detector 查准 | > 85% |
| Change Detector 查全 | > 70% |
| 4 h 运行无崩溃 | 100% |
### 管线 D — 巩固
| 指标 | 目标 |
|------|------|
| 巩固耗时 (full mode) | < 1 h |
| 应用事件 / pending 比 | 10–30% |
| Sanity reloc 通过率 | > 90% |
| 回滚触发率（健康下降） | < 1 / 月 |
---
## 13.4 System-wise 指标
### 健康分 (Health Score)
```python
def compute_health(mem: SpatialMemory) -> float:
"""0–100"""
s = 100.0
# 锚点数：每房间应 ≥ 2 个 anchor
rooms = mem.nodes_of_level(MemoryLevel.L3)
weak_rooms = sum(1 for r in rooms if len(r.anchors) < 2)
s -= 5 * weak_rooms
# 节点新鲜度：3 个月未观测的节点占比
now = time.time()
stale = sum(1 for n in mem.nodes.values()
if now - n.last_seen > 90*86400)
s -= 20 * (stale / max(1, len(mem.nodes)))
# delta 积压
pending = len(load_pending())
if pending > 2000: s -= 10
# validate 错误
errs = validate(mem)
s -= 5 * len(errs)
return max(0.0, min(100.0, s))
```
每天 23:55 跑一次，写入 `logs/metrics.parquet`。
| 健康分 | 含义 | 行动 |
|--------|------|------|
| 90+ | 优秀 | 无需干预 |
| 70–90 | 良 | 关注趋势 |
| 50–70 | 警告 | 安排巡检 |
| < 50 | 危险 | 强制重扫 / 回滚 |
### 长时压测（72 小时）
| 指标 | 目标 |
|------|------|
| 累计运行时长 | ≥ 72 h 无 crash |
| 内存 / GPU 内存 | < 5% 增长 |
| 重定位漂移（重启对比） | < 30 cm |
| LTM 节点数 | 仅微增（< 5%） |
| delta 处理无堆积 | pending 周期性归零 |
### SLO / SLI
| SLI | SLO（月度） |
|-----|------------|
| 重定位成功率 | ≥ 95% |
| 任务完成率 | ≥ 90% |
| 平均无故障时长 (MTBF) | ≥ 48 h |
| 错误巩固率 | ≤ 0.5% |
| 隐私违规事件 | 0 |
---
## 13.5 Task-wise 指标（上层应用）
### 任务 1：取物（"去 X 房间取 Y"）
| 指标 | 目标 |
|------|------|
| 整体成功率 | > 90% |
| 子任务：物品定位 | > 95% |
| 子任务：导航到位 | > 98% |
| 子任务：抓取成功 | > 90% |
| 平均耗时 | < 60 s |
### 任务 2：巡逻 + 异常检测
| 指标 | 目标 |
|------|------|
| 完整覆盖率（24 h 内） | > 95% |
| 真实异常检出 | > 80% |
| 误报率 | < 5 / 天 |
### 任务 3：自然语言问答（"床头柜在哪？"）
| 指标 | 目标 |
|------|------|
| Top-1 正确 | > 85% |
| Top-3 召回 | > 95% |
| 平均响应延迟 | < 1 s |
---
## 13.6 测试数据集
### 13.6.1 HotelScene-Bench v0.5（自建）
```
HotelScene-Bench/
├── splits/
│ ├── train/ 6 个房间扫描
│ ├── val/ 2 个房间扫描
│ └── test/ 2 个房间扫描（hold-out）
├── annotations/
│ ├── object_3d_bbox.json
│ ├── room_polygons.json
│ ├── relocalize_gt/ 500 张 frame + 6DoF GT
│ └── change_events_gt/ 50 个搬动/搬入事件
├── traces/
│ ├── trace_normal_24h.bag
│ ├── trace_with_changes.bag
│ └── trace_kidnap.bag
└── tasks/
├── fetch_100.jsonl 100 条取物任务
├── qa_200.jsonl 200 条问答
└── patrol_routes.json
```
### 13.6.2 公开 benchmarks（用于横向对比）
| Benchmark | 用途 |
|-----------|------|
| ScanNet++ | 几何精度对比 |
| Replica | 重定位评测 |
| Habitat-Matterport 3D | 任务级评测 |
| HM3D-Semantic | 语义对比 |
| OpenEQA | 问答评测 |
---
## 13.7 自动化评测脚本
```bash
# 单一指标
prism eval reloc --bench HotelScene-Bench/splits/test
prism eval change --bench HotelScene-Bench/annotations/change_events_gt
prism eval consolidate --bench HotelScene-Bench/traces/trace_normal_24h.bag
# 全套（生成 HTML 报告）
prism eval all --output reports/$(date +%F)/
```
实现：每个 sub-command 是一个 Python 脚本，统一输出 JSON + Markdown + PNG 图表：
```python
# tools/prism_eval_reloc.py
def main(bench):
gt_frames = load_gt(bench/"relocalize_gt/")
mem = load("robot_memory/ltm/spatial_memory.json")
results = []
for frame, gt_pose in gt_frames:
pred = relocalize(frame, mem)
results.append(dict(
frame_id=frame.id,
success=pred is not None and pred.fitness > 0.85,
pos_err=np.linalg.norm(pred.T[:3,3] - gt_pose[:3,3]) if pred else None,
rot_err=quat_angle(pred.T[:3,:3], gt_pose[:3,:3]) if pred else None,
latency_ms=pred.latency_ms))
df = pd.DataFrame(results)
print(df.describe())
df.to_parquet(f"reports/reloc_{datetime.now():%Y%m%d}.parquet")
save_markdown_report(df)
```
---
## 13.8 持续监控（Grafana / 飞书报表）
将 `logs/metrics.parquet` 接入 Grafana，监控板包含：
| Panel | 数据源 | 阈值 |
|-------|--------|------|
| 重定位成功率 (7d) | reloc.parquet | < 90% 报警 |
| Delta confirm 率 | consolidation.log | 10–30% 正常 |
| LTM 健康分 | health.parquet | < 70 报警 |
| Anchor 失活数 | health.parquet | 周增 > 3 提醒重扫 |
| Jetson 负载 | tegrastats.csv | CPU > 80% 持续 5 min 报警 |
| 任务成功率 (7d) | tasks.parquet | < 85% 报警 |
| MTBF | crash.log | < 24 h 报警 |
---
## 13.9 A/B 测试框架
当要试新算法（如把 CLIP 换 SigLIP），先 A/B：
```python
# tools/prism_ab.py
@click.command()
@click.option("--variant-a", required=True) # 配置文件 A
@click.option("--variant-b", required=True)
@click.option("--bench", required=True)
def main(variant_a, variant_b, bench):
res_a = run_eval_with_config(variant_a, bench)
res_b = run_eval_with_config(variant_b, bench)
print(compare(res_a, res_b))
significance_test(res_a, res_b)
```
只有 A/B 显著优于现状才合并 main。
---
## 13.10 失效案例库 (Failure Case Repo)
每次发现失败，按下表归档：
```
failures/
├── F0023_reloc_failed_in_303_at_dusk/
│ ├── description.md
│ ├── frames/ # 失败时的 RGB+depth
│ ├── ltm_snapshot.tar.zst
│ ├── log.txt
│ └── root_cause.md
```
每月把这一目录跑一遍当回归测试，确保已修复的不再回归。
---
## 13.11 关键报告模板
```markdown
# PRISM Weekly Report — Week 23
## Layer Health
- L1: 30.0 Hz ✓ ; P95 latency 32 ms ✓
- L2: Chamfer 2.4 cm ✓ ; no_update_zone hits 0 ✓
- L3: 路径成功率 99.6% ✓
- L4: mAP 71.3% ✓ ; CLIP@1 86% ✓
## Pipelines
- Reloc: success 96.4%, mean pos_err 7.8 cm ✓
- Online: 30 Hz稳定; CPU 38% / GPU 47% ✓
- Consolidate (3 runs): applied 412, rejected 1.3k, rollback 0 ✓
## System
- Health Score: 88 (last week 86) ↑
- 72 h smoke: ✓
- SLO breach: none
## Tasks
- Fetch: 92/100 ✓
- QA: Top-1 87%, latency 0.6 s ✓
## Open Risks
- R1.3 卫生间 305 镜面 noise 仍偏高，待修
- R6.1 房间 308 anchor 仅 1，下次重扫优先级 P1
## Next Week
- 部署 DINOv2 重排（A/B）
- 修复 R1.3
```
---
## 13.12 与 [`09_roadmap.md`](09_roadmap.md) Gate 的强绑定
每个 Gate 都要跑相应 eval 才能通过：
| Gate | 必跑 eval |
|------|-----------|
| W2 | `prism eval ingest` |
| W4 | `prism eval reloc` |
| W6 | `prism eval online` + 4h soak |
| W7 | `prism eval consolidate` + rollback test |
| W8 | `prism eval all` |
| M4 | 72 h `prism eval all` |
| M6 | 全套 + 公开 benchmark 对比 |
---
## 13.13 本章小结
| 评测层级 | 关键指标 | 工具 |
|----------|----------|------|
| Layer | Chamfer / mAP / CLIP@1 | 逐层脚本 |
| Pipeline | Success rate / latency / coverage | `prism eval X` |
| System | Health Score / MTBF / SLO | Grafana |
| Task | Fetch success / QA Top-1 | `prism eval all` |
读完本章你应能：
- ✅ 一行命令出周报
- ✅ 设置 SLO 与报警阈值
- ✅ 把"是否要合并某 PR"建立在数据而非感觉上
下一章 [`14_mvp.md`](14_mvp.md) 把 PRISM 浓缩成 **5 步最小可复现 demo**，让任何工程师周末就能跑通核心闭环。
---
**章节版本**：v1.0
**估计阅读时间**：12 分钟
**关键收获**：建立可量化、可监控、可回归的评测体系
plans/PRISM/14_mvp.md
+538
View File
@@ -0,0 +1,538 @@
# Chapter 14 — 最小可复现 demo (MVP)
> 本章目标：把 PRISM 浓缩成 **5 个核心步骤**，让任何具备基础工程能力的开发者，**用一个周末** 就能跑通"iPhone 扫描 → ZED 重定位 → 实时感知 → 一次记忆事件 → 简单查询"的全闭环。
---
## 14.1 MVP 的边界
MVP **是什么**：
- ✅ 1 个房间
- ✅ iPhone + ZED 2i + 一台带 GPU 的笔记本（或 Jetson）
- ✅ 跑通"建图 → 重定位 → 在线感知 → 记一条变化 → LLM 查询"
- ✅ 全部依赖均开源或可白嫖
MVP **不是什么**：
- ❌ 不要求 30 Hz 稳定
- ❌ 不要求多房间
- ❌ 不要求 Consolidate（手工触发即可）
- ❌ 不要求 3DGS（用 mesh 足够）
- ❌ 不要求 ROS 2（可用纯 Python）
---
## 14.2 准备清单（30 分钟）
```yaml
硬件:
- iPhone Pro (12 Pro 起，建议 15 Pro Max)
- ZED 2i (借一台也行)
- 一台 GPU 笔记本/台式（RTX 3060+ 即可）
- 30 cm × 30 cm 打印的 ArUco DICT_5X5_100 id=42
- 一卷胶带（贴 ArUco 到地面）
软件:
- Python 3.10 + conda env
- PyTorch 2.3 (CUDA 12)
- 装包:
pip install pyzed open3d open_clip_torch ultralytics \
trimesh networkx opencv-contrib-python click
iPhone App:
- Polycam (App Store 免费版即可) 或
- 3D Scanner App (导出 USDZ 模式)
场地:
- 一个 ~15 m² 的房间，光线均匀，无大面镜子
- 床/桌/椅 至少 3 件家具
```
---
## 14.3 Step 1：iPhone 扫描（15 分钟）
```
1. 把 ArUco 贴在房间地面正中（朝向不重要，记住即可）
2. 打开 Polycam，选 "Room (LiDAR)" 模式
3. 从门口开始顺时针环绕一圈，速度 0.3 m/s
4. 重点区域回扫（床、桌、椅）
5. 结束并导出：选 "USDZ"
6. AirDrop / scp 到笔记本：保存为 mvp/scan/room.usdz
```
同时拍一张能看见 ArUco 的 RGB（用 iPhone 标准相机），保存为 `mvp/scan/aruco_ref.jpg`。
> **没有 RoomPlan App 也没关系**：Polycam 导出 USDZ 即可。我们退化版的解析器（见 14.4）从 USDZ 直接读 mesh，按高度切层得到地面 polygon。
---
## 14.4 Step 2：Ingest 到 SpatialMemory（20 分钟）
```python
# mvp/ingest.py
"""
极简版 ingest：从 Polycam 导出的 USDZ 读 mesh，
按 z=0~1.5m 切片做 2D 房间 polygon；
用 YOLO-World 离线检测家具；
对齐 ArUco 得到 T_iphone→map。
"""
import open3d as o3d
import numpy as np
import cv2, json, uuid
from pathlib import Path
from PIL import Image
import torch, open_clip
from ultralytics import YOLO
SCAN_DIR = Path("mvp/scan")
OUT_DIR = Path("mvp/ltm")
OUT_DIR.mkdir(exist_ok=True, parents=True)
# 1) 读 USDZ → mesh（trimesh 能解 usdz；或先 unzip）
import trimesh
mesh = trimesh.load(SCAN_DIR/"room.usdz", force="mesh")
print(f"mesh: {len(mesh.vertices)} verts, {len(mesh.faces)} faces")
mesh.export(OUT_DIR/"room.glb")
# 2) ArUco 对齐
img = cv2.imread(str(SCAN_DIR/"aruco_ref.jpg"))
aruco = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_5X5_100)
corners, ids, _ = cv2.aruco.detectMarkers(img, aruco)
assert ids is not None and 42 in ids.flatten(), "ArUco 42 not found!"
# 用 iPhone EXIF 内参做 solvePnP（精度无所谓，MVP 只要不歪）
fx = fy = 1500.0; cx, cy = img.shape[1]/2, img.shape[0]/2
K = np.array([[fx,0,cx],[0,fy,cy],[0,0,1]])
obj = np.array([[0,0,0],[0.3,0,0],[0.3,0.3,0],[0,0.3,0]], dtype=np.float32)
i = list(ids.flatten()).index(42)
ok, rvec, tvec = cv2.solvePnP(obj, corners[i][0], K, None)
R_cm = cv2.Rodrigues(rvec)[0]
T_cam_to_map = np.eye(4); T_cam_to_map[:3,:3]=R_cm; T_cam_to_map[:3,3]=tvec.flatten()
# Polycam 没给 ARKit pose，简化：把 T_iphone→map 直接当 T_cam_to_map
T_iphone_to_map = T_cam_to_map
mesh.apply_transform(T_iphone_to_map)
mesh.export(OUT_DIR/"room_aligned.glb")
# 3) 切层得到房间 polygon（z 0.05–0.15m 上的占用 → footprint）
pts = mesh.sample(200_000)
floor_mask = (pts[:,2] > 0.05) & (pts[:,2] < 0.15)
xy = pts[floor_mask][:, :2]
# 用 alpha-shape 或 convex hull 都行
from scipy.spatial import ConvexHull
hull = ConvexHull(xy)
polygon = xy[hull.vertices]
print("room polygon vertices:", len(polygon))
# 4) YOLO-World 在 mesh 渲染图上检家具
import pyrender
scene = pyrender.Scene()
scene.add(pyrender.Mesh.from_trimesh(mesh))
cam = pyrender.PerspectiveCamera(yfov=np.pi/3)
# 在房间上方 2.5 m 多视角渲染
furniture = []
yolo = YOLO("yolov8x-worldv2.pt")
yolo.set_classes(["bed","chair","desk","sofa","table","tv",
"lamp","wardrobe","bathtub","toilet"])
for yaw in np.linspace(0, 2*np.pi, 8, endpoint=False):
cam_pose = np.eye(4)
cam_pose[:3,3] = [np.cos(yaw)*0.1, np.sin(yaw)*0.1, 1.6]
cam_pose[:3,:3] = look_at_yaw(yaw)
scene.add(cam, pose=cam_pose)
r = pyrender.OffscreenRenderer(640, 480)
color, depth = r.render(scene)
for det in yolo.predict(color, verbose=False)[0].boxes:
u,v,w,h = det.xywh[0].cpu().numpy()
# 用 depth 把 2D 中心反投到 3D
z = depth[int(v), int(u)]
if z <= 0: continue
x = (u - 320) * z / 525
y = (v - 240) * z / 525
center = (cam_pose @ np.array([x,y,z,1]))[:3]
furniture.append({
"uid": f"{yolo.names[int(det.cls)]}_{uuid.uuid4().hex[:6]}",
"label": yolo.names[int(det.cls)],
"center": center.tolist(),
"conf": float(det.conf)})
scene.clear_camera_nodes(); r.delete()
# 5) CLIP 房间向量
clip_model, _, prep = open_clip.create_model_and_transforms("ViT-B-32")
clip_model.eval().cuda()
room_rgb = Image.open(SCAN_DIR/"aruco_ref.jpg")
with torch.no_grad():
room_feat = clip_model.encode_image(
prep(room_rgb).unsqueeze(0).cuda()).cpu().numpy()[0]
room_feat /= np.linalg.norm(room_feat)
# 6) 落盘
mem = {
"schema_version": "1.0.0",
"world_frame": "map",
"nodes": {
"room_001": {
"uid":"room_001", "label":"Room", "level":"L3",
"source":"iphone", "polygon_2d": polygon.tolist(),
"clip_embedding": room_feat.tolist(),
"confidence": 0.9}
},
"anchors": [],
"edges": []
}
for f in furniture:
uid = f["uid"]
mem["nodes"][uid] = {
"uid": uid, "label": f["label"], "level":"L4",
"source":"iphone", "category":"furniture",
"pose": {"position": f["center"], "quaternion":[1,0,0,0]},
"confidence": f["conf"], "parent_room":"room_001",
"attributes":{"mobile": f["label"] in ("chair",)},
}
mem["edges"].append({"src_uid":"room_001","dst_uid":uid,
"relation":"contains"})
if not mem["nodes"][uid]["attributes"]["mobile"]:
mem["anchors"].append({"anchor_uid": uid, "node_label": f["label"]})
with open(OUT_DIR/"spatial_memory.json","w") as f:
json.dump(mem, f, indent=2)
print(f"✅ LTM written: {len(mem['nodes'])} nodes, "
f"{len(mem['anchors'])} anchors")
```
运行：
```bash
python mvp/ingest.py
```
期望输出：`✅ LTM written: 5–10 nodes, 2+ anchors`。
---
## 14.5 Step 3：ZED 重定位（30 分钟）
```python
# mvp/relocalize.py
"""极简两段式重定位"""
import pyzed.sl as sl, numpy as np, open3d as o3d, json
import open_clip, torch
from PIL import Image
mem = json.load(open("mvp/ltm/spatial_memory.json"))
room_feat = np.array(mem["nodes"]["room_001"]["clip_embedding"])
room_feat /= np.linalg.norm(room_feat)
mesh = o3d.io.read_triangle_mesh("mvp/ltm/room_aligned.glb")
tgt_pc = mesh.sample_points_uniformly(80_000).voxel_down_sample(0.05)
tgt_pc.estimate_normals()
# ZED 启动
cam = sl.Camera()
init = sl.InitParameters()
init.depth_mode = sl.DEPTH_MODE.QUALITY
init.coordinate_units = sl.UNIT.METER
init.coordinate_system = sl.COORDINATE_SYSTEM.RIGHT_HANDED_Z_UP
cam.open(init)
rt = sl.RuntimeParameters()
img_mat = sl.Mat(); depth_mat = sl.Mat(); pc_mat = sl.Mat()
# 抓 1 帧
assert cam.grab(rt) == sl.ERROR_CODE.SUCCESS
cam.retrieve_image(img_mat, sl.VIEW.LEFT)
cam.retrieve_measure(pc_mat, sl.MEASURE.XYZRGBA)
rgb = img_mat.get_data()[:,:,:3]
pc_xyz = pc_mat.get_data()[:,:,:3].reshape(-1, 3)
pc_xyz = pc_xyz[~np.isnan(pc_xyz).any(axis=1)]
# Stage 1: CLIP 验证（MVP 只有 1 个房间，跳过 Top-K）
clip_model, _, prep = open_clip.create_model_and_transforms("ViT-B-32")
clip_model.eval().cuda()
with torch.no_grad():
f = clip_model.encode_image(
prep(Image.fromarray(rgb)).unsqueeze(0).cuda()).cpu().numpy()[0]
f /= np.linalg.norm(f)
sim = float(np.dot(f, room_feat))
print(f"[Stage1] CLIP sim={sim:.3f}", "✓" if sim>0.3 else "✗")
# Stage 2: ICP
src = o3d.geometry.PointCloud()
src.points = o3d.utility.Vector3dVector(pc_xyz)
src = src.voxel_down_sample(0.05); src.estimate_normals()
src_fpfh = o3d.pipelines.registration.compute_fpfh_feature(
src, o3d.geometry.KDTreeSearchParamHybrid(0.25, 100))
tgt_fpfh = o3d.pipelines.registration.compute_fpfh_feature(
tgt_pc, o3d.geometry.KDTreeSearchParamHybrid(0.25, 100))
result = o3d.pipelines.registration.registration_ransac_based_on_feature_matching(
src, tgt_pc, src_fpfh, tgt_fpfh, True, 0.075,
o3d.pipelines.registration.TransformationEstimationPointToPoint(False),
4, [o3d.pipelines.registration.CorrespondenceCheckerBasedOnDistance(0.075)],
o3d.pipelines.registration.RANSACConvergenceCriteria(100_000, 0.999))
icp = o3d.pipelines.registration.registration_icp(
src, tgt_pc, 0.02, result.transformation,
o3d.pipelines.registration.TransformationEstimationPointToPlane())
print(f"[Stage2] ICP fitness={icp.fitness:.3f}, RMSE={icp.inlier_rmse:.4f}")
np.save("mvp/T_zed_to_map.npy", icp.transformation)
print(f"✅ T_zed→map saved")
cam.close()
```
运行：
```bash
python mvp/relocalize.py
```
期望：`fitness > 0.5`，RMSE < 5 cm。若不达标，多试几次（换站位）。
---
## 14.6 Step 4：在线感知 + 一次差异事件（30 分钟）
```python
# mvp/online.py
"""跑 60 秒：YOLO 检测家具 → 与 LTM 关联 → 把"新物"写 delta"""
import pyzed.sl as sl, numpy as np, json, time
from ultralytics import YOLO
from pathlib import Path
T = np.load("mvp/T_zed_to_map.npy")
mem = json.load(open("mvp/ltm/spatial_memory.json"))
delta_path = Path("mvp/delta/pending.jsonl"); delta_path.parent.mkdir(exist_ok=True)
yolo = YOLO("yolov8x-worldv2.pt")
all_labels = list(set(n["label"] for n in mem["nodes"].values()
if n["level"]=="L4")) + ["cup","book","backpack"]
yolo.set_classes(all_labels)
cam = sl.Camera()
init = sl.InitParameters()
init.depth_mode = sl.DEPTH_MODE.QUALITY
init.coordinate_units = sl.UNIT.METER
init.coordinate_system = sl.COORDINATE_SYSTEM.RIGHT_HANDED_Z_UP
cam.open(init)
cam.enable_positional_tracking(sl.PositionalTrackingParameters())
rt = sl.RuntimeParameters()
img_mat = sl.Mat(); pose = sl.Pose(); xyz_mat = sl.Mat()
t0 = time.time()
events = {}
while time.time() - t0 < 60:
if cam.grab(rt) != sl.ERROR_CODE.SUCCESS: continue
cam.retrieve_image(img_mat, sl.VIEW.LEFT)
cam.retrieve_measure(xyz_mat, sl.MEASURE.XYZRGBA)
cam.get_position(pose, sl.REFERENCE_FRAME.WORLD)
rgb = img_mat.get_data()[:,:,:3].copy()
# 当前 ZED 帧位姿 → map
T_robot_zed = pose.pose_data().numpy()
T_robot_map = T @ T_robot_zed
for det in yolo.predict(rgb, conf=0.4, verbose=False)[0].boxes:
label = yolo.names[int(det.cls)]
u,v,w,h = det.xywh[0].cpu().numpy()
z = xyz_mat.get_data()[int(v), int(u), 2]
if np.isnan(z) or z <= 0: continue
# 简化：把像素中心投回 3D 后乘 T_robot_map
# （这里跳过相机内参，使用 ZED 的 XYZ 直接读）
xyz_cam = xyz_mat.get_data()[int(v), int(u), :3]
if np.isnan(xyz_cam).any(): continue
xyz_map = (T_robot_map @ np.array([*xyz_cam, 1]))[:3]
# 找 LTM 中同 label 最近节点
cand = [n for n in mem["nodes"].values()
if n.get("label")==label and n["level"]=="L4"]
if not cand:
# 新物
sig = f"new|{label}|{tuple(np.round(xyz_map/0.3).astype(int))}"
ev = events.setdefault(sig, dict(event_type="object_added",
label=label, pos=xyz_map.tolist(),
count=0))
ev["count"] += 1
else:
nearest = min(cand, key=lambda n: np.linalg.norm(
np.array(n["pose"]["position"]) - xyz_map))
d = np.linalg.norm(np.array(nearest["pose"]["position"]) - xyz_map)
if d > 0.3: # 偏移 > 30 cm，记 moved
sig = f"moved|{nearest['uid']}"
ev = events.setdefault(sig, dict(event_type="object_moved",
target=nearest["uid"],
new_pos=xyz_map.tolist(),
count=0))
ev["count"] += 1
cam.close()
# 落盘累计观测 ≥ 3 的事件
with open(delta_path, "w") as f:
for sig, ev in events.items():
if ev["count"] >= 3:
f.write(json.dumps(ev) + "\n")
print(f"✅ wrote {sum(1 for e in events.values() if e['count']>=3)} delta events")
```
测试方法：**故意拿一个新物（如水杯）放到桌上**，跑 `online.py`，应看到至少 1 条 `object_added`。
---
## 14.7 Step 5：LLM 查询（10 分钟）
```python
# mvp/query.py
"""用 OpenAI / Qwen 调 PRISM API"""
import json
from openai import OpenAI
mem = json.load(open("mvp/ltm/spatial_memory.json"))
delta = [json.loads(l) for l in open("mvp/delta/pending.jsonl")]
# 把 LTM 序列化成自然语言
def describe_memory():
desc = ["The robot has memorized the following objects in this room:"]
for n in mem["nodes"].values():
if n["level"] == "L4":
p = n["pose"]["position"]
desc.append(f"- {n['label']} at ({p[0]:.2f}, {p[1]:.2f}, {p[2]:.2f})")
if delta:
desc.append("\nRecent changes detected:")
for ev in delta:
desc.append(f"- {ev['event_type']}: {json.dumps(ev)}")
return "\n".join(desc)
client = OpenAI() # 也可换 Qwen 的 base_url
prompt = f"""You are a service robot's memory assistant.
{describe_memory()}
User asks: "桌上现在有什么不在原始扫描里的东西？"
Answer in Chinese, concise."""
resp = client.chat.completions.create(
model="gpt-4o-mini",
messages=[{"role":"user","content": prompt}])
print(resp.choices[0].message.content)
```
期望输出：`检测到一个新增的物体"cup"…`
---
## 14.8 一键运行脚本
```bash
# mvp/run_all.sh
#!/usr/bin/env bash
set -e
echo "== Step 2: ingest =="
python mvp/ingest.py
echo "== Step 3: relocalize =="
python mvp/relocalize.py
echo "== Step 4: online 60s =="
python mvp/online.py
echo "== Step 5: query =="
python mvp/query.py
echo "✅ MVP done. See mvp/ltm/ and mvp/delta/"
```
---
## 14.9 验收清单
| 阶段 | 通过标准 |
|------|----------|
| Step 1 扫描 | `mvp/scan/room.usdz` 文件存在，> 5 MB |
| Step 2 ingest | `spatial_memory.json` 含 ≥ 5 个 L4 节点 |
| Step 3 relocalize | `T_zed_to_map.npy` 存在，fitness > 0.5 |
| Step 4 online | `delta/pending.jsonl` 至少 1 条 event |
| Step 5 query | LLM 给出含"cup"或新物名称的回答 |
跑通这 5 步 → **PRISM 核心闭环验证完成**。
---
## 14.10 从 MVP 到生产的下一步
MVP 跑通后，按 [`09_roadmap.md`](09_roadmap.md) 的 W2 起逐步升级：
| MVP 简化 | 生产版本对应章节 |
|----------|------------------|
| 单房间 | [`04_pipeline_A_iphone_offline.md`](04_pipeline_A_iphone_offline.md) §4.2.3 多 session 拼接 |
| 无 RoomPlan | 替换 ingest 为 RoomPlan 解析（§4.5） |
| 单帧重定位 | [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) §5.7 在线维护 |
| YOLO 简单关联 | [`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md) §6.5 hybrid associate |
| 无 TSDF | §6.4 Open3D VoxelBlockGrid 实时融合 |
| 无差异检测 | §6.6 TSDF vs LTM 几何差异 |
| 无巩固 | [`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md) 完整 6 步 |
| 无 ROS 2 | [`10_tech_stack.md`](10_tech_stack.md) §10.5 整套部署 |
| 无评测 | [`13_evaluation.md`](13_evaluation.md) 指标 + 自动化 |
---
## 14.11 常见踩坑 (FAQ)
**Q1: ZED grab 一直失败？**
A: 检查 USB 3.0（USB-C 必须是 3.x，2.0 会带宽不足），换条线试。
**Q2: ICP fitness 永远 < 0.3？**
A: 检查：
- ZED 当前视野是否真的在房间里（不是看着门外）
- mesh 是否已 `apply_transform(T_iphone_to_map)`
- 单位是否都是米（USDZ 可能是 cm）
**Q3: ArUco 检不到？**
A: 打印尺寸要正好 30 cm（不是 A4 缩放）；侧光避免反光；正面拍摄。
**Q4: YOLO 没检出任何家具？**
A: `yolo.set_classes(...)` 必须在 `predict` 之前；conf 阈值降到 0.25 再试；图像分辨率 ≥ 640。
**Q5: Polycam 导出的 USDZ trimesh 打不开？**
A: 用 `unzip room.usdz -d room/`，找里面的 `.usdc/.usda`，再用 `pxr` 读；或在 Polycam 里改导出为 `.glb`。
**Q6: GPU 内存不够？**
A: ZED 用 720p；YOLO 换 `yolov8s-worldv2.pt`（small）；CLIP 用 `laion/CLIP-ViT-B-16-laion2B-s34B-b88K`（更省）。
**Q7: 跑 60 s `online.py` 没出任何 delta？**
A: 检查 `T_zed_to_map.npy` 是否正确加载；手动在 ZED 视野里走过，并确认新物（杯子）有连续 3 帧被检出。
---
## 14.12 把 MVP 录成 demo 视频
强烈建议跑完后录 < 3 min 视频：
```
0:00–0:30 iPhone 扫房间（手持移动）
0:30–0:45 AirDrop 文件到笔记本
0:45–1:15 python ingest.py（看 LTM 节点输出）
1:15–1:45 把 ZED 放进房间，跑 relocalize.py
1:45–2:15 放一个杯子到桌上，跑 online.py（60 s）
2:15–2:45 跑 query.py，看 LLM 中文给出"杯子是新出现的"
2:45–3:00 打开 spatial_memory.json + delta/pending.jsonl 截图
```
这段视频就是你拿给老板 / 合作方 / 审稿人看的 PRISM **第一版 proof-of-concept**。
---
## 14.13 本章小结
| MVP 的 5 步 | 一句话 |
|------------|--------|
| **Step 1** | iPhone + Polycam 扫一个房间，导 USDZ |
| **Step 2** | 解析 USDZ + ArUco 对齐 → spatial_memory.json |
| **Step 3** | ZED 拍当前帧 → CLIP + ICP → T_zed_to_map |
| **Step 4** | YOLO 跑 60 s → 检测到新物，写 delta |
| **Step 5** | LLM 读 LTM + delta → 自然语言回答 |
**MVP 用时**：一个工程师 1–2 个工作日。
**硬件成本**：iPhone (借) + ZED 2i (借) + 自有 GPU 笔记本，**0–4 k 元**。
读完本章你应能：
- ✅ 今天就开工
- ✅ 周五前给团队演示 PRISM 核心闭环
- ✅ 评估"是否值得投入 8 周做完整原型"
---
**章节版本**：v1.0
**估计阅读时间**：15 分钟（不含动手时间）
**关键收获**：把 14 章理论压缩成"一个周末 + 5 个 Python 脚本"
---
> 🌈 **跑完 MVP 的那一刻，你就拥有了 PRISM v0.01——一个会用 iPhone 当教科书、用 ZED 当眼睛、还会自己记笔记的小机器人大脑。剩下 8 周路线图（[`09_roadmap.md`](09_roadmap.md)）就是把它从"小学生"变成"上岗员工"。**
plans/PRISM/15_glossary.md
+190
View File
@@ -0,0 +1,190 @@
# Chapter 15 — 缩略语表 (Glossary)
> 本章目标:全书所有缩略语和专有术语的统一速查表,按 **A–Z** 排序。每条给出**全称**、**中文释义**、**首次/重点出现章节**。
\begin{longtable}{@{}p{2.6cm}p{6cm}p{6.4cm}@{}}
\toprule
\textbf{缩写} & \textbf{全称 / 全名} & \textbf{中文释义 / 出现章节} \\
\midrule
\endfirsthead
\toprule
\textbf{缩写} & \textbf{全称 / 全名} & \textbf{中文释义 / 出现章节} \\
\midrule
\endhead
\bottomrule
\endfoot
\multicolumn{3}{@{}l}{\textbf{\large A — 项目核心}} \\
\midrule
\textbf{PRISM} & Prior-Registered Integrated Spatial Memory & 本项目代号:先验配准集成空间记忆 / 全书 \\
\textbf{L1} & Perceptual Buffer & 感知缓冲层(ms 级,ZED 主写) / 02 \\
\textbf{L2} & Metric Layer & 度量层(TSDF/3DGS/OctoMap) / 02, 04, 06 \\
\textbf{L3} & Topological Layer & 拓扑层(房间节点+边+锚点) / 02, 04 \\
\textbf{L4} & Semantic Layer & 语义层(场景图,Neo4j) / 02, 04 \\
\textbf{LTM} & Long-Term Memory & 长期记忆(iPhone 主写,近乎只读) / 02, 04, 07 \\
\textbf{STM} & Short-Term Memory & 短期记忆(运行时环形覆盖) / 03, 06 \\
\textbf{WM} & Working Memory & 工作记忆(等同 L1) / 02, 06 \\
\textbf{Anchor} & Relocalization Anchor & 重定位锚点 / 04, 05 \\
\textbf{Delta} & Delta / DeltaEvent & 差异记忆事件(待巩固) / 06, 07 \\
\textbf{Consolidator} & --- & 记忆巩固器(充电时跑) / 07 \\
\textbf{Registrar} & --- & 重定位握手器(CLIP+ICP) / 00, 05 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large B — 传感器与采集}} \\
\midrule
\textbf{ARKit} & Apple AR Kit & 苹果 AR 框架 / 04 \\
\textbf{ArUco} & ArUco fiducial marker & 方形二维码地标(坐标对齐) / 04, 08 \\
\textbf{AprilTag} & AprilTag fiducial marker & 另一种方形地标(同上) / 04, 12 \\
\textbf{IMU} & Inertial Measurement Unit & 惯性测量单元 / 03, 05, 06 \\
\textbf{LiDAR} & Light Detection and Ranging & 激光雷达 / 04, 10 \\
\textbf{RGB-D} & Color + Depth & 彩色+深度传感器 / 06, 10 \\
\textbf{RoomPlan} & Apple RoomPlan & 苹果室内结构化扫描 API / 04 \\
\textbf{ZED 2i} & Stereolabs ZED 2i & 双目+IMU 视觉惯性相机 / 全书 \\
\textbf{ZED SDK} & ZED Software Development Kit & ZED 官方驱动 / 03, 06, 10 \\
\textbf{VIO} & Visual-Inertial Odometry & 视觉惯性里程计 / 05, 06 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large C — SLAM 与几何}} \\
\midrule
\textbf{SLAM} & Simultaneous Localization and Mapping & 同步定位与建图 / 02, 10 \\
\textbf{TSDF} & Truncated Signed Distance Function & 截断符号距离场(稠密体素) / 02, 06 \\
\textbf{3DGS} & 3D Gaussian Splatting & 三维高斯泼溅(可微体渲染) / 02, 06, 10 \\
\textbf{NeRF} & Neural Radiance Fields & 神经辐射场 / 10, 11 \\
\textbf{OctoMap} & Octree-based Occupancy Map & 八叉树占据栅格 / 02, 10 \\
\textbf{OcTree} & Octree & 八叉树数据结构 / 02 \\
\textbf{SDF} & Signed Distance Function & 符号距离场 / 02 \\
\textbf{ICP} & Iterative Closest Point & 迭代最近点配准 / 05 \\
\textbf{RANSAC} & RANdom SAmple Consensus & 随机采样一致 / 05, 10 \\
\textbf{FPFH} & Fast Point Feature Histograms & 快速点特征直方图(配准描述子) / 05, 08 \\
\textbf{ORB} & Oriented FAST and Rotated BRIEF & 旋转鲁棒特征点 / 02, 10 \\
\textbf{SuperPoint} & SuperPoint & 学习型关键点 / 02 \\
\textbf{TEASER++} & Truncated least squares Estimation And SEmidefinite Relaxation & 鲁棒全局配准库 / 05, 10 \\
\textbf{TF} & ROS Transform Tree & ROS 坐标变换树 / 05, 10 \\
\textbf{TUM} & TUM trajectory format & 慕尼黑工业大学轨迹文本格式 / 04 \\
\textbf{SE(3)} & Special Euclidean group, 3D & 三维刚体变换群 / 03 \\
\textbf{DoF} & Degrees of Freedom & 自由度 / 05 \\
\textbf{RMSE} & Root Mean Squared Error & 均方根误差 / 13 \\
\textbf{EMA} & Exponential Moving Average & 指数移动平均(在线平滑) / 06, 08 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large D — 感知与 VLM/LLM}} \\
\midrule
\textbf{CLIP} & Contrastive Language-Image Pre-training & 图文对比预训练模型(向量检索) / 05, 10 \\
\textbf{DINO / DINOv2} & Self-DIstillation with NO labels v2 & Meta 自监督视觉骨干 / 05, 10 \\
\textbf{SigLIP} & Sigmoid-loss Language-Image Pre-training & Google 改进版 CLIP / 10 \\
\textbf{YOLO} & You Only Look Once & 单阶段目标检测器 / 06, 10 \\
\textbf{YOLO-World} & YOLO-World open-vocabulary detector & 开放词表 YOLO / 06, 10 \\
\textbf{SAM} & Segment Anything Model & Meta 通用分割模型 / 10 \\
\textbf{SAM 2} & Segment Anything Model 2 & SAM 第二代(支持视频) / 10 \\
\textbf{Grounding-DINO} & --- & 文本到 bbox 检测器 / 10 \\
\textbf{VLM} & Vision-Language Model & 视觉语言模型 / 06, 11 \\
\textbf{LLM} & Large Language Model & 大语言模型 / 00, 11 \\
\textbf{JEPA} & Joint-Embedding Predictive Architecture & Meta 联合嵌入预测架构 / 11 \\
\textbf{M-JEPA} & Meta-JEPA & Meta 出品 JEPA 系列 / 11 \\
\textbf{RSSM} & Recurrent State-Space Model & 循环状态空间模型(Dreamer 核心) / 11 \\
\textbf{DreamerV3} & --- & 基于 RSSM 的世界模型 / 11 \\
\textbf{TD-MPC2} & Temporal Difference Model-Predictive Control v2 & 模型预测控制 v2 / 11 \\
\textbf{GPT-4o} & GPT-4 omni & OpenAI 多模态模型 / 10 \\
\textbf{Qwen-VL} & Qwen Vision-Language & 阿里多模态模型 / 10 \\
\textbf{CoT} & Chain-of-Thought & 思维链提示 / 11 \\
\textbf{SayCan} & --- & Google 语言+可执行性规划 / 11 \\
\textbf{NetVLAD} & --- & 可学习的位置识别向量 / 10 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large E — 中间件 / ROS / 部署}} \\
\midrule
\textbf{ROS / ROS 2} & Robot Operating System & 机器人操作系统(Humble) / 10 \\
\textbf{RViz / RViz2} & ROS Visualization & ROS 可视化工具 / 04, 10 \\
\textbf{tf2} & ROS tf2 & ROS 2 坐标变换库 / 10 \\
\textbf{Nav2} & ROS 2 Navigation Stack & ROS 2 导航栈 / 08 \\
\textbf{Isaac ROS} & NVIDIA Isaac ROS & NVIDIA Robotics ROS 包 / 10 \\
\textbf{nvblox} & NVIDIA nvblox & GPU TSDF 库 / 02, 10 \\
\textbf{Open3D} & Open3D & 3D 几何与点云库 / 04, 05, 10 \\
\textbf{Jetson} & NVIDIA Jetson & 嵌入式 GPU 计算平台 / 10 \\
\textbf{Orin AGX} & Jetson Orin AGX & 高端 Jetson 模组 / 10 \\
\textbf{TensorRT} & TensorRT & NVIDIA 推理优化引擎 / 10 \\
\textbf{CUDA} & Compute Unified Device Architecture & NVIDIA GPU 计算平台 / 10 \\
\textbf{Docker} & Docker & 容器化部署 / 10 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large F — 数据格式 / 存储 / 协议}} \\
\midrule
\textbf{USDZ} & Universal Scene Description Zipped & 苹果场景文件格式 / 04, 10 \\
\textbf{USD} & Universal Scene Description & 通用场景描述 / 10 \\
\textbf{glTF / GLB} & GL Transmission Format & WebGL 通用 3D 格式 / 03 \\
\textbf{PLY} & Polygon File Format & 点云/网格文件 / 03 \\
\textbf{JSON} & JavaScript Object Notation & 通用对象序列化 / 03 \\
\textbf{JSONL} & JSON Lines & 每行一个 JSON 对象 / 06, 07 \\
\textbf{YAML} & YAML Ain't Markup Language & 配置文件格式 / 10 \\
\textbf{HDF5} & Hierarchical Data Format 5 & 大规模数值数据 / 04, 10 \\
\textbf{Faiss} & Facebook AI Similarity Search & 向量索引库 / 10 \\
\textbf{Neo4j} & --- & 图数据库 / 02, 10 \\
\textbf{SQLite} & --- & 嵌入式 SQL 数据库 / 10 \\
\textbf{TuGraph} & --- & 蚂蚁开源图数据库(备选) / 10 \\
\textbf{rosbag} & ROS bag & ROS 录制数据包(.bag) / 13 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large G — 评测与运维}} \\
\midrule
\textbf{KPI} & Key Performance Indicator & 关键绩效指标 / 08, 13 \\
\textbf{SOTA} & State-of-the-Art & 业界最佳 / 13 \\
\textbf{MVP} & Minimum Viable Product & 最小可行产品 / 14 \\
\textbf{MTBF} & Mean Time Between Failures & 平均故障间隔 / 13 \\
\textbf{SOP} & Standard Operating Procedure & 标准作业流程 / 04 \\
\textbf{SLO} & Service-Level Objective & 服务等级目标 / 13 \\
\textbf{QA} & Question Answering & 问答(评测任务) / 13 \\
\textbf{OOM} & Out Of Memory & 内存溢出 / 12 \\
\textbf{E2E} & End-to-End & 端到端 / 08, 13 \\
\textbf{GT} & Ground Truth & 真值 / 13 \\
\textbf{COCO} & Common Objects in COntext & 常见物体检测数据集 / 10 \\
\textbf{ScanNet} & --- & 室内 3D 扫描评测基准 / 13 \\
\textbf{HM3D-Semantic} & Habitat-Matterport 3D, Semantic & 大规模室内语义评测基准 / 13 \\
\textbf{OpenEQA} & Open Embodied Question Answering & 开放具身问答评测 / 13 \\
\textbf{HotelScene-Bench} & --- & 本项目自建酒店评测集 / 13 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large H — 法律 / 许可 / 隐私}} \\
\midrule
\textbf{AGPL / AGPLv3} & GNU Affero General Public License v3 & 强 copyleft 许可证 / 10, 12 \\
\textbf{BSD} & Berkeley Software Distribution license & 宽松许可证 / 10 \\
\textbf{MIT} & MIT License & 宽松许可证 / 10 \\
\textbf{Apache 2.0} & Apache License 2.0 & 宽松许可证(含专利条款) / 10 \\
\textbf{EULA} & End-User License Agreement & 最终用户许可协议(NVIDIA 等) / 10 \\
\textbf{GDPR} & General Data Protection Regulation & 欧盟通用数据保护条例 / 12 \\
\textbf{PSF} & Python Software Foundation & Python 软件基金会(许可) / 10 \\
\midrule
\multicolumn{3}{@{}l}{\textbf{\large I — 数学 / 工具 / 其他常见}} \\
\midrule
\textbf{API} & Application Programming Interface & 应用程序接口 / 全书 \\
\textbf{CLI} & Command-Line Interface & 命令行接口 / 10, 14 \\
\textbf{SDK} & Software Development Kit & 软件开发工具包 / 04, 10 \\
\textbf{CI} & Continuous Integration & 持续集成 / 13 \\
\textbf{CPU / GPU} & Central / Graphics Processing Unit & 中央 / 图形处理器 / 10 \\
\textbf{MPS} & Metal Performance Shaders & 苹果 GPU 加速后端 / 10 \\
\textbf{FP16 / INT8} & Float-16 / Integer-8 & 低精度推理量化 / 10 \\
\textbf{DBSCAN} & Density-Based Spatial Clustering of Applications with Noise & 密度聚类算法 / 07 \\
\textbf{KDTree} & K-Dimensional Tree & K 维树(最近邻查询) / 05 \\
\textbf{NetworkX} & --- & Python 图算法库 / 10 \\
\textbf{Trimesh} & --- & Python 网格库 / 10 \\
\textbf{pypdf} & --- & Python PDF 操作库(本书构建用) / 工具链 \\
\textbf{pandoc} & --- & 通用文档转换器(本书构建用) / 工具链 \\
\textbf{XeLaTeX} & --- & 支持 Unicode 的 LaTeX 引擎 / 工具链 \\
\textbf{mmdc} & mermaid CLI & Mermaid 命令行工具(本书图渲染) / 工具链 \\
\end{longtable}
---
## 15.1 阅读约定补充
- **"灌入"** 指 iPhone 离线扫描数据一次性写入 LTM(见 [`04_pipeline_A_iphone_offline.md`](04_pipeline_A_iphone_offline.md))。
- **"握手"** 指 ZED 上线后通过两段式重定位与 LTM 对齐(见 [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md))。
- **"巩固"** 指充电时把 `delta/` 中的事件筛选后应用到 LTM(见 [`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md))。
- 章节编号引用形如"04, 05"表示该缩略语在第 04、05 章首次或重点出现。
---
**章节版本**:v1.0
**最后更新**:2026-05-16
**关键收获**:全书一表通查;新成员入项目第一周读完即可上手。
plans/PRISM/16_stats.md
+98
View File
@@ -0,0 +1,98 @@
# Chapter 16 — 文档与代码统计 (Stats)
> 本章自动生成,反映本 PRISM 文档集在**构建时**的体量、结构与代码资产。每次 `bash build_book.sh` 都会刷新。
## 16.1 总览
\begin{center}
\begin{tabular}{@{}lr@{}}
\toprule
\textbf{指标} & \textbf{数值} \\
\midrule
章节数 & 18 \\
中文字数 & 25,171 \\
英文词数 & 5,515 \\
标点数 & 4,212 \\
Markdown 行数 & 7,021 \\
一级标题 (\#) & 18 \\
二级标题 (\#\#) & 194 \\
三级标题 (\#\#\#) & 177 \\
Markdown 表格行数 & 966 \\
Mermaid 图块数 & 23 \\
代码块数(非 mermaid) & 132 \\
代码总行数 & 2,716 \\
\bottomrule
\end{tabular}
\end{center}
## 16.2 各章字数与代码分布
\begin{longtable}{@{}
>{\raggedright\arraybackslash}p{0.28\linewidth}
>{\raggedleft\arraybackslash}p{0.10\linewidth}
>{\raggedleft\arraybackslash}p{0.10\linewidth}
>{\raggedleft\arraybackslash}p{0.10\linewidth}
>{\raggedleft\arraybackslash}p{0.10\linewidth}
>{\raggedleft\arraybackslash}p{0.10\linewidth}
>{\raggedleft\arraybackslash}p{0.10\linewidth}
@{}}
\toprule
\textbf{章节} & \textbf{中文字} & \textbf{英文词} & \textbf{md 行} & \textbf{mmd 图} & \textbf{代码块} & \textbf{代码行} \\
\midrule
\endfirsthead
\toprule
\textbf{章节} & \textbf{中文字} & \textbf{英文词} & \textbf{md 行} & \textbf{mmd 图} & \textbf{代码块} & \textbf{代码行} \\
\midrule
\endhead
\bottomrule
\endfoot
README & 750 & 246 & 131 & 1 & 1 & 4 \\
ch00 & 1,311 & 139 & 173 & 1 & 0 & 0 \\
ch01 & 1,139 & 146 & 244 & 1 & 3 & 100 \\
ch02 & 2,381 & 347 & 493 & 4 & 8 & 124 \\
ch03 & 1,312 & 281 & 654 & 0 & 14 & 424 \\
ch04 & 1,101 & 277 & 662 & 2 & 14 & 406 \\
ch05 & 1,028 & 117 & 368 & 2 & 8 & 166 \\
ch06 & 1,376 & 306 & 550 & 1 & 14 & 305 \\
ch07 & 1,925 & 268 & 551 & 2 & 11 & 256 \\
ch08 & 617 & 106 & 483 & 3 & 25 & 175 \\
ch09 & 1,820 & 508 & 326 & 1 & 1 & 17 \\
ch10 & 1,045 & 427 & 384 & 1 & 5 & 146 \\
ch11 & 1,126 & 265 & 266 & 1 & 7 & 62 \\
ch12 & 2,063 & 382 & 358 & 2 & 0 & 0 \\
ch13 & 1,042 & 249 & 394 & 0 & 8 & 109 \\
ch14 & 957 & 251 & 537 & 0 & 10 & 338 \\
ch15 & 1,085 & 742 & 190 & 0 & 0 & 0 \\
ch18 & 3,093 & 458 & 257 & 1 & 3 & 84 \\
\midrule\textbf{合计} & \textbf{25,171} & \textbf{5,515} & \textbf{7,021} & \textbf{23} & \textbf{132} & \textbf{2,716} \\
\end{longtable}
> 章节短名:`ch00` = `00\_overview.md`,依此类推;`ch15` 缩略语表,`ch16` 即本章。
## 16.3 代码块语言分布
\begin{center}
\begin{tabular}{@{}lrr@{}}
\toprule
\textbf{语言} & \textbf{代码块数} & \textbf{代码行数} \\
\midrule
python & 71 & 1,961 \\
text & 36 & 308 \\
yaml & 7 & 236 \\
bash & 11 & 74 \\
swift & 1 & 41 \\
json & 2 & 37 \\
markdown & 1 & 28 \\
cypher & 2 & 17 \\
dockerfile & 1 & 14 \\
\midrule
\textbf{合计} & \textbf{132} & \textbf{2,716} \\
\bottomrule
\end{tabular}
\end{center}
---
**章节版本**:auto-generated
**生成时间**:由 `gen_stats.py` 在每次构建时即时计算
**关键收获**:文档不只是文字,它本身也是一份可量化的工程产物。
plans/PRISM/17_changelog.md
+134
View File
@@ -0,0 +1,134 @@
# Chapter 17 — 变更日志 (Changelog)
> 本章直接镜像项目根的 [`CHANGELOG.md`](CHANGELOG.md);**当前版本 v1.5.0, 构建于 2026-05-17**。 每次发版时由 [`tools/gen_changelog_chapter.py`](tools/gen_changelog_chapter.py) 自动同步到 PDF。
## [1.5.0] — 2026-05-16
### Added (新增)
- **第 18 章 `18_lyra_inspirations.md`** — Lyra 2.0 启发的 3 大设计原则
(路由 vs 内容分离、keyframe-based 内容真源、self-augmentation 巩固)。
- **§ 2.7b L2-L3 数据流:路由 vs 内容** —— 在 [`02_architecture.md`](02_architecture.md)
新增小节,阐明 L2 几何降级为"路由信号",L3 内容来自 L1 高保真 keyframe。
- **§ 3.2.1 `KeyframeEvidence` dataclass** —— 在 [`03_data_schema.md`](03_data_schema.md)
新增字段定义、存储预算估算与 `SpatialNode` 的向后兼容扩展。
- **§ 6.6b v1.5 新写法:keyframe-based L3 内容更新** —— 在
[`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md)
新增 keyframe 采样、压缩与 L3 写入流程。
- **§ 7.13.1 Self-Augmentation 巩固训练算法** —— 在
[`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md)
新增训练范式 (随机视角扰动 + 重渲染监督, $p_{\text{aug}}=0.7$)。
- **同档对比综述 [`comparison.md`](comparison.md)** —— 379 行,8 项目对比
(ConceptGraphs / HOV-SG / OK-Robot / Clio / Hydra / SemanticFusion / Voxblox++ / OpenScene)
+ 4 个 Pipeline 维度矩阵。
- **评分文档** —— [`rate.md`](rate.md) 技术难度 62/100、
[`rate_industry.md`](rate_industry.md) 行业贡献 48/100。
- **Lyra 2.0 论文译读** —— [`research/lyra2_review.md`](../../research/lyra2_review.md)
作为本次升级的外部依据。
### Changed (变更)
- **L2 几何在 L3 写入中降级为"路由信号"**:L3 内容不再由 L2 voxel 反推几何/语义,
而是直接挂载 L1 高保真 keyframe 作为唯一真源。
- **`SpatialNode` 新增 `keyframe_evidence: List[KeyframeEvidence]` 字段**
(向后兼容,默认空列表,旧 schema 数据无须迁移)。
- **巩固期训练范式增加 self-augmentation**:$p_{\text{aug}}=0.7$
的概率对采样视角施加随机扰动,以重渲染 keyframe 作监督信号,提升 L4 语义稳健性。
### References (引用与启发)
- 上述设计借鉴 NVIDIA *Lyra 2.0: Explorable Generative 3D Worlds*
(arXiv:2604.13036),§ 3.2(routing-by-geometry)与 § 3.3(keyframe-augmented training)。
- 工程模板对标:**ConceptGraphs** (ICRA 2024)、**HOV-SG** (RSS 2024)、
**OK-Robot** (FAIR 2024)、**Clio** (RA-L 2024)。
---
## [1.2.0] — 2026-05-16
### Added (新增)
- **第 15 章 缩略语表 (`15_glossary.md`)**:9 个分组、约 125 条术语,A–I 主题化排列,
采用 LaTeX `longtable` + `booktabs` 跨页排版。
- **第 16 章 文档与代码统计 (`16_stats.md`)**:由 [`tools/gen_stats.py`](tools/gen_stats.py) 在每次构建时自动生成,
含总览表、各章字数/代码分布、按语言的代码块统计。
- **第 17 章 变更日志 (`17_changelog.md`)**:由 [`tools/gen_changelog_chapter.py`](tools/gen_changelog_chapter.py) 从本 `CHANGELOG.md` 自动注入 PDF。
- **版本管理**:新增 [`VERSION`](VERSION) 文件作为单一权威版本号,封面/页眉/PDF 元数据自动注入。
- **构建工具链**:
- [`tools/render_mermaid.py`](tools/render_mermaid.py):mmdc 把 mermaid 块预渲染为 PNG,带 hash 缓存与失败降级。
- [`tools/gen_stats.py`](tools/gen_stats.py):构建期统计脚本。
- [`tools/gen_changelog_chapter.py`](tools/gen_changelog_chapter.py):本 CHANGELOG → 章节 md 转换。
### Changed (变更)
- **全部 ASCII 框线图 → Mermaid 图 (21 个)**:
- flowchart 14 个 (00, 02, 04, 05, 06, 07, 10, 11, README)
- quadrantChart 2 个 (01 能力象限、12 风险矩阵)
- timeline / sequenceDiagram / stateDiagram 各 1 (08 章)
- gantt 1 个 (09 路线图)
- **构建管线 [`build_book.sh`](tools/build_book.sh)** 新增 4 个阶段:
- `1a` 生成统计章节
- `1b` mermaid → PNG 预渲染
- **后处理 LaTeX**:把 pandoc 默认的 `@{}lll@{}` 自然宽 longtable 自动改写为等宽 `p{w}` 列,
消除大量 `Overfull hbox` 警告
- **LaTeX 模板 [`book_template.tex`](tools/book_template.tex)**:
- 启用 `\XeTeXlinebreaklocale "zh"` 让 CJK 字符可逐字断行
- 引入 `fvextra` 让代码块自动换行 (`breaklines=true, breakanywhere=true`)
- 引入 `xurl` 让长 URL 任意位置断开
- 引入 `hyphenat[htt]` 让 monospace (`\texttt{...}`) 长下划线文件名可断行
- `\sloppy + \emergencystretch=3em` 全局抗溢出
- **Mermaid 图嵌入策略**:`\includegraphics` 同时约束 `width=0.92\linewidth, height=0.82\textheight, keepaspectratio`,
极端长宽比的图也不会冲出 A4 页面
### Fixed (修复)
- 修复 ~13 个 emoji/特殊字符无字体显示 (`🔬 🧑 💼 → ↔ ① ✓` 等),
在 [`build_book.sh`](tools/build_book.sh) 的 `EMOJI_MAP` 中扩展替换映射。
- 修复 mermaid `quadrantChart` 标签因 `()` `·` `.` 等字符报 Lexical error。
- 修复 mermaid `sequenceDiagram` label 含 `;` / `/` / `[v]` 等字符报 Parse error。
- 修复 mermaid `timeline` 单行多冒号导致解析失败,改用 `section` 分组。
- 修复极端纵向 mermaid 图溢出 A4 (高度 2700+ px) → 添加 height 约束。
- 修复代码块长注释溢出页宽 (最严重 167pt) → 用 `fvextra` 自动换行。
- 修复表格内长字符串/中文不换行溢出 → CJK locale + hyphenat 双管齐下。
- 修复第 16 章统计表 7 列窄列溢出 → 使用 28% + 6×10% 等宽列布局,章节名用短形式 `ch00`–`ch16`。
### Stats (统计)
- 章节总数: **18** (README + 17 章)
- 中文字数: **~19,000**
- 代码行数: **~2,500** (Python 1,750 + 其他)
- Mermaid 图数: **21**
- PDF 页数: **165** (含封面)
---
## [1.1.0] — 2026-05-09
### Added
- 拆分 `robot_brain_spatial_memory.md` (主方案 v1.0) 为 14 个独立章节 (`00_overview.md` ~ `14_mvp.md`),
目录化为 `plans/PRISM/`。
- 增加章节级元信息 (版本/估计阅读时间/关键收获)。
- 引入 [`tools/build_book.sh`](tools/build_book.sh) + [`tools/book_template.tex`](tools/book_template.tex) 出版级 PDF 构建工具链
(pandoc → XeLaTeX → pypdf 合并 cover)。
- 添加 ArUco 公共原点对齐方案 (R1.2 缓解)。
- 添加 LLM Agent 任务接口示例 (T3 取遥控器剧本)。
### Changed
- L2 默认稠密表示从纯 OctoMap 改为 **OctoMap + TSDF + 3DGS** 三件套。
- 重定位策略从一段式 ICP 改为 **CLIP 粗匹配 + ICP 精配准** 两段式。
### Fixed
- 修复多 session 间坐标系不一致问题 (强制 ArUco 兜底)。
---
## [1.0.0] — 2026-04-22
### Added
- PRISM 主方案首版 (`robot_brain_spatial_memory.md`):
- 四层空间记忆架构 L1–L4
- 两路输入 (iPhone RoomPlan 离线 + ZED 2i 在线)
- 四条管线 (A 离线建图 / B 重定位 / C 在线感知 / D 巩固)
- `SpatialNode` / `SpatialEdge` / `SpatialMemory` 统一 schema
---
## [Unreleased]
### Planned (待规划)
- v1.3:接入 M-JEPA / DreamerV3 的桥接代码 (目前仅为接口设计)
- v1.3:扩展评测体系到 HM3D-Semantic / OpenEQA 公开 benchmark
- v1.3:多机器人协同的初步设计
- v2.0:原型实现 (从文档到能跑的 ROS 2 节点)
plans/PRISM/18_lyra_inspirations.md
+258
View File
@@ -0,0 +1,258 @@
# Chapter 18 — Lyra 2.0 启发的设计原则
> 本章目标：把 Lyra 2.0（Self-Distillation Bootstraps 3D World Models from 2D Video Diffusion, 2025）译读 [`research/lyra2_review.md`](../../research/lyra2_review.md) 第 8.4 节归纳出的 **3 条核心思想** 正式纳入 PRISM 设计哲学——分别约束 **L2 ↔ L3 ↔ L4 的写入语义**、**SpatialMemory 的证据保留策略** 与 **Pipeline D 的巩固训练范式**。本章是 PRISM v1.5.0 的新增内容。
---
## 18.1 章节目的
Lyra 2.0 是 2025 年 NVIDIA 用 2D 视频扩散模型"自蒸馏"出一个 3D 世界模型的工作；它的"3D 缓存"在工程上和 PRISM 的"空间记忆"是**同类问题**：都需要在多帧观测之间保留 3D 一致性、又都必须容忍**几何精度不够好**这一事实。Lyra 没有 SLAM、没有 TSDF，却用三个看似反直觉的工程决策——**几何只做路由、per-frame 缓存绝不融合、巩固期用自己的不完美输出再训练**——把"精度不够"的几何先验稳稳压在了"够用"的工作点上。这些决策与 PRISM 当下的痛点（L2 漂移、L3 写入污染、Pipeline D 缺少自监督信号）高度对应，因此值得作为**独立章节**沉淀下来，避免在后续 v1.x 演进中被遗忘或被工程惯性悄悄抹去。本章按"原则陈述 → 工程对应 → 风险约束"的顺序组织：每条原则先讲 Lyra 怎么做、再讲 PRISM 该怎么对应，最后在 § 18.6 集中说明哪些地方**不**能照搬。更深的技术细节请回看 [`research/lyra2_review.md`](../../research/lyra2_review.md) § 3.2（3D 缓存设计）与 § 3.3（self-augmentation 公式）。
---
## 18.2 原则一：几何只做路由，不做合成
### 18.2.1 问题描述
PRISM v1.4 之前的写入逻辑里，L2 既要"提供占据栅格供避障"，又要"把局部几何投到 L3 节点的子图里当内容"。这意味着 **L2 的几何误差会污染 L3 的内容**——例如 TSDF 在长走廊里漂了 30 cm，L3 中"走廊节点"挂的那块点云也跟着歪 30 cm，后续 L4 的 bbox 估计也会跟着歪。
Lyra 2.0 § 3.2(a) 的做法给出了一个干净的解耦：**3D 缓存只用来"挑选哪一帧 keyframe 该被注意到"，真正的内容（颜色、纹理、语义）由原始 2D 帧的 cross-attention 提供**。换言之，几何只回答"路由问题"（who should write where），不回答"内容问题"（what to write）。
### 18.2.2 PRISM 旧逻辑 vs PRISM 新逻辑
```mermaid
flowchart TB
subgraph OLD["PRISM v1.4 旧：L2 既路由又写 L3"]
direction LR
O_L1["L1 keyframe<br/>(rgb+depth)"]
O_L2["L2 TSDF/Octomap"]
O_L3["L3 节点子图"]
O_L1 --> O_L2
O_L2 -- "投影点云<br/>(内容)" --> O_L3
O_L2 -- "路由<br/>(选哪个节点)" --> O_L3
end
subgraph NEW["PRISM v1.5 新：L2 只路由,L3 内容来自 L1"]
direction LR
N_L1["L1 keyframe<br/>(rgb+depth)"]
N_L2["L2 TSDF/Octomap<br/>(几何精度允许低)"]
N_L3["L3 节点子图"]
N_L1 --> N_L2
N_L2 -- "路由<br/>(only: select node)" --> N_L3
N_L1 -. "内容<br/>(keyframe ref)" .-> N_L3
end
style OLD fill:#fde2e2,stroke:#a33
style NEW fill:#d4f0d4,stroke:#2e7d32
```
关键差异：**新逻辑下 L3 节点存的不是"L2 几何投影出来的点云"，而是"指向 L1 keyframe 的引用 + L2 给出的路由决策"**。L2 即便漂 30 cm，只要它能正确挑出"这一帧属于走廊节点而不是卧室节点"，下游就不受影响——内容由 L1 原始观测兜底。
### 18.2.3 伪代码
```python
# prism/write/route_l3.py (v1.5 新增)
def route_and_write_l3(
keyframe: L1Keyframe, # 原始观测：rgb + depth + pose
l2: MetricLayer, # 可能有漂移的 TSDF/Octomap
l3: TopologicalLayer, # 节点 + 边
) -> None:
"""
几何只做路由：用 L2 决定 keyframe 该挂到哪个 L3 节点；
L3 的"内容"始终是 keyframe 的引用,不复制几何。
"""
# ---- 步骤 1: 用 L2 做路由（几何精度低也无妨） ----
pose_xy = keyframe.pose.translation[:2]
candidate_nodes = l2.query_room_id(pose_xy) # 返回若干候选节点
node_id = l3.disambiguate(candidate_nodes, keyframe) # 用 keyframe 视觉再确认
# ---- 步骤 2: L3 只写引用 + 路由元数据，不写几何 ----
l3.nodes[node_id].keyframe_refs.append(keyframe.id)
l3.nodes[node_id].routing_log.append({
"kf_id": keyframe.id,
"l2_pose": pose_xy.tolist(),
"l2_uncert": l2.local_uncertainty(pose_xy), # 路由置信度
"ts": keyframe.ts,
})
# ---- 步骤 3: 内容查询时，按需从 L1 读回 ----
# （L3.get_content(node_id) 实现里只做 L1Keyframe 的解引用，
# 不缓存任何 L2 投影点云。这是与旧版的根本区别。）
```
设计含义：**L2 漂移不再是 L3 的故障，而只是 L3 路由的噪声**——可以通过 § 18.3 的 keyframe 独立证据来事后翻案，而不必反向修补 L2。另一个常被忽视的好处是：当未来引入 NeRF / 3DGS 之类的精细几何表示时，它们也只是 L3 的"另一种内容引用"，**不会与 L2 抢"几何真相代言人"的位置**——L2 始终只代表"路由用的粗几何"，L3 内容由谁提供、提供得多精细，是写入策略的自由度而非架构约束。这一点对应 Lyra § 3.2(d) 中 canonical coordinates **只参与 Q/K 不参与 V** 的设计：粗坐标只用于"挑出谁该被注意到（Q/K 路由）"，但模型实际看到的内容（V）来自原始 2D 帧。
详见对 [`plans/PRISM/03_data_schema.md`](03_data_schema.md) 与 [`plans/PRISM/06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md) 的影响（§ 18.5 总表）。
---
## 18.3 原则二：Per-frame 独立 keyframe 证据，而非全局融合
### 18.3.1 问题描述
PRISM 现行的 L2 是**融合表示**：TSDF 把每一帧深度都"加权累计"到体素里、OctoMap 把每一帧 ray 都更新到八叉树占据概率上。融合带来两个好处（紧凑、平滑）和一个致命缺点：**一旦累计了漂移，原始证据就不可逆地丢了**——你拿不回"第 t=12.3 s 那一帧 ZED 看到桌子在哪"，因为它已经被融进了 240 万个体素的加权平均里。
Lyra 2.0 § 3.2(a) 把这点写得很明白："**3D 缓存绝不融合（never fuse）**"——每个 keyframe 是一条独立的、可追溯的证据，模型在 cross-attention 时挑出最相关的几条 keyframe，而不是查询一个"已经融合过的全局体素"。
### 18.3.2 三种策略对比
| 维度 | OctoMap-only（PRISM v1.0） | TSDF-only（PRISM v1.3） | Lyra-style per-frame | **PRISM v1.5 选择** |
|------|---|---|---|---|
| 表示 | 八叉树占据概率 | 体素 SDF | keyframe 列表（pose + depth + 特征） | **TSDF（路由）+ keyframe 列表（证据）** |
| 漂移可逆 | ✗（已融合） | ✗（已融合） | ✓（每帧独立） | ✓（keyframe 兜底） |
| 内存 | O(体素数) | O(体素数) | O(帧数 × 单帧大小) | **O(体素 + 关键帧子集)** |
| 避障查询 | 快（直接查体素） | 中 | 慢（要做近邻） | **快（用 TSDF）** |
| "翻案"能力 | 无 | 无 | 强 | **强（保留 keyframe）** |
| 写入语义 | 累加 | 加权融合 | append-only | **append-only on keyframes** |
PRISM v1.5 不是把 TSDF 扔掉，而是**让 TSDF 和 keyframe 列表并存**：TSDF 负责 § 18.2 的"路由 + 避障"两个对几何精度要求不高的任务；keyframe 列表负责"翻案"——当 L4 发现某个语义节点和 L3 节点对不齐时，可以回退到 keyframe 重新估计 bbox。
### 18.3.3 SpatialMemory 新增字段 schema
```python
# prism/schema/spatial_memory.py (v1.5 新增字段)
@dataclass
class KeyframeEvidence:
"""
每个 L1 keyframe 在巩固后保留的"独立证据"。
一律 append-only,绝不被 L2 融合操作覆盖。
"""
kf_id: str # 全局唯一,匹配 L1.frames[i].id
ts: float # 采集时间戳 (epoch sec)
pose: SE3 # 该帧位姿 (世界系)
pose_uncert: np.ndarray # 6x6 协方差 (路由置信度的来源)
depth_ref: str # 指向 L1 存储的深度图 blob (lazy load)
feat_ref: str # 指向 L1 存储的视觉特征 blob (DINO/CLIP)
routed_l3: Optional[str] # 当时路由到的 L3 节点 id (可空)
routed_l4: List[str] # 当时关联的 L4 实体 id 列表
fused_into_l2: bool = False # 是否曾参与 L2 TSDF 融合
immutable: bool = True # 一律 True；写入后不可被覆盖
@dataclass
class SpatialMemory:
l1: L1Buffer
l2: MetricLayer
l3: TopologicalLayer
l4: SemanticGraph
# ---- v1.5 新增 ----
keyframe_evidence: List[KeyframeEvidence] # append-only,巩固期挑选关键帧入库
```
实现要点：
1. **append-only**：`KeyframeEvidence` 一旦写入即 `immutable=True`，禁止任何 pipeline 反向修改；
2. **lazy load**：`depth_ref / feat_ref` 只存路径，原始张量留在 L1 的 zarr/parquet 里，避免内存爆掉；
3. **不是全部 keyframe 都进**：Pipeline D 巩固期会按"信息增益"挑一个子集（如每 0.5 m 或每 30° 视角变化保留一帧），典型规模 200–2000 帧/场景。
详见 § 18.5 对 [`plans/PRISM/03_data_schema.md`](03_data_schema.md) 的影响。
---
## 18.4 原则三：巩固期的 Self-Augmentation
### 18.4.1 问题描述
Pipeline D（巩固期，离线 / 夜间）目前的逻辑是：拿白天采集的 L1 keyframe，运行更慢更准的离线管线（高分辨率 depth、bundle adjustment、全局优化），把结果写回 L3/L4。这个过程**没有自监督信号**——它假设"离线版的输出就是 ground-truth"，但实际工程中离线版本身也会犯错，且模型从不学会"识别并纠正自己之前的错"。
Lyra 2.0 § 3.3(b) 给了一个简洁的 self-augmentation 公式：**用模型自己之前的不完美输出作为输入，用干净的 ground-truth 作为监督**。具体地，对训练样本的潜变量 $z$，以概率 $p_{\text{aug}}=0.7$ 做扰动：
$$
\tilde z_0 = z_t - t \cdot v_\theta(z_t, t, c), \quad t \sim \mathcal{U}(0, 0.5)
$$
其中 $v_\theta$ 是当前正在训练的流匹配模型自己。这相当于让模型把"自己一步去噪后的结果"当成新输入，再去预测真值——模型在训练中**反复见到自己会犯的错**，因此学会了自我纠错。
### 18.4.2 在 PRISM 巩固期的对应
把上面的范式翻译到 PRISM Pipeline D：
- **z**：当前场景的 L3/L4 表示（拓扑节点 + 场景图 bbox）；
- **不完美输入 $\tilde z_0$**：用 Pipeline C 在线版（实时但粗）的 L3/L4 输出；
- **ground-truth 监督**：用 Pipeline A 离线版（慢但准）的 L3/L4 输出。
模型（这里指 L3/L4 的精炼网络，未来 v2.0 的研究方向）学会的不是"从零产生 L3/L4"，而是"**从一份带错的 L3/L4 + 当时的 keyframe 证据**，产生干净的 L3/L4"——这恰好就是"巩固"的本质。
### 18.4.3 伪代码
```python
# prism/pipeline_d/self_aug.py (v1.5 新增,实际实现见 v2.0 路线图)
def consolidate_with_self_aug(
online_l3l4: SceneRepr, # Pipeline C 的不完美输出
offline_l3l4: SceneRepr, # Pipeline A 的"较干净"输出 (监督信号)
kf_evidence: List[KeyframeEvidence],
refiner: RefinerNet, # 待训练的精炼网络
p_aug: float = 0.7,
t_max: float = 0.5,
) -> RefinerNet:
"""
巩固期 self-augmentation:
以 p_aug 概率用模型自己产生的"伪不完美样本"替换 online_l3l4,
用 offline_l3l4 作为干净监督。
"""
for batch in iterate_scenes():
# ---- 1. 决定本步是否使用 self-aug ----
if np.random.rand() < p_aug:
# 用模型自己生成一个"扰动版"的输入
t = np.random.uniform(0.0, t_max)
z_clean = encode(offline_l3l4) # 干净潜变量
z_t = add_flow_noise(z_clean, t) # 沿流方向加噪
v_pred = refiner.predict_velocity(z_t, t, kf_evidence)
z_tilde = z_t - t * v_pred # 模型一步去噪结果
x_in = decode(z_tilde) # 解回 L3/L4 表示
else:
# 1 - p_aug = 0.3 概率用真实的"在线不完美输出"
x_in = online_l3l4
# ---- 2. 统一的监督：始终向 offline 版对齐 ----
x_pred = refiner(x_in, kf_evidence) # keyframe 证据可被检索
loss = scene_repr_loss(x_pred, offline_l3l4)
loss.backward(); refiner.step()
return refiner
```
### 18.4.4 关于 $p_{\text{aug}} = 0.7$ 的取值讨论
Lyra 2.0 实验里取 $p_{\text{aug}}=0.7$ 而非 0.5 或 1.0，有两个工程意涵值得我们沿用：
1. **0.7 > 0.5 表示模型主要见到的是"自己生成的扰动样本"**——这是它真正会犯的错；如果 $p$ 太低（如 0.3），模型大部分时间在做"干净到干净"的精炼，巩固期就退化成普通监督学习，失去自我纠错能力。
2. **0.7 < 1.0 留了 30% 真实的"在线不完美样本"**——这部分确保模型不会沉迷于自己的失败模式，仍然能见到 Pipeline C 在真实工况下产生的、分布与 self-aug 不一样的错（如 ZED 在暗光下的深度毛刺，模型自己生成的样本里不一定包含）。
经验上：当 Pipeline C 已经相当稳定（错误率低且类型集中）时，可降到 0.5；当 Pipeline C 仍在快速演化、错误模式多样时，可推到 0.8。PRISM v1.5 起步采用 **0.7**，作为后续 ablation 的中点。
$t \sim \mathcal{U}(0, 0.5)$ 的含义是：只在流匹配的"前半程"加噪——避免把样本扰动到几乎全噪声，那样监督信号会失效。
详见 § 18.5 对 [`plans/PRISM/07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md) 的影响。
---
## 18.5 与 PRISM 现有章节的关系
本章的三条原则并不是"另起炉灶"，而是**约束既有章节在 v1.5 及之后的演进方向**。下表列出影响面：
| 本章原则 | 影响的现有章节 | 具体影响内容 | 回链 |
|---|---|---|---|
| § 18.2 几何只做路由 | Chapter 02 架构 | L2 ↔ L3 边的语义从"投影内容 + 路由"收窄为"只做路由" | [`plans/PRISM/02_architecture.md`](02_architecture.md) |
| § 18.2 几何只做路由 | Chapter 03 数据 schema | `L3Node.content` 从"几何点云"改为"keyframe 引用列表" | [`plans/PRISM/03_data_schema.md`](03_data_schema.md) |
| § 18.2 几何只做路由 | Chapter 06 Pipeline C 在线 | `route_and_write_l3()` 替换原"投影 + 写内容"两步 | [`plans/PRISM/06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md) |
| § 18.3 keyframe 独立证据 | Chapter 03 数据 schema | 新增 `SpatialMemory.keyframe_evidence: List[KeyframeEvidence]` 字段 | [`plans/PRISM/03_data_schema.md`](03_data_schema.md) |
| § 18.3 keyframe 独立证据 | Chapter 13 评估 | 新增"翻案能力"指标：被 keyframe 推翻的 L4 bbox 占比 | [`plans/PRISM/13_evaluation.md`](13_evaluation.md) |
| § 18.4 self-augmentation | Chapter 07 Pipeline D 巩固 | 巩固期训练范式从"监督学习"升级为"self-aug 监督学习" | [`plans/PRISM/07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md) |
| § 18.4 self-augmentation | Chapter 13 评估 | 新增"巩固后纠错率"指标：online → offline 之间被纠正的错误数 | [`plans/PRISM/13_evaluation.md`](13_evaluation.md) |
上述章节本次 v1.5.0 升级**暂不强制改动**——本章先以"设计原则"形式独立存在；后续 v1.6 / v2.0 在演进对应章节时，再逐条把约束落地。
---
## 18.6 风险与限制
本章列出的三条原则源自 Lyra 2.0 的工程实践，但 Lyra 与 PRISM 在**问题设定上有根本差异**，因此必须把"借鉴"与"复制"严格区分：
1. **Lyra 是生成式，PRISM 是判别式 + 真实传感器**。Lyra 的"3D 缓存"服务于像素生成（视频扩散解码），它的几何粗糙是可以容忍的，因为下游 cross-attention 会用 2D 帧把内容补回；PRISM 的 L2 服务于**机器人避障**，几何粗糙的容忍度有刚性下限（栅格分辨率不能粗于半个机器人宽度）。**§ 18.2"几何只做路由"不能被滥用为"几何精度可以任意低"**——它只是说"L2 精度不要影响 L3 内容"，并不是说"L2 精度可以任意差到避障也失败"。
2. **Lyra 的训练数据是合成的（视频扩散先验），PRISM 的数据是真实 ZED / iPhone**。Self-augmentation 在 Lyra 里之所以好用，部分是因为合成数据的"分布"和模型 capacity 是匹配的；PRISM 的真实数据带有显著的传感器噪声（ZED 在反光面、低纹理面会大面积失败），把模型自己生成的样本当训练源，**可能把传感器特有的失败模式当成"正常分布"学下来**。§ 18.4 中保留 30% 真实在线样本（$1 - p_{\text{aug}} = 0.3$）就是为了对冲这个风险，但仍需要在 Pipeline D 的实施中加 ablation 验证。
3. **per-frame 证据保留有存储代价**。Lyra 的 keyframe 数量受限于扩散模型的 context 长度（数十帧）；PRISM 在长期部署中一个酒店场景可能积累数万帧。§ 18.3 的"保留子集"策略（每 0.5 m 或每 30° 视角一帧）是经验值，**长期边界尚未被验证**——可能需要 v2.0 引入"keyframe 退役 / 压缩"机制，否则会成为新的内存瓶颈。
4. **原则之间也存在张力**。"§ 18.2 只用 L2 路由"要求 L2 几何只承担轻任务；"§ 18.3 保留 keyframe 翻案"要求事后能纠正 L4——但**到底何时触发翻案？由谁判定 L4 出错？** 当前章节未给出闭环规则，留给 [`plans/PRISM/13_evaluation.md`](13_evaluation.md) 在新增"翻案能力"指标时配套定义。
综上，本章是 PRISM 在 v1.5 节点上对 Lyra 2.0 的**有节制的吸收**：原则被写下来，但落地节奏由后续章节按版本逐步推进，确保每一条都经过实验回归而非仅凭直觉。一个简明的检验标准是：v1.6 在改动 [`plans/PRISM/03_data_schema.md`](03_data_schema.md) 引入 `keyframe_evidence` 字段时，必须同步在 [`plans/PRISM/13_evaluation.md`](13_evaluation.md) 加入"翻案命中率"与"翻案误报率"两项指标——只有这两个数能稳定测出来，§ 18.3 的原则才算真正落地；否则就只是"看上去更优雅"的架构装饰。Self-aug 的 $p_{\text{aug}}$ 与 § 18.2 路由置信度阈值同理，必须以**可被 ablation 推翻的形式**写进评估章节，本章才算完成它的设计哲学使命。
---
**章节版本**：v1.0（新增于 PRISM v1.5.0）
**估计阅读时间**：8–10 分钟
**关键收获**：几何只做路由 / 不做合成；keyframe 独立证据；巩固期 self-aug。
plans/PRISM/19_v2_brain_inspired_upgrade.md
+1261
View File
File diff suppressed because it is too large Load Diff
plans/PRISM/CHANGELOG.md
+167
View File
@@ -0,0 +1,167 @@
# Changelog
> PRISM 项目变更日志。遵循 [Keep a Changelog](https://keepachangelog.com/zh-CN/1.1.0/) 风格,版本号遵循 [SemVer](https://semver.org/lang/zh-CN/) (MAJOR.MINOR.PATCH)。
>
> - **MAJOR** (1.x→2.x): 架构层面不向后兼容的改动 (例如 schema 变更、四层模型改动)
> - **MINOR** (1.0→1.1): 新增章节/管线/特性,向后兼容
> - **PATCH** (1.1.0→1.1.1): 拼写、排版、bug 修复
---
## [2.0.0] — 2026-05-17
### Added (新增)
- **第 19 章 `19_v2_brain_inspired_upgrade.md`** — PRISM 2.0 仿脑空间记忆架构升级
(11 个仿脑模块：GridMetric / DG Orthogonalizer / PredictiveLayer / CA3 Verifier /
ExplicitReplay / Reconsolidation / SalienceGate / BoundaryDistance /
SuccessorRepr / LandmarkRelative / EpisodicBinding)
- **脑科学研究文档 `research/human_spatial_memory.md`** — 人类空间记忆神经科学机制综述
(位置细胞 / 网格细胞 / 头方向细胞 / 边界细胞 / 模式分离与完成 /
系统巩固 / 预测编码 / 再巩固 / 继任表征 / 物体向量细胞)
- **P0 模块**: GridMetric (网格细胞式跨环境度量泛化) + DG 正交化 (模式分离) + PredictiveLayer (预测编码)
- **P1 模块**: CA3 双确认 + ExplicitReplay + Reconsolidation + SalienceGate + BoundaryDistance
- **P2 模块**: SuccessorRepr + LandmarkRelative + EpisodicBinding
- 四层架构更新为包含 GridMetric 层 + PredictiveLayer
- Pipeline A/B/C/D 全部更新为 v2.0 兼容流程
- README 新增 🧠 阅读路径
### Changed (变更)
- **L2 绝对坐标度量 → GridMetric 周期性编码** (L2 仍保留作为 fallback)
- **被动写入 → 预测驱动写入** (PredictiveLayer: 只在预测失败时写入)
- **写放大降低 ~99%** (SalienceGate + PredictiveLayer 联合效应)
- **Pipeline D 一次性 batch → 多轮压缩重播巩固** (ExplicitReplay)
- **直接覆写 → 再巩固协议** (解锁→更新→重新封印)
---
## [1.5.0] — 2026-05-16
### Added (新增)
- **第 18 章 `18_lyra_inspirations.md`** — Lyra 2.0 启发的 3 大设计原则
(路由 vs 内容分离、keyframe-based 内容真源、self-augmentation 巩固)。
- **§ 2.7b L2-L3 数据流:路由 vs 内容** —— 在 [`02_architecture.md`](02_architecture.md)
新增小节,阐明 L2 几何降级为"路由信号",L3 内容来自 L1 高保真 keyframe。
- **§ 3.2.1 `KeyframeEvidence` dataclass** —— 在 [`03_data_schema.md`](03_data_schema.md)
新增字段定义、存储预算估算与 `SpatialNode` 的向后兼容扩展。
- **§ 6.6b v1.5 新写法:keyframe-based L3 内容更新** —— 在
[`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md)
新增 keyframe 采样、压缩与 L3 写入流程。
- **§ 7.13.1 Self-Augmentation 巩固训练算法** —— 在
[`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md)
新增训练范式 (随机视角扰动 + 重渲染监督, $p_{\text{aug}}=0.7$)。
- **同档对比综述 [`comparison.md`](comparison.md)** —— 379 行,8 项目对比
(ConceptGraphs / HOV-SG / OK-Robot / Clio / Hydra / SemanticFusion / Voxblox++ / OpenScene)
+ 4 个 Pipeline 维度矩阵。
- **评分文档** —— [`rate.md`](rate.md) 技术难度 62/100、
[`rate_industry.md`](rate_industry.md) 行业贡献 48/100。
- **Lyra 2.0 论文译读** —— [`research/lyra2_review.md`](../../research/lyra2_review.md)
作为本次升级的外部依据。
### Changed (变更)
- **L2 几何在 L3 写入中降级为"路由信号"**:L3 内容不再由 L2 voxel 反推几何/语义,
而是直接挂载 L1 高保真 keyframe 作为唯一真源。
- **`SpatialNode` 新增 `keyframe_evidence: List[KeyframeEvidence]` 字段**
(向后兼容,默认空列表,旧 schema 数据无须迁移)。
- **巩固期训练范式增加 self-augmentation**:$p_{\text{aug}}=0.7$
的概率对采样视角施加随机扰动,以重渲染 keyframe 作监督信号,提升 L4 语义稳健性。
### References (引用与启发)
- 上述设计借鉴 NVIDIA *Lyra 2.0: Explorable Generative 3D Worlds*
(arXiv:2604.13036),§ 3.2(routing-by-geometry)与 § 3.3(keyframe-augmented training)。
- 工程模板对标:**ConceptGraphs** (ICRA 2024)、**HOV-SG** (RSS 2024)、
**OK-Robot** (FAIR 2024)、**Clio** (RA-L 2024)。
---
## [1.2.0] — 2026-05-16
### Added (新增)
- **第 15 章 缩略语表 (`15_glossary.md`)**:9 个分组、约 125 条术语,A–I 主题化排列,
采用 LaTeX `longtable` + `booktabs` 跨页排版。
- **第 16 章 文档与代码统计 (`16_stats.md`)**:由 [`tools/gen_stats.py`](tools/gen_stats.py) 在每次构建时自动生成,
含总览表、各章字数/代码分布、按语言的代码块统计。
- **第 17 章 变更日志 (`17_changelog.md`)**:由 [`tools/gen_changelog_chapter.py`](tools/gen_changelog_chapter.py) 从本 `CHANGELOG.md` 自动注入 PDF。
- **版本管理**:新增 [`VERSION`](VERSION) 文件作为单一权威版本号,封面/页眉/PDF 元数据自动注入。
- **构建工具链**:
- [`tools/render_mermaid.py`](tools/render_mermaid.py):mmdc 把 mermaid 块预渲染为 PNG,带 hash 缓存与失败降级。
- [`tools/gen_stats.py`](tools/gen_stats.py):构建期统计脚本。
- [`tools/gen_changelog_chapter.py`](tools/gen_changelog_chapter.py):本 CHANGELOG → 章节 md 转换。
### Changed (变更)
- **全部 ASCII 框线图 → Mermaid 图 (21 个)**:
- flowchart 14 个 (00, 02, 04, 05, 06, 07, 10, 11, README)
- quadrantChart 2 个 (01 能力象限、12 风险矩阵)
- timeline / sequenceDiagram / stateDiagram 各 1 (08 章)
- gantt 1 个 (09 路线图)
- **构建管线 [`build_book.sh`](tools/build_book.sh)** 新增 4 个阶段:
- `1a` 生成统计章节
- `1b` mermaid → PNG 预渲染
- **后处理 LaTeX**:把 pandoc 默认的 `@{}lll@{}` 自然宽 longtable 自动改写为等宽 `p{w}` 列,
消除大量 `Overfull hbox` 警告
- **LaTeX 模板 [`book_template.tex`](tools/book_template.tex)**:
- 启用 `\XeTeXlinebreaklocale "zh"` 让 CJK 字符可逐字断行
- 引入 `fvextra` 让代码块自动换行 (`breaklines=true, breakanywhere=true`)
- 引入 `xurl` 让长 URL 任意位置断开
- 引入 `hyphenat[htt]` 让 monospace (`\texttt{...}`) 长下划线文件名可断行
- `\sloppy + \emergencystretch=3em` 全局抗溢出
- **Mermaid 图嵌入策略**:`\includegraphics` 同时约束 `width=0.92\linewidth, height=0.82\textheight, keepaspectratio`,
极端长宽比的图也不会冲出 A4 页面
### Fixed (修复)
- 修复 ~13 个 emoji/特殊字符无字体显示 (`🔬 🧑 💼 → ↔ ① ✓` 等),
在 [`build_book.sh`](tools/build_book.sh) 的 `EMOJI_MAP` 中扩展替换映射。
- 修复 mermaid `quadrantChart` 标签因 `()` `·` `.` 等字符报 Lexical error。
- 修复 mermaid `sequenceDiagram` label 含 `;` / `/` / `[v]` 等字符报 Parse error。
- 修复 mermaid `timeline` 单行多冒号导致解析失败,改用 `section` 分组。
- 修复极端纵向 mermaid 图溢出 A4 (高度 2700+ px) → 添加 height 约束。
- 修复代码块长注释溢出页宽 (最严重 167pt) → 用 `fvextra` 自动换行。
- 修复表格内长字符串/中文不换行溢出 → CJK locale + hyphenat 双管齐下。
- 修复第 16 章统计表 7 列窄列溢出 → 使用 28% + 6×10% 等宽列布局,章节名用短形式 `ch00`–`ch16`。
### Stats (统计)
- 章节总数: **18** (README + 17 章)
- 中文字数: **~19,000**
- 代码行数: **~2,500** (Python 1,750 + 其他)
- Mermaid 图数: **21**
- PDF 页数: **165** (含封面)
---
## [1.1.0] — 2026-05-09
### Added
- 拆分 `robot_brain_spatial_memory.md` (主方案 v1.0) 为 14 个独立章节 (`00_overview.md` ~ `14_mvp.md`),
目录化为 `plans/PRISM/`。
- 增加章节级元信息 (版本/估计阅读时间/关键收获)。
- 引入 [`tools/build_book.sh`](tools/build_book.sh) + [`tools/book_template.tex`](tools/book_template.tex) 出版级 PDF 构建工具链
(pandoc → XeLaTeX → pypdf 合并 cover)。
- 添加 ArUco 公共原点对齐方案 (R1.2 缓解)。
- 添加 LLM Agent 任务接口示例 (T3 取遥控器剧本)。
### Changed
- L2 默认稠密表示从纯 OctoMap 改为 **OctoMap + TSDF + 3DGS** 三件套。
- 重定位策略从一段式 ICP 改为 **CLIP 粗匹配 + ICP 精配准** 两段式。
### Fixed
- 修复多 session 间坐标系不一致问题 (强制 ArUco 兜底)。
---
## [1.0.0] — 2026-04-22
### Added
- PRISM 主方案首版 (`robot_brain_spatial_memory.md`):
- 四层空间记忆架构 L1–L4
- 两路输入 (iPhone RoomPlan 离线 + ZED 2i 在线)
- 四条管线 (A 离线建图 / B 重定位 / C 在线感知 / D 巩固)
- `SpatialNode` / `SpatialEdge` / `SpatialMemory` 统一 schema
---
## [Unreleased]
### Planned (待规划)
- v1.3:接入 M-JEPA / DreamerV3 的桥接代码 (目前仅为接口设计)
- v1.3:扩展评测体系到 HM3D-Semantic / OpenEQA 公开 benchmark
- v1.3:多机器人协同的初步设计
- v2.0:原型实现 (从文档到能跑的 ROS 2 节点)
plans/PRISM/README.md
+136
View File
@@ -0,0 +1,136 @@
# PRISM
## **P**rior-**R**egistered **I**ntegrated **S**patial **M**emory
### —— 一种融合 iPhone 先验地图与 ZED 2i 在线感知的机器人空间记忆架构
> 🌈 **像棱镜把白光分成七色，PRISM 把两类异构 3D 数据分到四层空间记忆里，让机器人既"看得见当下"，又"记得住过去"。**
---
## 一、命名由来
| 字母 | 含义 | 对应技术 |
|------|------|----------|
| **P** | **Prior** （先验） | iPhone RoomPlan 离线扫描得到的"建筑师图纸" |
| **R** | **Registered** （已配准） | CLIP 粗匹配 + TEASER++/ICP 把实时观测对齐到先验 |
| **I** | **Integrated** （融合） | 两路数据在统一 `SpatialMemory` Schema 中共存 |
| **S** | **Spatial** （空间） | 四层结构：感知缓冲 / 度量 / 拓扑 / 语义 |
| **M** | **Memory** （记忆） | 工作 → 短期 → 长期 + 巩固机制（仿人脑） |
**为什么是"棱镜 (Prism)"**：
- 棱镜把**一束白光**分成**多种颜色** ↔ PRISM 把**机器人采集到的原始观测**分到**四层记忆**
- 棱镜也能把**多种颜色**合回**一束**↔ 机器人查询时各层信息又汇聚为一次决策
- 一目了然、好读好搜，与 ROS/SLAM 社区已有项目不重名
---
## 二、章节导航（一章一文件）
| # | 文件 | 标题 | 一句话内容 |
|---|------|------|-----------|
| 00 | [`00_overview.md`](00_overview.md) | 总览与命名由来 | 项目目标、核心理念、读者地图 |
| 01 | [`01_capability_decomposition.md`](01_capability_decomposition.md) | 两方案能力解构 | iPhone vs ZED 2i 角色画像与互补性 |
| 02 | [`02_architecture.md`](02_architecture.md) | 四层空间记忆架构 | L1 感知 / L2 度量 / L3 拓扑 / L4 语义 |
| 03 | [`03_data_schema.md`](03_data_schema.md) | 统一数据模型 | `SpatialNode` / `SpatialEdge` / `SpatialMemory` |
| 04 | [`04_pipeline_A_iphone_offline.md`](04_pipeline_A_iphone_offline.md) | 管线 A：iPhone 离线建图 | RoomPlan → 写入 L2/L3/L4 |
| 05 | [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) | 管线 B：ZED 上线握手 | 两段式重定位（CLIP→ICP） |
| 06 | [`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md) | 管线 C：在线感知 | TSDF 增量 + 差异检测 |
| 07 | [`07_pipeline_D_consolidation.md`](07_pipeline_D_consolidation.md) | 管线 D：记忆巩固 | 充电时"睡眠"，短期转长期 |
| 08 | [`08_runtime_timeline.md`](08_runtime_timeline.md) | 完整时序 | T0→T4 端到端运行剧本 |
| 09 | [`09_roadmap.md`](09_roadmap.md) | 8 周原型路线图 | 周度交付与里程碑 |
| 10 | [`10_tech_stack.md`](10_tech_stack.md) | 技术栈 | 存储/中间件/算法库选型 |
| 11 | [`11_world_model_bridge.md`](11_world_model_bridge.md) | 衔接世界模型 | 与 M-JEPA / DreamerV3 的接口 |
| 12 | [`12_risks.md`](12_risks.md) | 风险与对策 | 6 大风险及缓解 |
| 13 | [`13_evaluation.md`](13_evaluation.md) | 评测指标 | 重定位/一致性/语义/实时性 |
| 14 | [`14_mvp.md`](14_mvp.md) | 最小可复现 demo | 5 步验证全闭环 |
| 18 | [`18_lyra_inspirations.md`](18_lyra_inspirations.md) | Lyra 2.0 启发的设计原则 | 路由/内容分离 + keyframe 真源 + self-augmentation |
| 19 | [`19_v2_brain_inspired_upgrade.md`](19_v2_brain_inspired_upgrade.md) | 🧠 PRISM 2.0：仿脑升级 | 8 个仿脑模块 (GridMetric/DG/Predictive...)，v1.5 → v2.0 |
### 配套文档与附录
| 文件 | 用途 |
|------|------|
| [`comparison.md`](comparison.md) | 同档项目对比综述(8 项目 + 4 个 Pipeline 维度矩阵) |
| [`rate.md`](rate.md) | 技术难度评分(62/100) |
| [`rate_industry.md`](rate_industry.md) | 行业贡献评分(48/100) |
| [`CHANGELOG.md`](CHANGELOG.md) | 版本变更日志(Keep-a-Changelog 风格) |
---
## 三、阅读路径建议
### 🚀 30 分钟快速理解（决策者）
`README` → [`00_overview.md`](00_overview.md) → [`01_capability_decomposition.md`](01_capability_decomposition.md) → [`02_architecture.md`](02_architecture.md) → [`08_runtime_timeline.md`](08_runtime_timeline.md)
### 💻 实施前完整阅读（工程师）
按章节顺序 00 → 14
### 🔬 技术评审（研究员）
[`02_architecture.md`](02_architecture.md) → [`05_pipeline_B_relocalization.md`](05_pipeline_B_relocalization.md) → [`06_pipeline_C_online_perception.md`](06_pipeline_C_online_perception.md) → [`11_world_model_bridge.md`](11_world_model_bridge.md) → [`13_evaluation.md`](13_evaluation.md)
### 🧠 PRISM 2.0 仿脑升级
[`19_v2_brain_inspired_upgrade.md`](19_v2_brain_inspired_upgrade.md) → [`research/human_spatial_memory.md`](../../research/human_spatial_memory.md)
### 🧑‍💼 项目立项（PM）
[`00_overview.md`](00_overview.md) → [`09_roadmap.md`](09_roadmap.md) → [`12_risks.md`](12_risks.md) → [`10_tech_stack.md`](10_tech_stack.md)
---
## 四、与上层项目的关系
```mermaid
flowchart TB
R["worldmodel/<br/><i>总项目：物理世界理解</i>"]
P["plans/"]
H["hotel_scene_implementation_plan.md<br/><i>酒店场景建模总方案</i>"]
I["iphone/<br/><i>采集方案 1</i>"]
C["camera/<br/><i>采集方案 2 (ZED 2i)</i>"]
PR["<b>PRISM/</b><br/><i>本目录：把上面两者合成机器人大脑</i>"]
RS["research/<br/><i>文献与综述</i>"]
R --> P
R --> RS
P --> H
P --> I
P --> C
P --> PR
style PR fill:#ffe9b3,stroke:#c97a00,stroke-width:2px
```
- **PRISM 是"消化层"**：上游的 iPhone 与 ZED 2i 各自负责"产数据"，PRISM 负责"把数据变成机器人能记住、能查询的记忆"。
- **PRISM 是"承上启下"**：下游可对接 M-JEPA / 大模型 Agent，作为它们的空间上下文。
---
## 五、当前进度
```
[x] v1.0 主方案（robot_brain_spatial_memory.md）
[x] v1.1 拆分章节（本目录）
[x] v1.2 各章节细化（代码 + 接口 + 实验）+ PDF 出版管线
[x] v1.5 Lyra 2.0 启发升级 (路由/内容分离 + keyframe 真源 + self-augmentation)
[x] v2.0 仿脑设计 (8 个仿脑模块：GridMetric/DG/Predictive 等)
[ ] v2.0 原型实现 (P0 优先：GridMetric + DG + PredictiveLayer)
```
---
## 六、缩写速查表
| 缩写 | 全称 | 出现章节 |
|------|------|----------|
| PRISM | Prior-Registered Integrated Spatial Memory | 全部 |
| LTM | Long-Term Memory | 02, 04, 07 |
| WM | Working Memory | 02, 06 |
| L1–L4 | 四层记忆层 | 02 |
| VIO | Visual-Inertial Odometry | 05, 06 |
| TSDF | Truncated Signed Distance Function | 02, 06 |
| 3DGS | 3D Gaussian Splatting | 02, 06, 10 |
| Anchor | 重定位锚点 | 04, 05 |
| delta/ | 差异记忆目录 | 06, 07 |
---
**文档版本**：v2.0.0
**最后更新**：2026-05-17
**反馈**：在 GitHub Issues 提出章节改进建议
> 🧠 **PRISM 2.0 = 从「精准度量 + 被动记录」到「泛化度量 + 预测驱动」。大脑用几瓦功耗做到了终身空间记忆，没有理由不抄作业。**
plans/PRISM/VERSION
+1
View File
@@ -0,0 +1 @@
2.0.0
plans/PRISM/comparison.md
+380
View File
@@ -0,0 +1,380 @@
# PRISM 同档项目对比综述
> **范围**:对 `rate_industry.md` 第 3.1 节列出的 4 个项目 (ConceptGraphs / HOV-SG / OpenScene / 3D-LLM) 做技术深读,并扩展加入 4 个**强相关项目** (OK-Robot / Clio / ConceptFusion / OpenMask3D),共 8 个对比对象。
>
> **方法**:通过 arXiv API 抓 abstract、GitHub API 抓 stars/license/活跃度、原始 README 抓硬件平台与数据集。原始数据存在 [`plans/PRISM/.research/`](.research/) 目录,可复查。
>
> **撰写日期**:2026-05-16
> **PRISM 版本**:v1.2.0(假设全部完成态)
---
## 0. TL;DR(60 秒结论)
| 维度 | 当前 SOTA | PRISM 与之差距 |
|---|---|---|
| **3D 场景图代表作** | ConceptGraphs (ICRA 2024) | 类似的双层 + 巩固机制,PRISM 多出"先验地图 + 在线感知"双源融合 |
| **分层场景图** | HOV-SG (RSS 2024) | HOV-SG 是 floor/room/object 三层;PRISM 多出 L4 语义事件层 |
| **真机大规模评测** | OK-Robot (FAIR 2024) — 10 家 170 物体 | PRISM 目前 0 真机,这是**最大差距** |
| **任务驱动 + 信息瓶颈** | Clio (RA-L 2024) | PRISM 没有信息论框架,Clio 在理论侧领先 |
| **手机扫描 → 机器人 LTM 桥** | (空白) | **PRISM 独占** — 是 PRISM 最差异化的贡献 |
| **酒店/家居垂直运维栈** | (无人专注) | PRISM 独占该垂类工程模板 |
**核心判断**:PRISM 在**学术深度**上落后 ConceptGraphs/Clio 一个身位,但在**消费级先验地图集成**与**全栈工程化**上有独特位置。如能把"iPhone RoomPlan → ROS 2 LTM"做成开源标准 + 真机部署,可与 ConceptGraphs 同档。
---
## 1. 项目速览矩阵
| 项目 | 发表 | 单位 | Stars | 许可证 | 最后更新 | 真机部署 | 数据集 |
|---|---|---|---:|---|---|---|---|
| **PRISM** (v1.2 蓝图) | — | — | — | — (待定) | 2026-05-16 | ❌ 0 真机 | HotelScene-Bench (设计中) |
| **ConceptGraphs** | ICRA 2024 | CMU + MIT + Mila | **868** | MIT | 2025-10-16 | ✅ Jackal | Replica + 自建 |
| **HOV-SG** | RSS 2024 | Freiburg + Stuttgart | **470** | MIT | 2026-01-19 | ✅ Habitat → Real | HM3DSem |
| **Clio** | RA-L 2024 | MIT-SPARK | 239 | BSD-2 | 2025-09-01 | 部分 | 自建 4 场景 |
| **OK-Robot** | arXiv 2024 | FAIR | 596 | MIT | 2024-03-04 | ✅ Stretch × 10 家 | 现场真实家庭 |
| **3D-LLM** | NeurIPS 2023 Spotlight | UMass + MIT + UCLA | **1196** | MIT | 2024-06-06 | ❌ | 自建 300K QA |
| **OpenMask3D** | NeurIPS 2023 | ETH + Microsoft | 259 | MIT | 2023-12-15 | ❌ | ScanNet200, Replica |
| **OpenScene** | CVPR 2023 | ETH + Google | 820 | Apache-2.0 | 2023-10-27 | ❌ | ScanNet, MP3D, nuScenes |
| **ConceptFusion** | RSS 2023 | Mila + MIT + CMU | 233 | MIT | 2023-09-23 | ❌ | Replica + 真实采集 |
> *Stars 与 pushed_at 抓取于 2026-05-16 19:00 (UTC+8)。原始数据见 [`.research/projects.json`](.research/projects.json)。*
---
## 2. 每个项目深度分析
### 2.1 ConceptGraphs (CMU + MIT + Mila, ICRA 2024)
- **核心思想**:从一系列 posed RGB-D 帧出发,用 SAM 提物体掩码 + CLIP 编码,**多视图融合到 3D 物体节点**,再由 GPT-4 推理"物体-物体"关系生成场景图。
- **数据流**:RGB-D → SAM 物体 mask → CLIP 嵌入 → 3D 物体节点 → LLM 推理关系 → JSON 场景图。
- **真机验证**:开源了 Jackal 机器人版本 (子仓库 `concept_graphs_jackal`),包含 nav + manipulation。
- **关键创新**:**首个把场景图节点压缩为"单个 CLIP 嵌入 + 几何 mesh"** 的方案,远比 OpenScene 的"per-point feature"省内存。
- **局限**:
- 仅在静态 RGB-D 序列上工作,**没有 lifelong 概念**
- 关系推理依赖 GPT-4,延迟和成本未优化
- 实际真机评测以演示视频为主,缺定量指标
- **行业贡献分**: **52 / 100**
- **PRISM 对比**:
- PRISM 多出:**RoomPlan 先验、在线感知、巩固机制、版本控制**
- ConceptGraphs 多出:**经过真机部署、有 ICRA 论文、社区生态(868 stars + jackal 子项目)**
### 2.2 HOV-SG (Freiburg, RSS 2024)
- **核心思想**:**Hierarchical Open-Vocabulary Scene Graph**——三层(floor / room / object),每层节点都附带 CLIP 嵌入,支持自然语言查询。
- **关键创新**:
- 首次把"楼层级"语义引入场景图(多数前作只到 room 级)
- 在 HM3DSem 上做出**完整 navigation pipeline + benchmark**
- 8 月 2024 还公开了 `hm3dsem_walks` 数据集
- **真机验证**:论文里有 Habitat 仿真 + 真机 RGB-D 测试 (Boston Dynamics Spot)。
- **局限**:仍是离线建图,**不支持长期更新**;floor 层级仅用平面检测启发式判断。
- **行业贡献分**: **50 / 100**
- **PRISM 对比**:
- **结构上极相似**:PRISM L2/L3/L4 ↔ HOV-SG floor/room/object
- PRISM 多出:**iPhone 先验、巩固、酒店运维场景**
- HOV-SG 多出:**RSS 论文、HM3DSem benchmark、楼层分割算法**
### 2.3 Clio (MIT-SPARK, RA-L 2024)
- **核心思想**:用**信息瓶颈原理**判断"什么物体/区域应该被纳入场景图":**任务驱动**,给定一组自然语言任务(如 "Read brown textbook"),决定语义粒度。
- **关键创新**:
- **首次为开放词表场景图提供理论框架**(Information Bottleneck)
- 实时 (real-time) 运行,与 MIT-SPARK 的 Hydra / Kimera / Khronos 工具链深度集成
- 后续 *Bayesian Fields* 2025 在同数据集做了延伸
- **数据集**:自建 4 个场景 (Office / Apartment / Cubicle / Building),含 RGB-D + rosbag + GT 物体标注。
- **真机验证**:数据来自真实采集(MIT 园区),但**机器人 demo 不突出**。
- **局限**:依赖 MIT-SPARK 重型工具栈,门槛高;Spot 真机 demo 不在主 paper。
- **行业贡献分**: **55 / 100** (理论 + 工具链组合贡献)
- **PRISM 对比**:
- **理论高度上 PRISM 远逊**:Clio 用信息瓶颈,PRISM 用工程经验
- **工程完整度 PRISM 接近**:PRISM 4 条管线 vs Clio 单一在线建图
- PRISM 应**借鉴 Clio 的信息瓶颈思想**指导差异检测阈值
### 2.4 OK-Robot (Meta FAIR, arXiv 2024.01)
- **核心思想**:**Systems-first** — 不发明新模型,组合 VLM + 导航 + 抓取,直接在真实家庭跑零样本 pick-and-place。
- **关键证据**:**10 个真实家庭、170+ 物体、58.5% 成功率**——这是同档项目里**真机 deploy 规模最大**的工作。
- **硬件**:Hello Robot Stretch + Dex Wrist。
- **关键创新**:
- 首次定量证明"现成 open-knowledge models + 简单组合 = 可用的家庭机器人"
- 失败模式分析极为细致(每类失败概率)
- **局限**:每次任务前需要扫描房间生成嵌入,**没有长期记忆**;不做场景图。
- **行业贡献分**: **70 / 100** (真机部署规模 + 复现教程是杀手锏)
- **PRISM 对比**:
- OK-Robot 的"扫描房间"完全可以用 PRISM 的 RoomPlan 替代,**两者互补**
- PRISM 多出:**长期记忆、巩固、差异检测**
- OK-Robot 多出:**真机大规模数据、抓取闭环**
- **如果 PRISM 在 OK-Robot 底座上做长期记忆扩展,可一举进入 70+ 档**
### 2.5 3D-LLM (UMass + MIT + UCLA, NeurIPS 2023 Spotlight)
- **核心思想**:把 3D 点云特征注入 LLM,做 3D captioning / QA / grounding / navigation 等。
- **关键创新**:
- 设计 3 种 prompting 机制收集 **300K 3D-language pairs**(目前 3D-LLM 领域最大数据集)
- 把 BLIP-2 / Flamingo 等 2D VLM "升级"为 3D-VLM
- **局限**:**离线 demo 性质**;没真机部署;依赖现成 3D 表示(点云/scene mesh)。
- **行业贡献分**: **42 / 100**
- **PRISM 对比**:
- 不在同一个赛道(3D-LLM 是研究 paper, PRISM 是工程方案)
- PRISM 的"11_world_model_bridge"接口设计可参考 3D-LLM 的 prompting 机制
### 2.6 OpenMask3D (ETH + Microsoft, NeurIPS 2023)
- **核心思想**:3D **实例**分割的开放词表化——用 SAM 在 2D 提 mask,投影到 3D,用 CLIP 给每个 instance 取嵌入。
- **关键创新**:
- **首次解决"open-vocab 3D 实例分割"** (vs OpenScene 的 per-point feature)
- 在 ScanNet200 上把 zero-shot 实例分割 mAP 从近 0 提到 ~16
- **行业贡献分**: **45 / 100**
- **PRISM 对比**:
- OpenMask3D 的"2D SAM mask → 3D instance"算法 PRISM 直接可用,**应在 03_data_schema 中引用**
- PRISM 的 `SpatialNode.bounding_box + clip_emb` 字段实际就是 OpenMask3D 的输出
### 2.7 OpenScene (ETH + Google, CVPR 2023)
- **核心思想**:**首个把 2D CLIP 特征蒸馏到 3D 点云**的工作。每个 3D 点都有一个 CLIP 嵌入,可零样本语义分割 + 文本查询。
- **关键创新**:
- 提出 **2D-3D 特征蒸馏框架**,影响了后续几乎所有 open-vocab 3D 工作(ConceptGraphs / OpenMask3D / HOV-SG 都引用它)
- 三大数据集 (ScanNet + MP3D + nuScenes) 全部跑通
- **行业贡献分**: **48 / 100** (基础工作 ⇒ 类似 NeRF 之于 SLAM)
- **PRISM 对比**:
- OpenScene 是 **PRISM L2 稠密表示**的 prior art
- PRISM 应在论文/文档中**显式引用 OpenScene** 作为 L2 语义特征来源
### 2.8 ConceptFusion (Mila + MIT + CMU, RSS 2023)
- **核心思想**:**Open-set Multimodal 3D Mapping**——把 CLIP / DINO / 语言等多模态特征融合到 3D 地图,可用文本/图像/语音查询。
- **关键创新**:
- 首次把"open-set"提到 3D 场景表示的核心位置
- 多模态查询(文本 + 图像 + 音频 + 几何)
- **行业贡献分**: **45 / 100**
- **PRISM 对比**:
- 是 PRISM L2 + L4 的精神祖先,**应在 02_architecture 引用**
---
## 3. 同档分布图(贡献分 vs 真机规模)
```
贡献分
↑
70 | ★ OK-Robot (10 家庭, 真机 SOTA)
|
60 |
|
55 | ★ Clio (理论 + 工具链)
52 | ★ ConceptGraphs (ICRA, 真机 ★)
50 | ★ HOV-SG (RSS)
48 | ★ OpenScene (CVPR 基石)
48 | ☆ PRISM (假设完成,工程满分,理论空白)
45 | ★ OpenMask3D ★ ConceptFusion
42 | ★ 3D-LLM (大数据,无真机)
|
└──────────────────────────────────→ 真机规模
无 Habitat 仿真 开源 demo 10+ 真实家庭
```
---
## 4. 横向能力矩阵 (10 维)
> ✅ 完整支持, ⚙️ 部分支持, ❌ 不支持
| 维度 | PRISM | ConceptGraphs | HOV-SG | Clio | OK-Robot | OpenScene | ConceptFusion | OpenMask3D | 3D-LLM |
|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
| 几何稠密表示 (TSDF/Mesh/3DGS) | ✅ TSDF+3DGS | ✅ Mesh | ✅ Mesh | ✅ TSDF | ⚙️ pointcloud | ⚙️ pointcloud | ⚙️ pointcloud | ⚙️ mesh | ❌ |
| 语义节点 / 物体 | ✅ L3 节点 | ✅ | ✅ | ✅ | ⚙️ embed | ❌ per-point | ⚙️ per-region | ✅ instance | ❌ |
| 分层架构 (floor/room/obj) | ✅ L2-L4 | ⚙️ obj+rel | ✅ 3 层 | ✅ region | ❌ | ❌ | ❌ | ❌ | ❌ |
| 开放词表 (open-vocab) | ✅ CLIP | ✅ CLIP | ✅ CLIP | ✅ CLIP | ✅ CLIP | ✅ CLIP | ✅ CLIP+多模态 | ✅ CLIP | ✅ |
| 在线增量更新 | ✅ Pipeline C | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
| 差异检测 / 物体移动 | ✅ Pipeline C | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| 记忆巩固 / 长时一致性 | ✅ Pipeline D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| 重定位 (大场景启动) | ✅ Pipeline B | ⚙️ ICP | ⚙️ | ⚙️ | ⚙️ | ❌ | ❌ | ❌ | ❌ |
| 真机部署 | ❌ 0 验证 | ✅ Jackal | ✅ Spot | ⚙️ MIT | ✅ 10 家 | ❌ | ❌ | ❌ | ❌ |
| 信息论 / 任务驱动理论 | ❌ | ❌ | ❌ | ✅ IB | ❌ | ❌ | ❌ | ❌ | ❌ |
| 消费级先验(手机扫描) | ✅ RoomPlan | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
| 完整 17 章工程文档 | ✅ | ❌ | ❌ | ❌ | ⚙️ | ❌ | ❌ | ❌ | ❌ |
> 解读:
> - **PRISM 的独占维度** = "Pipeline B-D 三件套" + "RoomPlan 集成" + "完整工程文档"
> - **PRISM 的弱项** = "真机部署 0" + "理论 0"
> - 综合所有维度,PRISM 是这张表中**功能模块最齐全**的项目
---
## 5. 技术血缘图(谁影响了谁)
```mermaid
graph TD
OpenScene["OpenScene<br/>CVPR 2023<br/>2D→3D CLIP 蒸馏"] --> OpenMask3D
OpenScene --> ConceptFusion
OpenScene --> CG[ConceptGraphs]
OpenScene --> HOVSG[HOV-SG]
OpenScene --> Clio
ConceptFusion["ConceptFusion<br/>RSS 2023<br/>多模态 open-set"] --> CG
ConceptFusion --> HOVSG
OpenMask3D["OpenMask3D<br/>NeurIPS 2023<br/>open-vocab instance"] --> CG
OpenMask3D --> HOVSG
CG["ConceptGraphs<br/>ICRA 2024<br/>双层场景图"] --> HOVSG
CG --> Clio
CG --> PRISM
HOVSG["HOV-SG<br/>RSS 2024<br/>分层 scene graph"] --> PRISM
Clio["Clio<br/>RA-L 2024<br/>IB 任务驱动"] -.-> PRISM
OKRobot["OK-Robot<br/>arXiv 2024<br/>10 家真机"] -.-> PRISM
LLM3D["3D-LLM<br/>NeurIPS 2023<br/>3D + LLM"] -.-> PRISM
PRISM["PRISM v1.2<br/>(2026)<br/>RoomPlan + 巩固"]
style PRISM fill:#ffe6cc,stroke:#d79b00,stroke-width:3px
style OpenScene fill:#e1d5e7,stroke:#9673a6
style ConceptFusion fill:#e1d5e7,stroke:#9673a6
style OpenMask3D fill:#e1d5e7,stroke:#9673a6
```
- **实线**:直接技术继承(算法/数据格式)
- **虚线**:精神继承(借鉴思想,未直接复用代码)
- **关键观察**:PRISM 处在血缘树的**最下游**——这意味着 PRISM 是"集大成者",但**也最难有原创性突破**
---
## 6. PRISM 与各项目的"何处更强 / 何处更弱"
### 6.1 PRISM **强于** 所有人的方面
1. **iPhone RoomPlan 集成** — **独占**。其他 8 项工作中**无任何一个**用手机端 LiDAR 扫描作为先验。这是 PRISM 最差异化的工程优势。
2. **巩固机制(睡眠 replay)** — **独占**。受认知科学启发,模拟海马体→新皮层巩固。Clio 有理论但没具体实现,PRISM 有具体充电桩 SOP。
3. **差异检测 + 物体移动事件** — **独占**。Pipeline C 的 "L2 vs RoomPlan 先验" 对比逻辑,其他工作都没做。
4. **完整工程文档** — 17 章 / 风险矩阵 / Gantt / MVP / 版本号 / 缩略语表,**远超学术开源仓的 README 水平**。
5. **酒店服务机器人场景特化** — 独占垂类。其他都是家庭通用。
### 6.2 PRISM **弱于** 同档项目的方面
| 短板 | 谁更强 | 差距 |
|---|---|---|
| 真机部署数据 | OK-Robot (10 家庭) | **极大**(0 vs 实测数据) |
| ICRA/RSS 论文 | ConceptGraphs / HOV-SG / Clio | **极大**(无 vs 顶会论文) |
| 理论框架 | Clio (信息瓶颈) | 大 |
| 开源数据集 | HOV-SG (HM3DSem), Clio (4 场景) | 中(PRISM 设计了 HotelScene-Bench 但未发) |
| GitHub 影响力 | 3D-LLM (1196 ★) / ConceptGraphs (868 ★) | 中(PRISM 待开源) |
| 实时性能 | Clio (real-time) | 中(PRISM 设计目标 10 Hz, 未实测) |
### 6.3 与每个项目的"互补合作"机会
| 合作对象 | 可交换价值 |
|---|---|
| **ConceptGraphs** | 用 PRISM 的"先验 + 巩固"扩展 ConceptGraphs 为 lifelong;反过来用 ConceptGraphs 的 GPT-4 关系推理填 PRISM L4 |
| **OK-Robot** | PRISM 的 RoomPlan 替代 OK-Robot 的"每次扫描房间";OK-Robot 的 Stretch 抓取闭环填补 PRISM 缺失的操作层 |
| **HOV-SG** | 用 HOV-SG 的 floor 分割算法替换 PRISM 的简化版;反过来 PRISM 的 RoomPlan 给 HOV-SG 一个"工厂预装"楼层先验 |
| **Clio** | 借鉴 Clio 信息瓶颈思想给 PRISM 的差异检测/巩固提供阈值理论;反过来 PRISM 的"睡眠"机制给 Clio 一个 lifelong 扩展 |
| **OpenScene** | 直接复用 OpenScene 的 2D→3D CLIP 蒸馏代码作为 PRISM L2 的语义层 |
| **OpenMask3D** | 用 OpenMask3D 的 instance mask 算法生成 PRISM `SpatialNode.bounding_box` |
| **ConceptFusion** | 借鉴 ConceptFusion 的多模态查询接口设计 PRISM 的 LLM Agent 查询 API |
| **3D-LLM** | 把 PRISM 的 spatial memory 作为 3D-LLM 的输入,演示"机器人长期记忆 + LLM 推理"闭环 |
---
## 7. PRISM 应该 / 不该 复现的算法
> **决策树**:别人已经做到 SOTA 的不重做;别人没做的或做得粗糙的才是 PRISM 的舞台。
| 算法层 | 是否自研 | 建议方案 |
|---|---|---|
| 2D 检测 / 分割 | ❌ 不重做 | 直接用 YOLO-World / SAM-2(已是 SOTA) |
| 2D→3D CLIP 蒸馏 | ❌ 不重做 | 复用 OpenScene 代码或 OpenMask3D 的 instance 版 |
| 3D 场景图节点抽取 | ⚙️ 部分自研 | 借 ConceptGraphs 的 mask→node 流程;关系层用 GPT-4o-mini |
| 楼层分割 | ⚙️ 部分自研 | 用 HOV-SG 的平面检测做 baseline,叠加 RoomPlan 先验 |
| 重定位(CLIP 召回) | ❌ 不重做 | 用 Hloc / NetVLAD 工具链 |
| 重定位(ICP 精配) | ❌ 不重做 | 用 Open3D 内置 |
| **差异检测** | ✅ **自研** | 这是 PRISM 的差异化 — 设计 "L2 vs prior" 阈值与冲突仲裁 |
| **巩固调度** | ✅ **自研** | 充电桩 SOP + 优先级队列 + 回滚是 PRISM 独有 |
| **iPhone ingest 管道** | ✅ **自研** | RoomPlan → SpatialMemory schema 转换,这是 PRISM 的"桥" |
| LLM 关系推理 | ❌ 不重做 | 用 GPT-4o / Claude 3.5,做好 prompt 模板 |
| 任务驱动语义粒度 | ⚙️ 借鉴 | 学习 Clio 的信息瓶颈思路,简化为启发式 |
---
## 8. 6 个月行动建议(基于对比)
### 8.1 优先级 1 — 先做真机验证,弥补 0 真机短板
- 仿 OK-Robot,选 Hello Stretch / Unitree H1 / Pudu Robot 作为载体
- 选 1 个真实小户型(40 m² 单卧),完整跑通 [`14_mvp.md`](14_mvp.md) 的 5 步
- **3 个月内出第一组真实数字**(cm 误差 + ms 延迟 + 重定位成功率)
### 8.2 优先级 2 — ICRA / IROS 2027 投稿,弥补 0 论文短板
- 主线:"先验地图驱动的 lifelong 室内场景图(以酒店为例)"
- 与 ConceptGraphs / HOV-SG 的差异点写明:**手机扫描 + 巩固机制 + 差异检测**
- 用 HotelScene-Bench(10 个酒店房间)做评测,与 ConceptGraphs / HOV-SG 跑横向对比
### 8.3 优先级 3 — 开源发布 + ROS 2 SIG 推广
- 选 MIT 协议(与 ConceptGraphs / HOV-SG / OK-Robot 一致)
- 发起 "ROS 2 Indoor Prior Map SIG",推动 RoomPlan 格式标准化
- 目标:**12 个月内,3+ 公司或机构使用 PRISM 的 LTM schema**
### 8.4 优先级 4 — 引入信息瓶颈理论,弥补理论 0 短板
- 借 Clio 的 IB 思想,给 PRISM 的"差异置信度"+"巩固优先级"一个**形式化目标函数**
- 目标:**让评审能在论文里看到至少 1 个定理**
---
## 9. 引用与原始数据
### 9.1 arXiv 论文 ID
| 项目 | arXiv ID | 会议/期刊 |
|---|---|---|
| ConceptGraphs | [2309.16650](https://arxiv.org/abs/2309.16650) | ICRA 2024 |
| HOV-SG | [2403.17846](https://arxiv.org/abs/2403.17846) | RSS 2024 |
| OpenScene | [2211.15654](https://arxiv.org/abs/2211.15654) | CVPR 2023 |
| 3D-LLM | [2307.12981](https://arxiv.org/abs/2307.12981) | NeurIPS 2023 Spotlight |
| OK-Robot | [2401.12202](https://arxiv.org/abs/2401.12202) | arXiv 2024.01 |
| OpenMask3D | [2306.13631](https://arxiv.org/abs/2306.13631) | NeurIPS 2023 |
| ConceptFusion | [2302.07241](https://arxiv.org/abs/2302.07241) | RSS 2023 |
| Clio | [2404.13696](https://arxiv.org/abs/2404.13696) | RA-L 2024 (10.10) |
### 9.2 GitHub 项目
| 项目 | 仓库 | Stars | 许可证 |
|---|---|---:|---|
| ConceptGraphs | [concept-graphs/concept-graphs](https://github.com/concept-graphs/concept-graphs) | 868 | MIT |
| HOV-SG | [hovsg/HOV-SG](https://github.com/hovsg/HOV-SG) | 470 | MIT |
| OpenScene | [pengsongyou/openscene](https://github.com/pengsongyou/openscene) | 820 | Apache-2.0 |
| 3D-LLM | [UMass-Foundation-Model/3D-LLM](https://github.com/UMass-Foundation-Model/3D-LLM) | 1196 | MIT |
| OK-Robot | [ok-robot/ok-robot](https://github.com/ok-robot/ok-robot) | 596 | MIT |
| OpenMask3D | [OpenMask3D/openmask3d](https://github.com/OpenMask3D/openmask3d) | 259 | MIT |
| ConceptFusion | [concept-fusion/concept-fusion](https://github.com/concept-fusion/concept-fusion) | 233 | MIT |
| Clio | [MIT-SPARK/Clio](https://github.com/MIT-SPARK/Clio) | 239 | BSD-2-Clause |
### 9.3 原始抓取数据(可复查)
存放于 [`plans/PRISM/.research/`](.research/) 目录:
- [`projects.json`](.research/projects.json) — GitHub API 抓取的元数据(stars / license / pushed_at / topics)
- [`arxiv_data.json`](.research/arxiv_data.json) — arXiv API 抓取的 title / abstract / 作者 / 主类
- [`readmes/*.md`](.research/readmes/) — 每个项目的 README 头部 ≤ 16 KB 快照
- [`fetch_projects.py`](.research/fetch_projects.py) / [`fetch_arxiv.py`](.research/fetch_arxiv.py) / [`fetch_readmes.py`](.research/fetch_readmes.py) — 抓取脚本,可重跑获取最新数据
### 9.4 数据时效
- 所有 **stars / pushed_at** 抓取于 **2026-05-16 ~19:00 UTC+8**
- **arXiv abstract** 是论文首版定稿后通常不再变,可信
- **README** 是各项目仓库 main/master 分支的当下快照,可能与论文版有差异
---
## 10. 一句话总结
> **"PRISM 站在 OpenScene → ConceptGraphs → HOV-SG → Clio 这条血脉的最下游,集成度最高、文档最美,但理论与真机两个维度都还在起步线之外。要进入这张表的第一梯队,只需做对两件事:(1) 真机跑出来,(2) 写一篇能投 ICRA 的论文。"**
---
**对比综述版本**:v1.0
**对应 PRISM 版本**:v1.2.0
**最后更新**:2026-05-16
**配套文件**:
- [`rate.md`](rate.md) — 技术难度评分(62/100)
- [`rate_industry.md`](rate_industry.md) — 行业贡献评分(48/100)
- [`.research/`](.research/) — 原始抓取数据 + 复现脚本
plans/PRISM/rate.md
+134
View File
@@ -0,0 +1,134 @@
# PRISM 技术难度评分
> **总分：62 / 100**
> 评分日期:2026-05-16
> 评分版本:PRISM v1.2.0
> 评分人:Code Assistant (基于公开机器人/感知技术常识与 PRISM v1.2.0 全文档)
---
## 1. 评分坐标系
用机器人 / 感知圈常见难度对标:
| 分数 | 系统画像 |
|---|---|
| **100** | 通用人形机器人开放世界 (Tesla Optimus / Figure 02 等) |
| **90** | 端到端 VLA + 长时记忆 (PaLM-E / RT-2 / OpenVLA 在真机稳定跑) |
| **80** | 大规模多机协同 SLAM (Spot 机群) |
| **70** | 单机 Lifelong SLAM + 语义记忆 (ConceptGraphs / OK-Robot / Habitat-Lab 上线) |
| **60** | **PRISM 这一档** — 单楼层语义 + 几何混合记忆 + 重定位 + 巩固 |
| **50** | ROS 2 + ZED VIO + YOLO 跑通基础感知 |
| **30** | 装好 ROS 2,能录 rosbag |
| **0** | 没装环境 |
---
## 2. 分项打分(合计 62/100)
| 维度 | 满分 | PRISM | 评注 |
|---|---:|---:|---|
| **架构原创性** | 15 | **11** | 四层 L1–L4 + 巩固机制清晰但**已有先例**(CMU CoBot 三层、ConceptGraphs 双层、HSSM 多层);"先验是宪法、实时是补丁"是有价值的工程哲学,但不算颠覆性。 |
| **系统集成复杂度** | 15 | **12** | 同时接 iPhone RoomPlan + ZED SDK + ROS 2 + Open3D + CLIP + YOLO-World + Neo4j + 3DGS 等 ≥ 12 个异构组件,跨 iOS/Linux/Jetson 三平台,**集成本身就是硬骨头**。 |
| **算法深度** | 15 | **7** | **几乎不发明新算法** — CLIP 检索、ICP 配准、TSDF 融合、YOLO 检测、DBSCAN 聚类全是现成。"两段式重定位"是工程组合而非新算法。**本项明显失分**。 |
| **核心问题难度** | 10 | **8** | "异构传感器混合时空记忆 + 长期一致性"本身是博士论文级问题,PRISM 没回避它(直面差异检测 + 巩固 + 冲突仲裁)。 |
| **工程可落地性** | 10 | **9** | 极少见的"读完真能跑"的方案 — 5 步 MVP、8 周路线图、确定硬件、确定传输协议。**本项非常突出**。 |
| **失败案例预案** | 8 | **8** | 6 大风险分类 × 概率/影响矩阵 × 缓解 × 监控指标。AGPL 传染、ArUco 兜底、巩固回滚都覆盖,**完备度罕见**。 |
| **评测体系** | 7 | **5** | 给出自建 HotelScene-Bench + 横向对接公开 benchmark,**但都未真正跑过**,只是接口描述。 |
| **代码就绪度** | 10 | **6** | ~2,500 行示例代码可直接 copy-paste,但**没有任何一行被运行过** — schema/parser/relocalizer 全是"理论代码",distance-to-working-bits ≈ 4-6 周。 |
| **可扩展性** | 5 | **3** | 户外、多机器人、动态变形物体(窗帘/门)几乎未覆盖;停留在"单楼层室内"。 |
| **理论深度** | 5 | **1** | **几乎没有数学** — 没有定理、没有收敛性证明、没有信息论或贝叶斯框架的统一描述。 |
| **合计** | **100** | **62** | — |
---
## 3. 三个分数支撑
### 3.1 为什么是 62 分(中上)
1. **工程合理性满分** — 这套方案给一支 4-人小组 8 周,**真的能跑出 demo**。这是大量学术系统做不到的(参考 NeRF-SLAM / Splat-SLAM 论文一堆但工程上跑不通)。
2. **覆盖维度齐** — 从 iPhone 端 Swift App、Mac 工作站 ingest、Jetson 上线推理、充电桩巩固,到 LLM Agent 查询,**端到端闭环**。
3. **现实风险预案优秀** — AGPL 传染、Apple SDK 闭源、ArUco 兜底、隐私过滤、回滚机制,这些在学术论文里**几乎不会出现**,反映出明显的工业经验。
### 3.2 为什么不是 80 分(上等偏上)
1. **没有新算法贡献** — 所有"创新点"都是**已知算法的工程组合**:
- CLIP+ICP 二段重定位 → Hloc 2020 / NetVLAD 2016 已用
- TSDF + 3DGS 双稠密 → NeRF-SLAM / Splat-SLAM 2023 已做
- 场景图 + Neo4j → ConceptGraphs 2023 已做
- 睡眠巩固 → 受 hippocampus replay 启发,Sun et al. 2019 已有原型
2. **代码 0 运行** — 所有 schema / 算法 / 接口都是**纸上代码**。真上手会发现:
- ARKit 帧同步问题
- ZED 与 iPhone 坐标系手性差异
- TSDF 与 OctoMap 双写一致性
- `pending.jsonl` 写并发竞争
…… 都是会咬人的实战陷阱。
3. **避开了真硬核** — 动态环境 SLAM 漂移补偿、3DGS 增量重训(其实远不止"warm-start")、多模态特征对齐数值稳定性、长时特征漂移 — **全跳过了**。
### 3.3 为什么不是 40 分(中下)
1. **没有犯方向性错误**:
- 选 iPhone + ZED 而不是搞自定义双目硬件 ✓
- 选 CLIP 召回再 ICP 精配,不死磕端到端 reloc 网络 ✓
- 选 Neo4j,不死磕自研图数据库 ✓
**每个选择都符合 2026 当下工业 ROI**。
2. **文档质量本身是 A+** — 结构 + 版本号 + 缩略语表 + 风险矩阵 + Gantt 路线图 + MVP demo + 统计自动化,**这是学术发表也常不及的**。
---
## 4. 一句话评价
> **"它不是一项研究,它是一份非常专业的产品蓝图。"**
>
> 所有难点都被识别、被分类、被指派给现成工具;但**真正难的部分(lifelong consistency, dynamic scene, semantic drift, RL-style 决策)几乎都被'委托给未来'或者'交给外部模型'**。
### 4.1 双视角加权
| 难度视角 | 分数 | 解释 |
|---|---:|---|
| **"完成它需要多少新发明"** | ~35 | 几乎所有算法都是 off-the-shelf |
| **"完成它需要多少工程能力"** | ~75 | 12+ 个异构组件、3 平台集成、SOP 级运维 |
| **加权平均** | **~62** | **强工程、弱研究**的优秀技术蓝图 |
---
## 5. 把它推到 80+ 的 4 条建议
1. **加 1-2 个算法新点**:比如
- "基于 CLIP 嵌入相似度的差异聚类自动阈值"
- "3DGS 增量重训的 forgetting-free 策略"
- "在线特征漂移的对抗式校准"
选一个跑出对比实验。
2. **真跑一个 baseline**:哪怕只跑 [`14_mvp.md`](14_mvp.md) 的 5 步,把数字(cm 误差、ms 延迟)替换成**真实测量**,PRISM 立刻从"方案"升级为"系统"。
3. **形式化四层耦合**:用一个统一的概率图模型 / 信念更新方程描述 L1 → L4 的写入与查询,PRISM 就有理论骨架,**可以投 ICRA / RSS**。
4. **覆盖动态场景**:现在 PRISM 假设"家具大部分不动"。把"人走动 + 物品频繁移位"的真实 hotel 场景纳入正式评测,难度立刻 **+15 分**。
---
## 6. 同类项目对标
| 项目 | 大致分数 | 备注 |
|---|---:|---|
| **ConceptGraphs** (CMU 2023) | 68 | 双层场景图 + LLM,有真实数据集 + 跑通 |
| **OK-Robot** (FAIR 2024) | 72 | open-vocabulary 抓取,跑了 10 个家庭 |
| **Spot SDK + Atlas** | 80 | Boston Dynamics 商用栈,多机协同 |
| **PRISM v1.2.0** | **62** | 完备的设计 + 0 运行验证 |
| **PRISM v2.0** (目标) | 75+ | 跑通 MVP + 加 1 个算法贡献 |
---
## 7. 评分注意事项 / 局限
- 本评分**未考虑商业价值**,仅看技术难度。商业上 PRISM 在酒店服务机器人垂类有很强落地性,这是另一维度。
- 评分参考了 PRISM v1.2.0 的 17 章正式文档,**没有访问真实代码仓库**(因为目前还没有)。
- 如未来 PRISM 发布原型实现 + 真实数据 + 新算法,分数将显著上调。
---
**评分版本**:v1.0
**对应 PRISM 版本**:v1.2.0
**最后更新**:2026-05-16
plans/PRISM/rate_industry.md
+176
View File
@@ -0,0 +1,176 @@
# PRISM 对机器人行业贡献的评分
> **总分：48 / 100**（假设项目全部完成,真机部署 + 开源 + 至少 1 篇顶会论文)
> 评分日期:2026-05-16
> 评分维度:对**机器人行业**的实际推进价值,而非技术难度本身
> 评分人:Code Assistant (基于公开行业格局与 PRISM v1.2.0 全文档)
---
## 1. 为什么不是 80+（行业贡献不等于技术完成度）
技术完成度评分 (rate.md) **62 分** 不能直接换算成行业贡献。后者还要看:
- 是否打开**新方向**(像 NeRF 2020 / Foundation Model 2022 那样)
- 是否成为**他人的基础设施**(像 ROS / OpenCV / SLAM toolbox)
- 是否**解决产业关键卡点**(像 Spot 把四足从研究变成商用)
- 是否**重构行业认知**(像 OpenAI Manipulation Hand 2019 让 RL+真机变可行)
按这 4 条审视 PRISM,大部分指标**不显著**。下面分项打分。
---
## 2. 分项打分(合计 48/100)
| 维度 | 满分 | 得分 | 评注 |
|---|---:|---:|---|
| **学术新方向** | 20 | **5** | 没开新方向。"四层记忆 + 巩固"是已有思想的工程整合,不是 paradigm shift。 |
| **开源基础设施价值** | 15 | **9** | 若按 MIT/Apache 开源,**iPhone RoomPlan → ROS 2 LTM 这条桥几乎填补空白**,中等价值。 |
| **产业卡点解决** | 15 | **8** | 解决了"室内机器人冷启动"+"长期一致性"两个真实痛点,但**酒店/家居赛道规模有限**,服务机器人本身还在低谷。 |
| **可复制性 / 工程模板** | 10 | **8** | "8 周复刻 + 12 周生产"路线图是**优秀模板**,其他公司可直接抄。该项突出。 |
| **数据集 / Benchmark 贡献** | 10 | **3** | HotelScene-Bench 只是接口设计,**真实数据若发布**才能加分。 |
| **生态影响 / 社区效应** | 10 | **4** | 单一团队风格,**没有围绕 PRISM 形成生态的明显信号**(没 plugin、没竞赛、没大厂背书)。 |
| **降低行业进入门槛** | 8 | **6** | 让"手机扫描 → 机器人记忆"白盒化,降低门槛**对中小公司**有用,对大公司贡献低。 |
| **文档/方法论传播价值** | 7 | **6** | 17 章工程文档 + 4 周时间表 + 风险地图,**是优秀写作样板**,值得机器人创业团队学习。 |
| **真机部署能力** | 5 | **0** | 当前 v1.2.0 **0 真机验证**;假设全部完成给 5,这里折扣后给 0。 |
| **跨学科溢出** | 5 | **(-)** | 几乎没向 ML / 认知科学 / HCI 输出新概念。 |
**注**:此评分**已假设"项目全部完成"** — 即真机跑通、开源、发表 1-2 篇论文、有 100 GitHub stars 级别热度。如果只算文档,行业贡献分约 **20-25**。
---
## 3. 同类/相近项目对比
### 3.1 同档(对行业贡献 30–55 分)
| 项目 | 年份 | 单位 | 行业贡献分 | 主要差异 |
|---|---|---|---:|---|
| **ConceptGraphs** | 2023 | CMU + MIT | **52** | 双层场景图 + open-vocabulary,**真跑过家庭场景**;论文 ICRA 2024,Github 1k+ star。PRISM 在系统完整性上更强,但 ConceptGraphs 有真实数据。 |
| **HOV-SG** | 2024 | Freiburg | **48** | 分层 open-vocabulary 3D 场景图,Habitat 测过。学术影响力略强。 |
| **OpenScene** | 2023 | ETH | **45** | 把 CLIP 投到 3D 点云,**开了开放词表场景理解一条线**。PRISM 是它的下游使用者。 |
| **3D-LLM** | 2023 | UCLA | **42** | 让 LLM 直接读 3D,但**还是 demo 级**。 |
| **PRISM (假设完成)** | 2026 | — | **~48** | 系统更工程化,但缺新算法/新数据/新范式。 |
### 3.2 高一档(60–75)— PRISM 想冲但够不到
| 项目 | 行业贡献分 | 为何高一档 |
|---|---:|---|
| **OK-Robot** (FAIR 2024) | **70** | 跑了 10 个真实家庭,**首次让 open-vocabulary 抓取在零样本家庭里 work**;真数据 + 真硬件 + 真复现指南。 |
| **NeRF-SLAM 系列** (2022-) | **65** | 把可微渲染引入 SLAM 主流,**改变了 SLAM 社区话题**。PRISM 用了它们的产出。 |
| **SayCan / PaLM-E** (Google 2022-23) | **78** | **首次让 LLM + 机器人形成闭环**,改变行业认知。 |
| **Habitat 2.0 / 3.0** (FAIR) | **72** | 室内导航/操作的事实标准 benchmark。 |
### 3.3 顶档(85+)— 改写规则的工作
| 项目 | 贡献分 | 影响 |
|---|---:|---|
| **ROS / ROS 2** (OSRF) | **96** | 机器人软件**默认中间件**。无可替代。 |
| **Boston Dynamics Spot** | **88** | 把四足从"实验室视频"变成"商用 SKU"。 |
| **NVIDIA Isaac Sim / Isaac ROS** | **82** | 仿真到真机的事实标准。 |
| **CARLA / Waymo Open Dataset** | **80** | 自动驾驶 benchmark 重塑了整个研究流程。 |
| **Tesla FSD / Mobileye REM** | **88** | 把数据驱动定位提到工业级。 |
### 3.4 低一档(20–40)— PRISM 容易掉进去的位置
| 类型 | 贡献分 | 为何低 |
|---|---:|---|
| 一个公司的内部技术栈, 不开源 | **25** | 只对单一公司有用 |
| 一篇 ICRA workshop 论文 | **18** | 引用不到 30 次, 无人复现 |
| 单一场景的 demo video | **15** | "看起来很酷"但没下文 |
---
## 4. PRISM 真正能给行业留下什么
如果项目**全部完成 + 开源 + 发表**,以下 4 项是最现实的贡献:
### 4.1 桥接价值 (★★★★ 中高)
**首次把 Apple RoomPlan → ROS 2 / 机器人 LTM 这条链路做完整**。
目前(2026 年)市面上**没有**公开方案让"业主自己用 iPhone 扫一下,机器人就能用"。
这是一个**实用的桥**,对消费级服务机器人有真实价值。
### 4.2 工程模板 (★★★ 中)
完整的 17 章 + Gantt + 风险矩阵 + MVP 模板,**值得机器人创业团队 ctrl-c**。
"先验是宪法、实时是补丁"这一句论据足够强,可能被引用为设计原则。
### 4.3 隐性贡献:对"记忆"在机器人领域的再讨论 (★★ 中低)
PRISM 提出"睡眠巩固"在工业服务机器人的具体实现 — **这一点很少被工程论文认真讨论**。
有助于把"长期记忆"从研究话题往产品话题推 1 小步。
### 4.4 不太可能的贡献 (★)
- 不会催生新算法
- 不会重构 SLAM 范式
- 不会让 LLM + 机器人产生新模式
- 不会成为 ROS 2 那样的基础设施(因为 stack 太专一)
---
## 5. 为什么大部分项目都打不到 80+
机器人行业贡献 80+ 的项目通常具备**至少 2 个**:
- ✅ 开了一个新方向(NeRF / DreamerV3 / RT-2)
- ✅ 让一类硬件变成商品(Spot / DJI Mavic)
- ✅ 成为别人 80% 工作流的依赖(ROS / Isaac / Habitat)
- ✅ 大规模真机部署 + 公开数据(Waymo / Tesla FSD)
- ✅ 学术界引用 1k+,产业界 deploy 10+
PRISM 即使全部完成:
- ❌ 没新方向
- ❌ 不影响硬件商品化
- ✅ 可能成为部分小厂的依赖 (+1)
- ❌ 单楼层级别,谈不上"大规模部署"
- ❓ 学术影响:乐观估计 ICRA 1 篇,引用 50-100
→ 落在 **40-55 区间**,中位数 **~48**。
---
## 6. 如何把行业贡献从 48 推到 65+
### 6.1 + 10 分:**真做 RoomPlan-to-Robot 开源标准**
明确 schema、写成 RFC,推动 ROS 2 SIG 接纳。让 1 年内 5+ 公司用上 PRISM 的 LTM 格式。
### 6.2 + 8 分:**发布 HotelScene-Bench 数据**
真实采集 10+ 酒店,标注 6DoF + 物品移动事件 + QA 任务。这就是**有数据有论文有引用**的硬通货。
### 6.3 + 5 分:**催生一个研究热点**
比如"长期感知一致性下的 forgetting-free 3DGS 增量训练" — 单挑这一个子问题,做出第一篇 paper,后续就是 follow up。
### 6.4 + 4 分:**生态:Plugin / 竞赛**
- 出一个 PRISM-Plugin SDK 让人接 Spot、Unitree H1、Pudu Robot
- 在 ROS World 2027 办一个 PRISM Track
→ 总计 **48 + 27 = 75**,挤入 ConceptGraphs / OK-Robot 同档。
---
## 7. 行业贡献 vs 商业价值的差异
| 维度 | 行业贡献分 | 商业价值分 |
|---|---:|---:|
| PRISM 评分 | **48** | **75**(在酒店服务机器人垂类) |
| 解读 | 不改写行业 | 但能让某个垂类提速 6-12 个月 |
**结论**:PRISM 是一份**值得做也值得做好**的项目,但应**清醒**——它是"很好用的轮子",**不是"新发动机"**。
---
## 8. 一句话评价
> **"PRISM 完成后,行业会感谢有这样一个干净的轮子,但不会因此改变方向。"**
---
## 9. 评分注意事项 / 局限
- 行业贡献评分**高度依赖未来事件**(是否开源、是否被采纳、是否有论文)。本评分按**最乐观假设**(全部完成 + 开源 + 论文)给出。
- 实际可能在 35–55 之间漂动。
- 对标项目选取以 2023–2026 间的 ICRA / RSS / CoRL / 工业新品为主,**不含强化学习/操作类**专门工作。
---
**评分版本**:v1.0
**对应 PRISM 版本**:v1.2.0(假设完成态)
**最后更新**:2026-05-16
**配套文件**:[`rate.md`](rate.md) 技术难度评分(62/100)
plans/PRISM/robot_brain_spatial_memory.md
+85
View File
@@ -0,0 +1,85 @@
# 🌈 PRISM —— 机器人大脑空间记忆架构
> 这个文件已演化为 **PRISM 项目** 的总入口索引页。
> 完整 14 章方案文档已迁移到 [`plans/PRISM/`](PRISM/) 目录。
---
## 一句话定位
> **PRISM = Prior-Registered Integrated Spatial Memory**
> 把 **iPhone (RoomPlan/LiDAR) 离线先验地图** 与 **ZED 2i (双目+IMU) 在线感知** 融合进机器人大脑的"四层空间记忆 + 记忆巩固"架构。
---
## 跳转到完整文档
👉 **[`plans/PRISM/README.md`](PRISM/README.md)** —— 章节导航 + 缩写说明
### 14 章速览
| # | 文件 | 主题 |
|---|------|------|
| 00 | [`PRISM/00_overview.md`](PRISM/00_overview.md) | 总览与命名由来 |
| 01 | [`PRISM/01_capability_decomposition.md`](PRISM/01_capability_decomposition.md) | 两方案能力解构（iPhone vs ZED 2i） |
| 02 | [`PRISM/02_architecture.md`](PRISM/02_architecture.md) | 四层空间记忆架构 (L1–L4) |
| 03 | [`PRISM/03_data_schema.md`](PRISM/03_data_schema.md) | 统一数据模型 `SpatialMemory` |
| 04 | [`PRISM/04_pipeline_A_iphone_offline.md`](PRISM/04_pipeline_A_iphone_offline.md) | 管线 A：iPhone 离线建图 |
| 05 | [`PRISM/05_pipeline_B_relocalization.md`](PRISM/05_pipeline_B_relocalization.md) | 管线 B：ZED 重定位握手 |
| 06 | [`PRISM/06_pipeline_C_online_perception.md`](PRISM/06_pipeline_C_online_perception.md) | 管线 C：在线感知与差异检测 |
| 07 | [`PRISM/07_pipeline_D_consolidation.md`](PRISM/07_pipeline_D_consolidation.md) | 管线 D：记忆巩固 |
| 08 | [`PRISM/08_runtime_timeline.md`](PRISM/08_runtime_timeline.md) | 端到端运行时序 (T0–T4) |
| 09 | [`PRISM/09_roadmap.md`](PRISM/09_roadmap.md) | 8 周原型路线图 + 4 个月生产 |
| 10 | [`PRISM/10_tech_stack.md`](PRISM/10_tech_stack.md) | 技术栈选型 |
| 11 | [`PRISM/11_world_model_bridge.md`](PRISM/11_world_model_bridge.md) | 衔接世界模型 (M-JEPA / DreamerV3) |
| 12 | [`PRISM/12_risks.md`](PRISM/12_risks.md) | 风险与对策 |
| 13 | [`PRISM/13_evaluation.md`](PRISM/13_evaluation.md) | 评测指标 |
| 14 | [`PRISM/14_mvp.md`](PRISM/14_mvp.md) | 最小可复现 demo (一个周末跑通) |
---
## 核心思想图
```
┌────────────────────────────────────────────┐
│ PRISM Brain │
│ │
┌────────┐ │ ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ │ ┌────────┐
│ iPhone │─┼─►│ L4 │◄►│ L3 │◄►│ L2 │◄►│ L1 │ │ │ Agent /│
│RoomPlan│ │ │语义图│ │拓扑图│ │ 度量 │ │ 缓冲 │ │ │ LLM │
└────────┘ │ └──────┘ └──────┘ └──────┘ └──────┘ │ └────┬───┘
离线一次 │ ▲ ▲ ▲ ▲ │ │ 查询
│ └────────┴────┬───┴────────┘ │ ▼
│ │ │ ┌────────┐
┌────────┐ │ ┌──────┴──────┐ │ │ Plan/ │
│ ZED 2i │─┼─────────────►│ Registrar │ │ │ Act │
│ VIO+D │ │ 实时 30 Hz │ CLIP + ICP │ │ └────────┘
└────────┘ │ └─────────────┘ │
│ │
│ ┌──────────────────────────────────┐ │
│ │ Consolidator（充电时跑） │ │
│ │ delta → 写回 LTM + 更新锚点 │ │
│ └──────────────────────────────────┘ │
└────────────────────────────────────────────┘
```
---
## 一行直觉
> **iPhone 是"教科书"，ZED 2i 是"眼睛"，机器人大脑是"既会查书又会观察、还会在书页空白处做笔记"的学生。**
---
## 推荐阅读路径
- **30 分钟决策者快读**：[`PRISM/00_overview.md`](PRISM/00_overview.md) → [`01`](PRISM/01_capability_decomposition.md) → [`02`](PRISM/02_architecture.md) → [`08`](PRISM/08_runtime_timeline.md)
- **工程师完整阅读**：从 00 顺序读到 14
- **PM 立项**：[`00`](PRISM/00_overview.md) → [`09`](PRISM/09_roadmap.md) → [`12`](PRISM/12_risks.md) → [`10`](PRISM/10_tech_stack.md)
- **想立刻动手**：[`PRISM/14_mvp.md`](PRISM/14_mvp.md) —— 一个周末 + 5 个 Python 脚本
---
**文档版本**：v1.1（拆分为章节后的索引页）
**最后更新**：2026-05-16
**Project Codename**：🌈 **PRISM**
plans/PRISM/tools/book_template.tex
+275
View File
@@ -0,0 +1,275 @@
% =====================================================================
% PRISM Book — Pandoc/XeLaTeX 模板
% 适配:macOS + TinyTeX + ctex + xeCJK,可独立 xelatex 编译
% =====================================================================
\documentclass[11pt,a4paper,oneside,openany]{ctexbook}
% ── 几何 / 字体 / 中文 ────────────────────────────────────────────────
\usepackage[a4paper,margin=2.4cm,headheight=24pt,headsep=0.5cm,footskip=1.2cm]{geometry}
\usepackage{amssymb} % 提供 \boxtimes, \checkmark 等任务列表符号
\usepackage{amsmath}
\usepackage{xeCJK}
\setCJKmainfont[BoldFont={STHeiti},ItalicFont={STKaiti}]{STSong}
\setCJKsansfont{STHeiti}
\setCJKmonofont{STHeiti}
\setmainfont{Helvetica Neue}
\setsansfont{Helvetica Neue}
\setmonofont{Menlo}[Scale=0.85]
\linespread{1.20}
% 让 xelatex 允许在 CJK 字符之间断行(关键!);
% 否则一行长中文(常见于表格单元、长描述)即使在 p{w} 列里也不会自动换行,
% 导致 Overfull \hbox 大量溢出。
\XeTeXlinebreaklocale "zh"
\XeTeXlinebreakskip = 0pt plus 1pt minus 0.1pt
% URL 也允许任意位置断开,避免长链接溢出
\usepackage{xurl}
% sloppy 模式:允许 inter-word 间距稍微变化以避免溢出
\sloppy
\emergencystretch=3em
% 改用 hyphenat:允许在所有字符之间断行,主要为长下划线文件名/URL/路径。
% 比 seqsplit 安全 — 它能在 \section / \texttt 中正常工作。
\usepackage[htt]{hyphenat}
% 对 etoolbox 提供的 \pretocmd 用 \texttt 包裹时启用低惩罚断字
% (\texttt 内部已是 \ttfamily,hyphenat 的 htt 选项让 ttfamily 也能断字)
% ── 颜色 / 框 / 代码 ──────────────────────────────────────────────────
\usepackage{xcolor}
\definecolor{PrismRed}{HTML}{E63946}
\definecolor{PrismOrange}{HTML}{F77F00}
\definecolor{PrismYellow}{HTML}{FCBF49}
\definecolor{PrismTeal}{HTML}{2A9D8F}
\definecolor{PrismBlue}{HTML}{1D7AB8}
\definecolor{PrismPurple}{HTML}{5A189A}
\definecolor{InkDark}{HTML}{0E1116}
\definecolor{InkMuted}{HTML}{4A5560}
\definecolor{BoxBG}{HTML}{F4F2EB}
\definecolor{CodeBG}{HTML}{F6F7F9}
\definecolor{CodeBorder}{HTML}{D7DBE0}
\definecolor{Accent}{HTML}{1D7AB8}
\definecolor{GoldRule}{HTML}{C9A227}
\usepackage[most]{tcolorbox}
\tcbset{
enhanced, breakable, boxrule=0pt, arc=2pt,
colback=BoxBG, colframe=Accent, fonttitle=\bfseries,
before skip=4pt, after skip=4pt,
left=8pt, right=8pt, top=4pt, bottom=4pt,
beforeafter skip balanced
}
\usepackage{listings}
\lstdefinestyle{prism}{
basicstyle=\ttfamily\footnotesize,
backgroundcolor=\color{CodeBG},
frame=single, rulecolor=\color{CodeBorder},
framesep=4pt, framerule=0.4pt,
xleftmargin=8pt, xrightmargin=8pt,
showstringspaces=false,
breaklines=true, breakatwhitespace=false,
keywordstyle=\color{PrismPurple}\bfseries,
commentstyle=\color{InkMuted}\itshape,
stringstyle=\color{PrismTeal},
numberstyle=\tiny\color{InkMuted}, numbers=none,
upquote=true, columns=flexible,
literate=
{á}{{\'a}}1 {é}{{\'e}}1 {í}{{\'i}}1 {ó}{{\'o}}1 {ú}{{\'u}}1
{→}{{$\rightarrow$}}1 {←}{{$\leftarrow$}}1
{≥}{{$\geq$}}1 {≤}{{$\leq$}}1 {±}{{$\pm$}}1 {≈}{{$\approx$}}1
{×}{{$\times$}}1 {⋯}{{$\cdots$}}1 {·}{{·}}1
{…}{{\ldots}}1 {⌶}{{$\Box$}}1
}
\lstset{style=prism}
% pandoc 用 fancyvrb 转 verbatim;用 fvextra 扩展自动换行能力,避免长注释/长字符串超出页宽
\usepackage{fvextra}
\DefineVerbatimEnvironment{Verbatim}{Verbatim}{%
fontsize=\footnotesize,%
breaklines=true,%
breakanywhere=true,%
breaksymbolleft={},%
breaksymbolright={},%
}
% 用 tcolorbox 来包代码块,既能套色又能正确处理换页
\definecolor{CodeBoxBG}{HTML}{F6F7F9}
\definecolor{CodeBoxRule}{HTML}{D7DBE0}
\makeatletter
\@ifundefined{Shaded}{%
\newtcolorbox{ShadedBox}{
enhanced, breakable,
colback=CodeBoxBG, colframe=CodeBoxRule,
boxrule=0.4pt, arc=2pt,
left=6pt, right=6pt, top=4pt, bottom=4pt,
before skip=4pt, after skip=4pt
}
\newenvironment{Shaded}{\begin{ShadedBox}}{\end{ShadedBox}}%
}{}
\makeatother
% pandoc syntax-highlight 需要的环境与命令
\usepackage{upquote}
% pandoc 用法:\begin{Shaded}\begin{Highlighting}[options]...\end{Highlighting}\end{Shaded}
% 所以 Highlighting 内部 只做 Verbatim,背景已由外层 Shaded(=ShadedBox tcolorbox)负责
\makeatletter
\@ifundefined{Highlighting}{%
\newenvironment{Highlighting}[1][]{%
\VerbatimEnvironment%
\begin{Verbatim}[fontsize=\footnotesize,commandchars=\\\{\},#1]%
}{%
\end{Verbatim}%
}%
}{}
\makeatother
% ── 表格 / 列表 ────────────────────────────────────────────────────────
\usepackage{booktabs}
\usepackage{longtable}
\usepackage{array}
\usepackage{tabularx}
\usepackage{makecell}
\usepackage{enumitem}
\setlist{nosep,leftmargin=*}
% ── 图 / 链接 ──────────────────────────────────────────────────────────
\usepackage{graphicx}
\usepackage{float}
\usepackage[unicode=true,colorlinks=true,
linkcolor=Accent, citecolor=Accent, urlcolor=Accent,
pdftitle={PRISM v__PRISM_VERSION__ — Prior-Registered Integrated Spatial Memory},
pdfauthor={Project Planning Team},
pdfsubject={PRISM v__PRISM_VERSION__, built __PRISM_BUILD_DATE__},
pdfkeywords={PRISM, RoomPlan, ZED 2i, SLAM, Spatial Memory, Robot, World Model}]{hyperref}
% ── 页眉页脚 ──────────────────────────────────────────────────────────
\usepackage{fancyhdr}
\pagestyle{fancy}
\fancyhf{}
\fancyhead[L]{\small\textsf{\textcolor{InkMuted}{PRISM \textbullet{} 机器人大脑空间记忆架构 \textbullet{} v__PRISM_VERSION__}}}
\fancyhead[R]{\small\textsf{\textcolor{InkMuted}{\leftmark}}}
\fancyfoot[L]{\small\textsf{\textcolor{InkMuted}{Build: __PRISM_BUILD_DATE__}}}
\fancyfoot[C]{\small\textsf{\textcolor{InkMuted}{\thepage{} / \pageref*{LastPage}}}}
\fancyfoot[R]{\small\textsf{\textcolor{InkMuted}{v__PRISM_VERSION__}}}
\renewcommand{\headrulewidth}{0.3pt}
\renewcommand{\headrule}{\hbox to\headwidth{\color{GoldRule}\leaders\hrule height \headrulewidth\hfill}}
\renewcommand{\footrulewidth}{0pt}
\usepackage{lastpage}
% ── 章节样式 ──────────────────────────────────────────────────────────
\ctexset{
chapter={
name={第,章},
number=\arabic{chapter},
format=\Large\bfseries\sffamily\raggedright,
titleformat=\Large\bfseries\sffamily\raggedright,
aftername={\hspace{0.6em}},
beforeskip=1.2em, afterskip=1.0em
},
section={
format=\large\bfseries\sffamily\raggedright,
aftername={\hspace{0.5em}}
},
subsection={
format=\normalsize\bfseries\sffamily\raggedright,
},
subsubsection={
format=\normalsize\sffamily\raggedright,
}
}
% ── pandoc 兼容包 ─────────────────────────────────────────────────────
\usepackage{calc} % pandoc 表格列宽需要 calc 包
\providecommand{\tightlist}{\setlength{\itemsep}{0pt}\setlength{\parskip}{0pt}}
\providecommand{\pandocbounded}[1]{#1}
\providecommand{\real}[1]{#1} % pandoc:把 0.4545 原样传入 calc
\providecommand{\textquotesingle}{\textsf{'}}
% Pandoc 用 \def\LTcaptype{none};需要先定义 none counter 避免 \refstepcounter{none} 报错
\newcounter{none}
% 一些 pandoc highlight 占位 (避免缺命令报错)
\providecommand{\NormalTok}[1]{#1}
\providecommand{\KeywordTok}[1]{\textcolor{PrismPurple}{\textbf{#1}}}
\providecommand{\DataTypeTok}[1]{\textcolor{PrismTeal}{#1}}
\providecommand{\DecValTok}[1]{\textcolor{PrismOrange}{#1}}
\providecommand{\BaseNTok}[1]{\textcolor{PrismOrange}{#1}}
\providecommand{\FloatTok}[1]{\textcolor{PrismOrange}{#1}}
\providecommand{\CharTok}[1]{\textcolor{PrismTeal}{#1}}
\providecommand{\StringTok}[1]{\textcolor{PrismTeal}{#1}}
\providecommand{\CommentTok}[1]{\textcolor{InkMuted}{\textit{#1}}}
\providecommand{\OtherTok}[1]{#1}
\providecommand{\AlertTok}[1]{\textcolor{PrismRed}{\textbf{#1}}}
\providecommand{\FunctionTok}[1]{\textcolor{Accent}{#1}}
\providecommand{\RegionMarkerTok}[1]{#1}
\providecommand{\ErrorTok}[1]{\textcolor{PrismRed}{\textbf{#1}}}
\providecommand{\BuiltInTok}[1]{#1}
\providecommand{\ExtensionTok}[1]{#1}
\providecommand{\PreprocessorTok}[1]{\textcolor{PrismOrange}{#1}}
\providecommand{\AttributeTok}[1]{#1}
\providecommand{\InformationTok}[1]{\textcolor{InkMuted}{#1}}
\providecommand{\WarningTok}[1]{\textcolor{PrismOrange}{\textbf{#1}}}
\providecommand{\AnnotationTok}[1]{\textcolor{InkMuted}{\textit{#1}}}
\providecommand{\VariableTok}[1]{#1}
\providecommand{\ImportTok}[1]{\textcolor{PrismPurple}{#1}}
\providecommand{\OperatorTok}[1]{#1}
\providecommand{\ControlFlowTok}[1]{\textcolor{PrismPurple}{\textbf{#1}}}
\providecommand{\SpecialCharTok}[1]{#1}
\providecommand{\VerbatimStringTok}[1]{\textcolor{PrismTeal}{#1}}
\providecommand{\SpecialStringTok}[1]{\textcolor{PrismTeal}{#1}}
\providecommand{\ConstantTok}[1]{\textcolor{PrismOrange}{#1}}
\providecommand{\DocumentationTok}[1]{\textcolor{InkMuted}{\textit{#1}}}
\providecommand{\CommentVarTok}[1]{\textcolor{InkMuted}{\textit{#1}}}
% pandoc 引文环境
\newenvironment{cslreferences}{}{}
\newcommand{\citeproctext}{}
\newcommand{\citeproc}[2]{#2}
% emoji / unicode 兜底 (xeCJK 大多直接显示;不显示的用 textsf 代替)
\newcommand{\emoji}[1]{\texorpdfstring{#1}{}}
% ─────────────────────────────────────────────────────────────────────
\title{\bfseries PRISM \\[6pt]
\large Prior-Registered Integrated Spatial Memory \\[3pt]
\normalsize 机器人大脑空间记忆架构 \\[6pt]
\small\sffamily\textcolor{InkMuted}{v__PRISM_VERSION__ \textperiodcentered{} __PRISM_BUILD_DATE__}}
\author{项目规划组 \textperiodcentered{} Project Planning Team}
\date{__PRISM_BUILD_DATE__ \textperiodcentered{} v__PRISM_VERSION__}
\begin{document}
\frontmatter
\maketitle
% 简短前言 (一页)
\thispagestyle{empty}
\vspace*{2em}
\begin{tcolorbox}[title={关于本书 / About this Book},
colback=BoxBG, colframe=Accent]
\textbf{PRISM} (Prior-Registered Integrated Spatial Memory) 是一种把
\emph{iPhone RoomPlan 离线先验地图} 与 \emph{ZED 2i 双目+IMU 在线感知}
融合进同一个机器人大脑的\textbf{四层空间记忆架构}。
本书完整收录 PRISM v1.1 方案的 15 个章节
(README + 14 章),含可立即复制运行的 Python schema、ROS 2 节点编排、
8 周路线图、风险登记、评测体系、以及一份周末可跑通的最小 MVP。
\end{tcolorbox}
\vfill
\noindent\textsf{\textcolor{InkMuted}{\small
\textbf{License}: CC BY-NC 4.0 \hfill
\textbf{Repo}: \texttt{github.com/<org>/prism}\\
\textbf{文档版本}: v1.1 \hfill
\textbf{最后更新}: 2026-05-16
}}
\cleardoublepage
\tableofcontents
\cleardoublepage
\mainmatter
% ─────────── 正文由 pandoc 注入 ───────────
__PRISM_BODY_PLACEHOLDER__
\end{document}
plans/PRISM/tools/build_book.sh
+318
View File
@@ -0,0 +1,318 @@
#!/usr/bin/env bash
# 构建 PRISM 出版级 PDF 全书
# 1. 预处理 14 个 markdown(剥离 emoji、改链接)
# 2. pandoc 转 LaTeX body
# 3. 注入模板 → XeLaTeX 编译两次(目录)
# 4. 用 pypdf 合并 cover + body → PRISM_Whole.pdf
#
# 用法:
# bash plans/PRISM/tools/build_book.sh
#
# 输出:
# plans/PRISM/PRISM_Book.pdf 正文部分
# plans/PRISM/PRISM_Whole.pdf 封面 + 正文 完整版
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
PRISM_DIR="$ROOT/plans/PRISM"
TOOLS_DIR="$PRISM_DIR/tools"
BUILD_DIR="$PRISM_DIR/.build"
mkdir -p "$BUILD_DIR"
# ── 版本与构建日期(注入封面/页眉/章节正文)──────────────────────────
if [[ -f "$PRISM_DIR/VERSION" ]]; then
VERSION="$(tr -d '[:space:]' < "$PRISM_DIR/VERSION")"
else
VERSION="0.0.0"
fi
BUILD_DATE="$(date +'%Y-%m-%d')"
echo "==> Building PRISM Book v${VERSION} (${BUILD_DATE})"
echo "==> 1a) 生成统计章节 16_stats.md"
python3 "$TOOLS_DIR/gen_stats.py" "$PRISM_DIR" "$PRISM_DIR/16_stats.md"
echo "==> 1a') 生成变更日志章节 17_changelog.md (从 CHANGELOG.md)"
python3 "$TOOLS_DIR/gen_changelog_chapter.py" \
"$PRISM_DIR/CHANGELOG.md" \
"$PRISM_DIR/17_changelog.md" \
"$VERSION" "$BUILD_DATE"
echo "==> 1) 预处理 markdown 文件"
# 章节顺序
CHAPTERS=(
"README.md"
"00_overview.md"
"01_capability_decomposition.md"
"02_architecture.md"
"03_data_schema.md"
"04_pipeline_A_iphone_offline.md"
"05_pipeline_B_relocalization.md"
"06_pipeline_C_online_perception.md"
"07_pipeline_D_consolidation.md"
"08_runtime_timeline.md"
"09_roadmap.md"
"10_tech_stack.md"
"11_world_model_bridge.md"
"12_risks.md"
"13_evaluation.md"
"14_mvp.md"
"15_glossary.md"
"16_stats.md"
"18_lyra_inspirations.md"
"17_changelog.md"
)
# Python 预处理:把每个 md 的一级标题改成 LaTeX \chapter,
# 移除 emoji / 修复链接 / 删除 raw HTML
python3 - <<'PYEOF'
import re, sys, pathlib
PRISM_DIR = pathlib.Path("plans/PRISM")
OUT = PRISM_DIR / ".build" / "merged.md"
# 章节顺序
order = [
"README.md",
"00_overview.md",
"01_capability_decomposition.md",
"02_architecture.md",
"03_data_schema.md",
"04_pipeline_A_iphone_offline.md",
"05_pipeline_B_relocalization.md",
"06_pipeline_C_online_perception.md",
"07_pipeline_D_consolidation.md",
"08_runtime_timeline.md",
"09_roadmap.md",
"10_tech_stack.md",
"11_world_model_bridge.md",
"12_risks.md",
"13_evaluation.md",
"14_mvp.md",
"15_glossary.md",
"16_stats.md",
"18_lyra_inspirations.md",
"17_changelog.md",
]
# 章节显示名(替换原一级标题,使目录整齐)
chapter_titles = {
"README.md": "项目总览 (README)",
"00_overview.md": "第 0 章 总览与命名由来",
"01_capability_decomposition.md": "第 1 章 两方案能力解构",
"02_architecture.md": "第 2 章 四层空间记忆架构",
"03_data_schema.md": "第 3 章 统一数据模型",
"04_pipeline_A_iphone_offline.md": "第 4 章 管线 A:iPhone 离线建图",
"05_pipeline_B_relocalization.md": "第 5 章 管线 B:ZED 重定位握手",
"06_pipeline_C_online_perception.md": "第 6 章 管线 C:在线感知与差异检测",
"07_pipeline_D_consolidation.md": "第 7 章 管线 D:记忆巩固",
"08_runtime_timeline.md": "第 8 章 端到端运行时序",
"09_roadmap.md": "第 9 章 实施路线图",
"10_tech_stack.md": "第 10 章 技术栈选型",
"11_world_model_bridge.md": "第 11 章 与世界模型衔接",
"12_risks.md": "第 12 章 风险与对策",
"13_evaluation.md": "第 13 章 评测指标",
"14_mvp.md": "第 14 章 最小可复现 demo",
"15_glossary.md": "第 15 章 缩略语表",
"16_stats.md": "第 16 章 文档与代码统计",
"18_lyra_inspirations.md": "第 18 章 Lyra 2.0 启发的设计原则",
"17_changelog.md": "第 17 章 变更日志",
}
# 常见 emoji / 装饰符号 → 文字或删除
EMOJI_MAP = {
"🌈":"", "📱":"", "📷":"", "🤖":"", "🚀":"", "💻":"", "📊":"", "📋":"",
"📚":"", "🎓":"", "🎬":"", "🏗️":"", "🏨":"", "💰":"", "📐":"", "🌟":"",
"⚠️":"[!]", "✅":"[OK]", "❌":"[X]", "🟢":"[L]", "🟡":"[M]", "🔴":"[H]",
"🎯":"", "📈":"", "📞":"", "📍":"", "🗺️":"", "📦":"", "🔧":"", "🤝":"",
"📝":"", "🌐":"", "📄":"", "📧":"", "💡":"", "💬":"", "📤":"", "📥":"",
"▶":">", "◀":"<", "⭐":"★",
"↑":"↑", "↓":"↓",
# 第二轮补充:xelatex_pass2.log 中报告的缺失字符
"🔬":"", "🧑":"", "💼":"", "🎤":"", "📹":"",
"→":"->", "↔":"<->",
"①":"(1)", "②":"(2)", "③":"(3)", "④":"(4)",
"✓":"[v]",
"・":"·",
}
def preprocess(text: str, source_name: str) -> str:
# 1) emoji → 替换
for k, v in EMOJI_MAP.items():
text = text.replace(k, v)
# 2) 移除起首 yaml/front matter(不会有,跳过)
# 3) 删除 raw HTML 注释 / details
text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
text = re.sub(r"</?details[^>]*>", "", text)
text = re.sub(r"</?summary[^>]*>", "", text)
# 4) 替换文件名链接 [`xx.md`](path) → \emph{xx.md} 等;
# Pandoc 默认会把这些保留为超链接(指向不存在的文件),
# 我们把 .md 链接收敛为只显示文本,避免大量"红色断链"。
# 直接降级为反引号代码标记,让 pandoc 自己处理转义(它会输出 \texttt{}+下划线转义)
def md_link_replace(m):
token = m.group(1)
# 保留为 inline-code: `xxx`
return f"`{token}`"
text = re.sub(r"\[`([^`\]]+)`\]\([^)]+\)", md_link_replace, text)
# 普通 [text](xxx.md...) 不带反引号的:仅保留文本(纯文本无需转义)
text = re.sub(r"\[([^\]]+?)\]\([^)]*\.md[^)]*\)",
lambda m: m.group(1), text)
# 5) 把第一行 H1 替换为指定章节标题
title = chapter_titles.get(source_name, source_name)
lines = text.splitlines()
for i, l in enumerate(lines):
if l.startswith("# "):
lines[i] = f"# {title}"
break
else:
lines.insert(0, f"# {title}")
# 6) 降级:把 ## 起的所有标题保留;一级 # 后面紧跟的 "## —— xxx" 类副标题
# 保持原样,但避免 pandoc 把它认成另一个 chapter
out = "\n".join(lines)
# 7) 把 horizontal rule 上下的"章节版本/估计阅读时间/关键收获"等元信息
# 保留(它们是有用尾页内容)
return out
# 拼接
parts = []
for fn in order:
p = PRISM_DIR / fn
if not p.exists():
print(f"[WARN] missing {p}")
continue
body = p.read_text(encoding="utf-8")
body = preprocess(body, fn)
# 每章前换页
parts.append("\n\n\\newpage\n\n" + body + "\n")
OUT.write_text("\n".join(parts), encoding="utf-8")
print(f"[ok] merged → {OUT} ({OUT.stat().st_size/1024:.1f} KB)")
PYEOF
echo "==> 1b) 渲染 mermaid 图块 (mmdc → PNG)"
FIG_DIR="$BUILD_DIR/figs"
if [[ ! -x "$TOOLS_DIR/node_modules/.bin/mmdc" ]]; then
echo "[ERROR] mmdc 未安装,请先执行:"
echo " cd $TOOLS_DIR && npm install @mermaid-js/mermaid-cli"
exit 1
fi
python3 "$TOOLS_DIR/render_mermaid.py" \
"$BUILD_DIR/merged.md" \
"$BUILD_DIR/merged.md" \
"$FIG_DIR"
echo "==> 2) pandoc → LaTeX body"
pandoc "$BUILD_DIR/merged.md" \
--from=markdown+pipe_tables+grid_tables+raw_tex+task_lists+yaml_metadata_block \
--to=latex \
--top-level-division=chapter \
--listings=false \
--wrap=preserve \
--highlight-style=tango \
-o "$BUILD_DIR/body.tex"
# 把 body 注入模板,同时把 __PRISM_VERSION__ / __PRISM_BUILD_DATE__ 占位符替换为真实值;
# 并对 pandoc 生成的"自然宽度 longtable"做自适应换行后处理。
export PRISM_VERSION="$VERSION"
export PRISM_BUILD_DATE="$BUILD_DATE"
python3 - <<'PYEOF'
import os, pathlib, re
tpl = pathlib.Path("plans/PRISM/tools/book_template.tex").read_text(encoding="utf-8")
body = pathlib.Path("plans/PRISM/.build/body.tex").read_text(encoding="utf-8")
version = os.environ.get("PRISM_VERSION", "0.0.0")
build_date = os.environ.get("PRISM_BUILD_DATE", "unknown")
tpl = tpl.replace("__PRISM_VERSION__", version)
tpl = tpl.replace("__PRISM_BUILD_DATE__", build_date)
# pandoc 默认生成 \begin{longtable}[]{@{}lll@{}} 这种自然宽列规范,
# 长内容(URL/中文长句/代码)会冲出页面。把所有形如 @{}[lcr]+@{} 的
# 自然宽列规范替换为等宽 p{w} 列,启用单元格内换行。
LT_HEAD_RE = re.compile(
r"\\begin\{longtable\}(\[[^\]]*\])?\{@\{\}([lcr]+)@\{\}\}"
)
def _replace_lt(m):
opts = m.group(1) or ""
cols = m.group(2)
n = len(cols)
col_spec = " ".join(
r">{\raggedright\arraybackslash}p{(\linewidth - "
f"{2*n}" + r"\tabcolsep)/" + f"{n}" + r"}"
for _ in range(n)
)
return f"\\begin{{longtable}}{opts}{{@{{}}{col_spec}@{{}}}}"
n_fixed = len(LT_HEAD_RE.findall(body))
body = LT_HEAD_RE.sub(_replace_lt, body)
print(f"[longtable] auto-wrapped {n_fixed} natural-width tables -> p columns")
out = tpl.replace("__PRISM_BODY_PLACEHOLDER__", body)
pathlib.Path("plans/PRISM/.build/book.tex").write_text(out, encoding="utf-8")
print("[ok] injected -> plans/PRISM/.build/book.tex")
PYEOF
echo "==> 3) XeLaTeX 编译 (两次,生成目录)"
cd "$BUILD_DIR"
# 用 -interaction=nonstopmode 跑两遍;第二遍解决 \tableofcontents 引用
for i in 1 2; do
echo " pass $i ..."
xelatex -interaction=nonstopmode -halt-on-error -file-line-error \
book.tex > "xelatex_pass${i}.log" 2>&1 || {
echo "[ERROR] XeLaTeX pass $i failed; tail of log:"
tail -60 "xelatex_pass${i}.log"
exit 1
}
done
cd - >/dev/null
cp "$BUILD_DIR/book.pdf" "$PRISM_DIR/PRISM_Book.pdf"
echo "[ok] body PDF → $PRISM_DIR/PRISM_Book.pdf ($(du -h "$PRISM_DIR/PRISM_Book.pdf" | cut -f1))"
echo "==> 4) 合并 cover + body → PRISM_Whole.pdf"
python3 - <<PYEOF
from pypdf import PdfWriter, PdfReader
import pathlib
prism = pathlib.Path("plans/PRISM")
cover = prism / "PRISM_Cover.pdf"
body = prism / "PRISM_Book.pdf"
out = prism / "PRISM_Whole.pdf"
w = PdfWriter()
if cover.exists():
for p in PdfReader(str(cover)).pages:
w.add_page(p)
print(f" + cover: {len(PdfReader(str(cover)).pages)} pages")
else:
print(f" [WARN] cover not found: {cover}")
# 从正文 PDF 继承 metadata(Title/Author/Subject/Keywords),
# 这样合并后的 Whole.pdf 也带 v__VERSION__ 标识。
body_reader = PdfReader(str(body))
for p in body_reader.pages:
w.add_page(p)
print(f" + body: {len(body_reader.pages)} pages")
src_meta = body_reader.metadata or {}
w.add_metadata({
"/Title": src_meta.get("/Title", "PRISM"),
"/Author": src_meta.get("/Author", "Project Planning Team"),
"/Subject": src_meta.get("/Subject", "PRISM"),
"/Keywords": src_meta.get("/Keywords", "PRISM"),
"/Creator": src_meta.get("/Creator", "LaTeX + pypdf"),
})
with open(out, "wb") as f:
w.write(f)
print(f"[ok] merged → {out} ({out.stat().st_size/1024/1024:.2f} MB)")
print(f" Title: {src_meta.get('/Title', '<none>')}")
print(f" Subject: {src_meta.get('/Subject', '<none>')}")
PYEOF
echo ""
echo "============================================================"
echo " ✅ PRISM 全书构建完成"
echo "------------------------------------------------------------"
echo " 封面: $PRISM_DIR/PRISM_Cover.pdf"
echo " 正文: $PRISM_DIR/PRISM_Book.pdf"
echo " 整合全本: $PRISM_DIR/PRISM_Whole.pdf"
echo "============================================================"
plans/PRISM/tools/gen_changelog_chapter.py
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""
gen_changelog_chapter.py
========================
把 CHANGELOG.md 转换为 PRISM 全书的 "第 17 章 变更日志" 章节文件,
便于 build_book.sh 把它当作普通章节加入 PDF。
用法:
python3 gen_changelog_chapter.py <changelog.md> <out_md> <version> <build_date>
"""
from __future__ import annotations
import pathlib
import sys
def main(changelog: pathlib.Path, out_md: pathlib.Path,
version: str, build_date: str) -> None:
raw = changelog.read_text(encoding="utf-8")
# 给章节加一个统一的标题(由 build_book.sh 的 chapter_titles 覆盖,
# 但这里也写一个 H1 以保证 md 渲染时仍是一个完整章节)。
header = "\n".join([
"# Chapter 17 — 变更日志 (Changelog)",
"",
f"> 本章直接镜像项目根的 [`CHANGELOG.md`](CHANGELOG.md);**当前版本 v{version}, 构建于 {build_date}**。"
" 每次发版时由 [`tools/gen_changelog_chapter.py`](tools/gen_changelog_chapter.py) 自动同步到 PDF。",
"",
])
# 把原 CHANGELOG.md 的 H1 (`# Changelog`) + 顶层引言 + 首个 --- 全部跳过,
# 从第一个真正的 ## 二级标题开始拼接,避免重复说明。
lines = raw.splitlines()
body_lines: list[str] = []
in_body = False
for ln in lines:
if not in_body:
# 第一个 ## 之前的所有内容都跳过
if ln.startswith("## "):
in_body = True
body_lines.append(ln)
continue
body_lines.append(ln)
full = header + "\n".join(body_lines).rstrip() + "\n"
out_md.write_text(full, encoding="utf-8")
print(f"[changelog] wrote {out_md} (v{version}, {build_date})")
if __name__ == "__main__":
if len(sys.argv) != 5:
sys.stderr.write(
"usage: gen_changelog_chapter.py <changelog.md> <out.md> <version> <build_date>\n"
)
sys.exit(1)
main(
pathlib.Path(sys.argv[1]),
pathlib.Path(sys.argv[2]),
sys.argv[3],
sys.argv[4],
)
plans/PRISM/tools/gen_stats.py
+271
View File
@@ -0,0 +1,271 @@
#!/usr/bin/env python3
"""
gen_stats.py
============
统计 PRISM 全书章节的:
- 中文字数 / 英文词数 / 标点数
- 各语言代码块行数
- Mermaid 图数 / 表格数 / 标题层级数
生成 16_stats.md(LaTeX longtable 版),供 build_book.sh 注入到 PDF 末尾。
用法:
python3 gen_stats.py <prism_dir> <out_md>
"""
from __future__ import annotations
import pathlib
import re
import sys
from collections import Counter, defaultdict
CJK_RE = re.compile(r"[\u4e00-\u9fff\u3000-\u303f\uff00-\uffef]")
EN_WORD_RE = re.compile(r"[A-Za-z][A-Za-z0-9_'-]*")
PUNCT_RE = re.compile(r"[，。;；:：!！?？、（）()\[\]【】「」“”\"']")
# 章节顺序(同 build_book.sh)
ORDER = [
"README.md",
"00_overview.md",
"01_capability_decomposition.md",
"02_architecture.md",
"03_data_schema.md",
"04_pipeline_A_iphone_offline.md",
"05_pipeline_B_relocalization.md",
"06_pipeline_C_online_perception.md",
"07_pipeline_D_consolidation.md",
"08_runtime_timeline.md",
"09_roadmap.md",
"10_tech_stack.md",
"11_world_model_bridge.md",
"12_risks.md",
"13_evaluation.md",
"14_mvp.md",
"15_glossary.md",
"18_lyra_inspirations.md",
]
def analyze(text: str) -> dict:
"""对一篇 md 做综合统计。先把代码块抠掉,然后再统计正文。"""
# 抠出代码块(``` 包围)
code_blocks: list[tuple[str, int]] = [] # (lang, line_count)
def _strip_code(m: re.Match) -> str:
fence_info = m.group(1) or ""
body = m.group(2)
lang = fence_info.strip().split()[0] if fence_info.strip() else "text"
# 行数 = 内容行数(不含闭合 ```)
n_lines = body.count("\n")
code_blocks.append((lang, n_lines))
return "\n[[CODE_REMOVED]]\n"
body = re.sub(r"```([^\n]*)\n(.*?)\n```", _strip_code, text, flags=re.S)
# 统计 mermaid 块(已被剥离,从 code_blocks 找)
n_mermaid = sum(1 for lang, _ in code_blocks if lang == "mermaid")
n_code_total = sum(n for lang, n in code_blocks if lang not in ("mermaid",))
# 各语言代码行数
lang_counter: Counter[str] = Counter()
for lang, n in code_blocks:
if lang == "mermaid":
continue
# 归一化
lang_norm = lang.lower() if lang else "text"
if lang_norm in ("py", "python3"):
lang_norm = "python"
if lang_norm in ("sh", "shell", "zsh"):
lang_norm = "bash"
lang_counter[lang_norm] += n
# 表格行(markdown |...| 风格)
n_table_rows = sum(1 for ln in body.splitlines()
if re.match(r"^\s*\|.+\|\s*$", ln))
# 标题
n_h1 = sum(1 for ln in body.splitlines() if ln.startswith("# "))
n_h2 = sum(1 for ln in body.splitlines() if ln.startswith("## "))
n_h3 = sum(1 for ln in body.splitlines() if ln.startswith("### "))
# 中英标点
cjk_chars = len(CJK_RE.findall(body))
en_words = len(EN_WORD_RE.findall(body))
punct = len(PUNCT_RE.findall(body))
return dict(
cjk_chars=cjk_chars,
en_words=en_words,
punct=punct,
n_mermaid=n_mermaid,
n_code_blocks=len(code_blocks) - n_mermaid,
n_code_lines=n_code_total,
lang_counter=lang_counter,
n_table_rows=n_table_rows,
n_h1=n_h1,
n_h2=n_h2,
n_h3=n_h3,
raw_lines=text.count("\n"),
)
def fmt_int(n: int) -> str:
return f"{n:,}"
def main(prism_dir: pathlib.Path, out_md: pathlib.Path) -> None:
total = defaultdict(int)
total_lang: Counter[str] = Counter()
per_chap: list[tuple[str, dict]] = []
for fn in ORDER:
p = prism_dir / fn
if not p.exists():
continue
text = p.read_text(encoding="utf-8")
s = analyze(text)
per_chap.append((fn, s))
for k, v in s.items():
if isinstance(v, int):
total[k] += v
elif isinstance(v, Counter):
total_lang.update(v)
# 生成 markdown(内嵌 LaTeX)
lines: list[str] = []
lines.append("# Chapter 16 — 文档与代码统计 (Stats)")
lines.append("")
lines.append("> 本章自动生成,反映本 PRISM 文档集在**构建时**的体量、结构与代码资产。每次 `bash build_book.sh` 都会刷新。")
lines.append("")
lines.append("## 16.1 总览")
lines.append("")
# ── 总览表
lines.append("\\begin{center}")
lines.append("\\begin{tabular}{@{}lr@{}}")
lines.append("\\toprule")
lines.append("\\textbf{指标} & \\textbf{数值} \\\\")
lines.append("\\midrule")
lines.append(f"章节数 & {fmt_int(len(per_chap))} \\\\")
lines.append(f"中文字数 & {fmt_int(total['cjk_chars'])} \\\\")
lines.append(f"英文词数 & {fmt_int(total['en_words'])} \\\\")
lines.append(f"标点数 & {fmt_int(total['punct'])} \\\\")
lines.append(f"Markdown 行数 & {fmt_int(total['raw_lines'])} \\\\")
lines.append(f"一级标题 (\\#) & {fmt_int(total['n_h1'])} \\\\")
lines.append(f"二级标题 (\\#\\#) & {fmt_int(total['n_h2'])} \\\\")
lines.append(f"三级标题 (\\#\\#\\#) & {fmt_int(total['n_h3'])} \\\\")
lines.append(f"Markdown 表格行数 & {fmt_int(total['n_table_rows'])} \\\\")
lines.append(f"Mermaid 图块数 & {fmt_int(total['n_mermaid'])} \\\\")
lines.append(f"代码块数(非 mermaid) & {fmt_int(total['n_code_blocks'])} \\\\")
lines.append(f"代码总行数 & {fmt_int(total['n_code_lines'])} \\\\")
lines.append("\\bottomrule")
lines.append("\\end{tabular}")
lines.append("\\end{center}")
lines.append("")
# ── 各章字数/行数 用宽列章节名(28%)+ 6 个等宽数值列(各 10%)
lines.append("## 16.2 各章字数与代码分布")
lines.append("")
lines.append("\\begin{longtable}{@{}")
lines.append(" >{\\raggedright\\arraybackslash}p{0.28\\linewidth}")
for _ in range(6):
lines.append(" >{\\raggedleft\\arraybackslash}p{0.10\\linewidth}")
lines.append("@{}}")
lines.append("\\toprule")
header = "\\textbf{章节} & \\textbf{中文字} & \\textbf{英文词} & \\textbf{md 行} & \\textbf{mmd 图} & \\textbf{代码块} & \\textbf{代码行} \\\\"
lines.append(header)
lines.append("\\midrule")
lines.append("\\endfirsthead")
lines.append("\\toprule")
lines.append(header)
lines.append("\\midrule")
lines.append("\\endhead")
lines.append("\\bottomrule")
lines.append("\\endfoot")
def short_label(fn: str) -> str:
stem = fn.replace(".md", "")
if stem == "README":
return "README"
m = re.match(r"(\d+)_", stem)
if m:
return f"ch{m.group(1)}"
return stem.replace("_", "\\_")
for fn, s in per_chap:
lines.append(
f"{short_label(fn)} & {fmt_int(s['cjk_chars'])} & {fmt_int(s['en_words'])} & "
f"{fmt_int(s['raw_lines'])} & {s['n_mermaid']} & {s['n_code_blocks']} & "
f"{fmt_int(s['n_code_lines'])} \\\\"
)
lines.append(
f"\\midrule\\textbf{{合计}} & \\textbf{{{fmt_int(total['cjk_chars'])}}} & "
f"\\textbf{{{fmt_int(total['en_words'])}}} & \\textbf{{{fmt_int(total['raw_lines'])}}} & "
f"\\textbf{{{total['n_mermaid']}}} & \\textbf{{{total['n_code_blocks']}}} & "
f"\\textbf{{{fmt_int(total['n_code_lines'])}}} \\\\"
)
lines.append("\\end{longtable}")
lines.append("")
lines.append("> 章节短名:`ch00` = `00\\_overview.md`,依此类推;`ch15` 缩略语表,`ch16` 即本章。")
lines.append("")
# ── 各语言代码行数
lines.append("## 16.3 代码块语言分布")
lines.append("")
lines.append("\\begin{center}")
lines.append("\\begin{tabular}{@{}lrr@{}}")
lines.append("\\toprule")
lines.append("\\textbf{语言} & \\textbf{代码块数} & \\textbf{代码行数} \\\\")
lines.append("\\midrule")
lang_block_counter: Counter[str] = Counter()
# 重算块数(分语言)
for _, s in per_chap:
# 这里需要拿原 lang_counter — 但 analyze() 没把"块数"按语言留下,只留了行数。
# 直接重新跑一遍代码块抽取(快速,正则一遍)
pass
# 重新统计每语言的"块数"(独立循环)
lang_block_n: Counter[str] = Counter()
for fn in ORDER:
p = prism_dir / fn
if not p.exists():
continue
for m in re.finditer(r"```([^\n]*)\n(.*?)\n```", p.read_text(encoding="utf-8"), flags=re.S):
lang = (m.group(1).strip() or "text").split()[0].lower()
if lang in ("py", "python3"):
lang = "python"
if lang in ("sh", "shell", "zsh"):
lang = "bash"
if lang == "mermaid":
continue
lang_block_n[lang] += 1
for lang, n_lines in total_lang.most_common():
lang_safe = lang.replace("_", "\\_")
n_blk = lang_block_n.get(lang, 0)
lines.append(f"{lang_safe} & {n_blk} & {fmt_int(n_lines)} \\\\")
lines.append("\\midrule")
lines.append(
f"\\textbf{{合计}} & \\textbf{{{sum(lang_block_n.values())}}} & "
f"\\textbf{{{fmt_int(sum(total_lang.values()))}}} \\\\"
)
lines.append("\\bottomrule")
lines.append("\\end{tabular}")
lines.append("\\end{center}")
lines.append("")
lines.append("---")
lines.append("")
lines.append("**章节版本**:auto-generated ")
lines.append("**生成时间**:由 `gen_stats.py` 在每次构建时即时计算 ")
lines.append("**关键收获**:文档不只是文字,它本身也是一份可量化的工程产物。")
lines.append("")
out_md.write_text("\n".join(lines), encoding="utf-8")
print(f"[stats] wrote {out_md}")
print(f" 中文 {total['cjk_chars']:,} 字, 英文 {total['en_words']:,} 词, "
f"代码 {total['n_code_lines']:,} 行, mermaid {total['n_mermaid']} 图")
if __name__ == "__main__":
if len(sys.argv) != 3:
sys.stderr.write("usage: gen_stats.py <prism_dir> <out_md>\n")
sys.exit(1)
main(pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]))
plans/PRISM/tools/make_cover_pdf.py
+324
View File
@@ -0,0 +1,324 @@
"""
Generate the A4 cover PDF for the PRISM project.
Output: plans/PRISM/PRISM_Cover.pdf
Layout (A4, 210 × 297 mm):
┌──────────────────────────────────────────┐
│ 彩色棱镜装饰 (左上 / 右下) │
│ │
│ 🌈 │
│ PRISM │
│ Prior-Registered Integrated │
│ Spatial Memory │
│ ──── 一行 tagline ──── │
│ │
│ [一句话摘要框] │
│ │
│ 项目介绍 (3-4 段) │
│ │
│ 版本 / 日期 / 作者 / 仓库 │
└──────────────────────────────────────────┘
"""
from __future__ import annotations
from pathlib import Path
from reportlab.lib.pagesizes import A4
from reportlab.lib.units import mm
from reportlab.lib.colors import HexColor, white, Color
from reportlab.pdfgen import canvas
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.platypus import Paragraph, Frame
from reportlab.lib.styles import ParagraphStyle
from reportlab.lib.enums import TA_LEFT, TA_CENTER
# ────────────── 元数据 ──────────────
PROJECT_NAME = "PRISM"
PROJECT_FULL = "Prior-Registered Integrated Spatial Memory"
PROJECT_SUBTITLE = "机器人大脑空间记忆架构"
TAGLINE = "融合 iPhone 先验地图 与 ZED 2i 在线感知 的四层空间记忆系统"
ONE_LINER = ("iPhone 是教科书,ZED 2i 是眼睛,"
"PRISM 大脑是一名既会查书又会观察、还会在书页空白处做笔记的学生。")
DESCRIPTION_PARAS = [
("<b>PRISM</b> 是一种面向室内服务机器人(酒店 / 家居 / 办公)的"
"<b>空间记忆架构</b>。它把<b>偶发的、高质量的离线扫描数据</b>"
"(iPhone RoomPlan + LiDAR)与<b>持续的、视锥级的在线感知数据</b>"
"(ZED 2i 双目+IMU)融合进同一套<b>四层记忆体系</b>:"
"L1 感知缓冲 → L2 度量 → L3 拓扑 → L4 语义,"
"并通过 <b>CLIP+ICP 配准</b>与<b>充电时的记忆巩固</b>形成闭环。"),
("<b>关键创新</b>:(1) 首次把消费级 iPhone RoomPlan 作为先验地图源;"
"(2) 分层而非分库,所有数据共享 SpatialMemory 单一根对象;"
"(3) 仿人脑睡眠的延迟巩固机制,避免单次错误观测污染长期记忆;"
"(4) 通过锚点持续校正 VIO 长时漂移。"),
("<b>文档结构</b>:15 个 Markdown 文件(README + 14 章),含可立即复制运行的 "
"Python schema、ROS 2 节点编排、8 周路线图、国产替代矩阵、"
"风险登记表、评测指标、以及一份周末可跑通的最小 MVP。"),
("<b>预期产出</b>:8 周原型 → 4 个月生产 · MTBF ≥ 48h · "
"冷启动重定位成功率 &gt; 90% · 任务成功率 &gt; 90%。"),
]
VERSION = "v1.1"
DATE = "2026-05-16"
AUTHOR = "项目规划组"
LICENSE = "CC BY-NC 4.0"
REPO = "github.com/<org>/prism (TBD)"
# ────────────── 字体注册(macOS 中文) ──────────────
def register_chinese_fonts() -> tuple[str, str]:
"""Returns (regular_font_name, bold_font_name)."""
candidates = [
# (regular_path, regular_subfontidx, bold_path, bold_subfontidx, name_base)
("/System/Library/Fonts/PingFang.ttc", 2,
"/System/Library/Fonts/PingFang.ttc", 5, "PingFang"),
("/System/Library/Fonts/STHeiti Light.ttc", 0,
"/System/Library/Fonts/STHeiti Medium.ttc", 0, "STHeiti"),
("/System/Library/Fonts/Hiragino Sans GB.ttc", 0,
"/System/Library/Fonts/Hiragino Sans GB.ttc", 1, "Hiragino"),
]
for reg, ri, bold, bi, base in candidates:
try:
pdfmetrics.registerFont(TTFont(f"{base}-R", reg, subfontIndex=ri))
pdfmetrics.registerFont(TTFont(f"{base}-B", bold, subfontIndex=bi))
return f"{base}-R", f"{base}-B"
except Exception as e:
print(f" 字体 {base} 注册失败: {e}")
continue
# 全失败:回退到内置 Helvetica(可能无法显示中文)
print(" ⚠️ 未找到中文字体,中文可能显示不全")
return "Helvetica", "Helvetica-Bold"
# ────────────── 色彩(棱镜光谱) ──────────────
PRISM_COLORS = [
HexColor("#E63946"), # red
HexColor("#F77F00"), # orange
HexColor("#FCBF49"), # yellow
HexColor("#2A9D8F"), # teal-green
HexColor("#1D7AB8"), # blue
HexColor("#5A189A"), # purple
]
INK_DARK = HexColor("#0E1116")
INK_MUTED = HexColor("#4A5560")
INK_BG = HexColor("#FBFAF7")
ACCENT = HexColor("#1D7AB8")
GOLD_LINE = HexColor("#C9A227")
def draw_prism_decoration(c: canvas.Canvas, page_w: float, page_h: float):
"""左上角:三角形棱镜分光示意 + 右下角:光谱条带"""
# ── 左上 棱镜三角形 + 入射/出射光 ──
cx, cy = 30 * mm, page_h - 35 * mm
size = 18 * mm
# 入射"白光"
c.setLineWidth(1.4); c.setStrokeColor(INK_DARK)
c.line(cx - size*1.5, cy + 2*mm, cx - 1*mm, cy + 1*mm)
# 三角棱镜
c.setFillColor(HexColor("#EAEAEA")); c.setStrokeColor(INK_DARK)
p = c.beginPath()
p.moveTo(cx, cy + size)
p.lineTo(cx + size*0.866, cy - size*0.5)
p.lineTo(cx - size*0.866, cy - size*0.5)
p.close()
c.drawPath(p, stroke=1, fill=1)
# 七色出射
for i, col in enumerate(PRISM_COLORS):
c.setStrokeColor(col); c.setLineWidth(1.6)
y_off = (i - len(PRISM_COLORS)/2 + 0.5) * 2.0 * mm
x_start = cx + size*0.6
x_end = cx + size*3.0
y_start = cy
y_end = cy + y_off
c.line(x_start, y_start, x_end, y_end)
# ── 右下 光谱条带(放在 footer 上方,避免与 footer 文字重叠)──
bar_w = 60 * mm
bar_h = 3 * mm
bx = page_w - 20*mm - bar_w
by = 21 * mm # 上移到 meta 与 footer 之间的留白处
seg = bar_w / len(PRISM_COLORS)
for i, col in enumerate(PRISM_COLORS):
c.setFillColor(col); c.setStrokeColor(col)
c.rect(bx + i*seg, by, seg, bar_h, fill=1, stroke=0)
def draw_letter_glyphs(c: canvas.Canvas, x: float, y: float,
letters: list[tuple[str, str]],
font: str, font_size: float, gap: float):
"""彩色大字母 P R I S M ,每个字母带颜色,下方写英文 token"""
for (letter, _), col, i in zip(letters, PRISM_COLORS, range(len(letters))):
# 这里我们让 P R I S M 取前 5 个颜色;Hex 不取第 6 个
pass
def main():
out = Path(__file__).resolve().parents[1] / "PRISM_Cover.pdf"
print(f"→ output: {out}")
reg, bold = register_chinese_fonts()
print(f" fonts: regular={reg}, bold={bold}")
page_w, page_h = A4
c = canvas.Canvas(str(out), pagesize=A4)
c.setTitle("PRISM — Prior-Registered Integrated Spatial Memory")
c.setAuthor(AUTHOR)
c.setSubject(PROJECT_SUBTITLE)
c.setKeywords("PRISM, spatial memory, robot, iPhone RoomPlan, ZED 2i, SLAM, world model")
c.setCreator("PRISM make_cover_pdf.py")
# ── 背景纸色 ──
c.setFillColor(INK_BG)
c.rect(0, 0, page_w, page_h, fill=1, stroke=0)
# ── 装饰 ──
draw_prism_decoration(c, page_w, page_h)
# ── 顶部小标签 ──
c.setFont(reg, 9)
c.setFillColor(INK_MUTED)
c.drawString(20*mm, page_h - 15*mm, "ROBOT · SPATIAL · MEMORY")
c.drawRightString(page_w - 20*mm, page_h - 15*mm,
f"VERSION {VERSION} · {DATE}")
# ── 主标题 PRISM ──
title_y = page_h - 75 * mm
c.setFont(bold, 78)
title = "PRISM"
# 等距彩色 5 个字母
spectrum5 = PRISM_COLORS[:5]
title_total_w = c.stringWidth(title, bold, 78)
start_x = (page_w - title_total_w) / 2
cur_x = start_x
for ch, col in zip(title, spectrum5):
c.setFillColor(col)
c.drawString(cur_x, title_y, ch)
cur_x += c.stringWidth(ch, bold, 78)
# ── 副标题英文全称 ──
c.setFont(bold, 14)
c.setFillColor(INK_DARK)
sub_en = PROJECT_FULL
c.drawCentredString(page_w/2, title_y - 13*mm, sub_en)
# ── 中文副标题 ──
c.setFont(reg, 16)
c.setFillColor(INK_DARK)
c.drawCentredString(page_w/2, title_y - 23*mm, PROJECT_SUBTITLE)
# ── 分隔线 + tagline ──
rule_y = title_y - 33*mm
c.setStrokeColor(GOLD_LINE); c.setLineWidth(0.8)
rule_half = 60*mm
c.line(page_w/2 - rule_half, rule_y, page_w/2 + rule_half, rule_y)
c.setFont(reg, 11); c.setFillColor(INK_MUTED)
c.drawCentredString(page_w/2, rule_y - 7*mm, TAGLINE)
# ── 字母含义注解 P-R-I-S-M ──
legend_y = rule_y - 23*mm
legend_items = [
("P", "Prior"),
("R", "Registered"),
("I", "Integrated"),
("S", "Spatial"),
("M", "Memory"),
]
item_w = 28*mm
total_w = item_w * len(legend_items)
start_x = (page_w - total_w) / 2
for i, (letter, word) in enumerate(legend_items):
x = start_x + i*item_w + item_w/2
c.setFont(bold, 22); c.setFillColor(PRISM_COLORS[i])
c.drawCentredString(x, legend_y, letter)
c.setFont(reg, 9); c.setFillColor(INK_DARK)
c.drawCentredString(x, legend_y - 6*mm, word)
# ── 一句话摘要框 ──
box_y_top = legend_y - 18*mm
box_h = 28*mm
box_x = 25*mm
box_w = page_w - 50*mm
# 浅色背景
c.setFillColor(HexColor("#F0EEE6")); c.setStrokeColor(ACCENT)
c.setLineWidth(0.6)
c.roundRect(box_x, box_y_top - box_h, box_w, box_h, 4*mm, fill=1, stroke=1)
# 标签
c.setFont(bold, 9); c.setFillColor(ACCENT)
c.drawString(box_x + 6*mm, box_y_top - 6*mm, "一句话摘要 · ONE-LINER")
# 引言文字
quote_style = ParagraphStyle(
"quote", fontName=reg, fontSize=12.5, leading=18,
textColor=INK_DARK, alignment=TA_LEFT,
firstLineIndent=0, leftIndent=0, rightIndent=0)
para = Paragraph(f"&ldquo;{ONE_LINER}&rdquo;", quote_style)
frame = Frame(box_x + 6*mm, box_y_top - box_h + 3*mm,
box_w - 12*mm, box_h - 12*mm,
showBoundary=0, leftPadding=0, rightPadding=0,
topPadding=0, bottomPadding=0)
frame.addFromList([para], c)
# ── 项目介绍正文 ──
intro_y_top = box_y_top - box_h - 8*mm
intro_x = 25*mm
intro_w = page_w - 50*mm
intro_h = intro_y_top - 50*mm # 底部留 50mm 给元信息
body_style = ParagraphStyle(
"body", fontName=reg, fontSize=9.5, leading=14,
textColor=INK_DARK, alignment=TA_LEFT,
spaceAfter=5, firstLineIndent=0)
# 给底部 meta 块预留 45mm:meta 顶在 45mm,文字行底 40mm,值底 33mm
META_TOP = 45 * mm
INTRO_BOT = META_TOP + 5*mm # intro frame 底部留 5mm 空白
intro_h = intro_y_top - INTRO_BOT
paragraphs = [Paragraph(p, body_style) for p in DESCRIPTION_PARAS]
frame = Frame(intro_x, INTRO_BOT,
intro_w, intro_h,
showBoundary=0,
leftPadding=0, rightPadding=0,
topPadding=0, bottomPadding=0)
frame.addFromList(paragraphs, c)
# ── 底部元信息块 ──
meta_y = 40 * mm # 标签行
val_y = meta_y - 5.5*mm # 值行
c.setStrokeColor(INK_MUTED); c.setLineWidth(0.4)
c.line(20*mm, META_TOP, page_w - 20*mm, META_TOP)
meta_items = [
("VERSION", VERSION),
("DATE", DATE),
("AUTHOR", AUTHOR),
("LICENSE", LICENSE),
]
col_w = (page_w - 40*mm) / len(meta_items)
for i, (k, v) in enumerate(meta_items):
x = 20*mm + i*col_w
c.setFont(bold, 8); c.setFillColor(INK_MUTED)
c.drawString(x, meta_y, k)
c.setFont(reg, 10); c.setFillColor(INK_DARK)
c.drawString(x, val_y, v)
# ── 底部 footer(左:仓库;右:slogan;两者避开右下光谱条所在的 21mm 处)──
c.setFont(reg, 7.5); c.setFillColor(INK_MUTED)
c.drawString(20*mm, 12*mm,
"plans/PRISM/ · 15 documents (README + Ch.00–14)")
c.drawString(20*mm, 8*mm,
"Repo: " + REPO)
# 右侧 slogan 放在光谱条上方(避免覆盖)
c.setFont(reg, 8); c.setFillColor(INK_MUTED)
c.drawRightString(page_w - 20*mm, 12*mm,
"Prism splits perception into memory.")
c.showPage()
c.save()
print(f"✅ done: {out} ({out.stat().st_size/1024:.1f} KB)")
if __name__ == "__main__":
main()
plans/PRISM/tools/package-lock.json
Generated
+4015
View File
File diff suppressed because it is too large Load Diff
plans/PRISM/tools/package.json
+15
View File
@@ -0,0 +1,15 @@
{
"name": "tools",
"version": "1.0.0",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
"author": "",
"license": "ISC",
"description": "",
"dependencies": {
"@mermaid-js/mermaid-cli": "^11.15.0"
}
}
plans/PRISM/tools/render_mermaid.py
+132
View File
@@ -0,0 +1,132 @@
#!/usr/bin/env python3
"""
render_mermaid.py
=================
扫描合并后的 markdown 中的 ```mermaid 代码块,用 mmdc(@mermaid-js/mermaid-cli)
渲染为 PNG,并把这些块替换为可被 pandoc 直接吃下的 LaTeX includegraphics 命令。
用法(被 build_book.sh 调用):
python3 render_mermaid.py <input_md> <out_md> <fig_dir>
特性:
- 内容 hash 缓存,二次构建不重复渲染
- 单个块渲染失败不会中断整体构建,会降级为 verbatim 源码 + 警告
- 详细错误信息打到 stderr 方便调试
依赖:
mmdc 必须在 plans/PRISM/tools/node_modules/.bin/mmdc
"""
from __future__ import annotations
import hashlib
import pathlib
import re
import subprocess
import sys
TOOLS_DIR = pathlib.Path(__file__).resolve().parent
MMDC = TOOLS_DIR / "node_modules" / ".bin" / "mmdc"
def render_one(code: str, fig_dir: pathlib.Path) -> pathlib.Path:
"""渲染一个 mermaid 代码块,基于内容 hash 缓存。"""
digest = hashlib.sha1(code.encode("utf-8")).hexdigest()[:12]
out_png = fig_dir / f"mmd_{digest}.png"
if out_png.exists():
return out_png
src_mmd = fig_dir / f"mmd_{digest}.mmd"
src_mmd.write_text(code, encoding="utf-8")
cmd = [
str(MMDC),
"-i", str(src_mmd),
"-o", str(out_png),
"-b", "white",
"-s", "2",
"--width", "1400",
]
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
# 抽取 mermaid 的 Parse / Lexical error 行
err_lines = [ln for ln in (proc.stdout + proc.stderr).splitlines()
if "Error" in ln or "error" in ln]
err_head = "\n ".join(err_lines[:5]) or "(no error line found)"
sys.stderr.write(
f"\n[mmdc ERROR] block {digest}:\n"
f" {err_head}\n"
f" --- mermaid source ---\n{code}\n"
f" ----------------------\n"
)
raise RuntimeError(f"mmdc failed on block {digest}")
return out_png
MERMAID_RE = re.compile(
r"```mermaid\s*\n(.*?)\n```",
re.DOTALL,
)
def process(input_md: pathlib.Path, out_md: pathlib.Path, fig_dir: pathlib.Path) -> int:
if not MMDC.exists():
sys.stderr.write(
f"[ERROR] mmdc not found at {MMDC}\n"
f" run: cd plans/PRISM/tools && npm install @mermaid-js/mermaid-cli\n"
)
sys.exit(2)
fig_dir.mkdir(parents=True, exist_ok=True)
text = input_md.read_text(encoding="utf-8")
ok = 0
fail = 0
failures: list[str] = []
def _sub(m: re.Match) -> str:
nonlocal ok, fail
code = m.group(1)
try:
png = render_one(code, fig_dir)
except RuntimeError:
fail += 1
digest = hashlib.sha1(code.encode("utf-8")).hexdigest()[:12]
failures.append(digest)
# 降级:渲染失败 → 在 PDF 中插入"渲染失败"标记 + 源码 verbatim,
# 这样书构建不会因为一个块挂掉而中断。
return (
f"\n\\begin{{quote}}\\textbf{{[mermaid 渲染失败 {digest}]}}"
f"\\end{{quote}}\n\n"
f"\\begin{{verbatim}}\n{code}\n\\end{{verbatim}}\n"
)
ok += 1
rel = png.relative_to(input_md.parent)
# 同时限制宽 (0.92 linewidth) 与高 (0.82 textheight),
# keepaspectratio 自动取较紧者,保证任何极端长宽比的图都不溢出 A4 页面。
return (
f"\n\\begin{{center}}\n"
f"\\includegraphics[width=0.92\\linewidth,"
f"height=0.82\\textheight,keepaspectratio]{{{rel.as_posix()}}}\n"
f"\\end{{center}}\n"
)
new_text = MERMAID_RE.sub(_sub, text)
out_md.write_text(new_text, encoding="utf-8")
if fail:
print(f"[mermaid] rendered {ok} ok, {fail} FAILED → 降级为 verbatim")
print(f"[mermaid] failed digests: {', '.join(failures)}")
else:
print(f"[mermaid] rendered {ok} block(s) → {fig_dir}")
return ok
if __name__ == "__main__":
if len(sys.argv) != 4:
sys.stderr.write(
"usage: render_mermaid.py <input_md> <out_md> <fig_dir>\n"
)
sys.exit(1)
process(
pathlib.Path(sys.argv[1]),
pathlib.Path(sys.argv[2]),
pathlib.Path(sys.argv[3]),
)