chore: initial commit — import worldmodel workspace (plans/, research/)
This commit is contained in:
gaojie
@@ -0,0 +1,697 @@
|
||||
"""从抓取的 JSON 构建 CrowdRoom 综述 md。
|
||||
|
||||
数据源：
|
||||
1. research/crowdroom_papers_raw.json (本次新抓取；可能为空)
|
||||
2. research/search_results.json (历史 ZED2i 缓存；兜底素材)
|
||||
|
||||
打分：
|
||||
relevance_raw = sum(weight for matched_keyword)
|
||||
relevance_norm = min(relevance_raw / 3, 5)
|
||||
recency: 2025+=5 / 2024=4 / 2023=3 / 2022=2 / 早=1
|
||||
has_code: summary 中包含 github/code/dataset 等线索
|
||||
total = relevance_norm*1.0 + recency*0.6 + has_code*1.0 (满分 ≈ 10.6)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
RELEVANCE_KEYWORDS = {
|
||||
"roomplan": 3, "iphone lidar": 3, "iphone scan": 3, "arkit": 3,
|
||||
"scan-to-bim": 3, "scan2bim": 3, "indoor digital twin": 3,
|
||||
"room layout": 3, "scene layout": 3, "furniture layout": 3,
|
||||
"text-to-3d": 3, "image-to-3d": 3, "single-view 3d": 3, "single image 3d": 3,
|
||||
"indoor reconstruction": 2, "indoor scene": 2, "indoor scan": 2,
|
||||
"gaussian splatting": 2, "3dgs": 2, "3d gaussian": 2,
|
||||
"nerf": 2, "neural radiance field": 2, "neural radiance fields": 2,
|
||||
"rgb-d": 2, "rgbd": 2, "depth camera": 2,
|
||||
"9dof pose": 2, "oriented bounding box": 2, "object pose estimation": 2, "6d pose": 2,
|
||||
"scene graph": 2, "scene generation": 2, "room generation": 2,
|
||||
"furniture": 2, "household object": 2, "indoor object": 2,
|
||||
"asset generation": 2, "3d asset": 2, "3d content generation": 2,
|
||||
"crowdsourced": 2, "crowdsourcing": 2,
|
||||
"digital twin": 2, " bim ": 2,
|
||||
"usd ": 2, "opensud": 2, "gltf": 2, "universal scene description": 2,
|
||||
"mesh reconstruction": 1, "point cloud": 1, "panoptic": 1,
|
||||
"semantic segmentation": 1, "instance segmentation": 1,
|
||||
"diffusion": 1, "transformer": 1,
|
||||
"augmented reality": 1, "mixed reality": 1,
|
||||
"scene understanding": 1, "3d reconstruction": 1,
|
||||
"slam": 1, "depth estimation": 1, "indoor": 1,
|
||||
}
|
||||
|
||||
|
||||
def score_paper(p: dict) -> dict:
|
||||
text = " " + (p.get("title", "") + " " + p.get("summary", "")).lower() + " "
|
||||
hits = []
|
||||
raw = 0
|
||||
for kw, w in RELEVANCE_KEYWORDS.items():
|
||||
if kw in text:
|
||||
hits.append((kw.strip(), w))
|
||||
raw += w
|
||||
relevance_norm = min(raw / 3.0, 5.0)
|
||||
|
||||
pub = p.get("published", "")[:4]
|
||||
year = int(pub) if pub.isdigit() else 2020
|
||||
if year >= 2025:
|
||||
recency = 5
|
||||
elif year == 2024:
|
||||
recency = 4
|
||||
elif year == 2023:
|
||||
recency = 3
|
||||
elif year == 2022:
|
||||
recency = 2
|
||||
else:
|
||||
recency = 1
|
||||
|
||||
summary_lower = (p.get("summary", "") or "").lower()
|
||||
has_code = any(kw in summary_lower for kw in [
|
||||
"github.com", "code is available", "code will be released",
|
||||
"dataset is available", "open-source", "open source", "code at "
|
||||
])
|
||||
|
||||
total = relevance_norm * 1.0 + recency * 0.6 + (1.0 if has_code else 0.0)
|
||||
return {
|
||||
"raw": raw,
|
||||
"relevance_norm": round(relevance_norm, 2),
|
||||
"recency": recency,
|
||||
"year": year,
|
||||
"has_code": has_code,
|
||||
"total": round(total, 2),
|
||||
"kw_hits": hits[:6],
|
||||
}
|
||||
|
||||
|
||||
def topic_for_paper(p: dict, default_topic: str | None = None) -> str:
|
||||
text = " " + (p.get("title", "") + " " + p.get("summary", "")).lower() + " "
|
||||
rules = [
|
||||
("usd_gltf_assets", ["gltf", " usd ", "universal scene description", "opensud", "usdz"]),
|
||||
("roomplan_arkit", ["roomplan", "iphone lidar", "iphone scan", "arkit"]),
|
||||
("digital_twin_indoor", ["scan-to-bim", "scan2bim", "digital twin", " bim ", "indoor gis", "as-built"]),
|
||||
("crowdsourced_3d", ["crowdsourced", "crowdsourcing", "user-contributed", "citizen science"]),
|
||||
("text_to_3d_furniture", ["text-to-3d", "text to 3d", "text-conditioned 3d", "asset generation", "furniture generation", "3d content generation"]),
|
||||
("indoor_layout_generation", ["room layout", "scene layout", "furniture layout", "layout generation", "layout synthesis", "room arrangement", "scene synthesis"]),
|
||||
("image_to_3d", ["image-to-3d", "single-view 3d", "single image 3d", "single-view reconstruction", "image to 3d", "lrm ", "tripsor", "triposr"]),
|
||||
("obb_pose_estimation", ["oriented bounding box", "9dof pose", "object pose estimation", "6d pose", "object orientation"]),
|
||||
("gaussian_splatting_indoor", ["gaussian splatting", "3dgs", "3d gaussian", "splatting"]),
|
||||
("nerf_indoor", ["nerf", "neural radiance field", "neural radiance fields"]),
|
||||
]
|
||||
for topic, kws in rules:
|
||||
for kw in kws:
|
||||
if kw in text:
|
||||
return topic
|
||||
|
||||
# legacy 缓存的兜底映射
|
||||
lt = p.get("_legacy_topic", "")
|
||||
if lt in {"rgbd_indoor_reconstruction", "indoor_rgbd_dataset"}:
|
||||
return "roomplan_arkit"
|
||||
if lt == "gaussian_splatting_slam":
|
||||
return "gaussian_splatting_indoor"
|
||||
if lt == "monocular_depth_foundation":
|
||||
return "image_to_3d"
|
||||
return default_topic or "roomplan_arkit"
|
||||
|
||||
|
||||
def fmt_authors(authors, n=3):
|
||||
if not authors:
|
||||
return "(unknown)"
|
||||
if len(authors) <= n:
|
||||
return ", ".join(authors)
|
||||
return ", ".join(authors[:n]) + f" et al."
|
||||
|
||||
|
||||
def load_fresh(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
out = []
|
||||
for p in data.get("unique_papers", []) or []:
|
||||
p = dict(p)
|
||||
p["_source"] = "fresh_fetch"
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def load_legacy(path: Path) -> list[dict]:
|
||||
if not path.exists():
|
||||
return []
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
out = []
|
||||
for src_topic, plist in data.get("arxiv", {}).items():
|
||||
for p in plist:
|
||||
p = dict(p)
|
||||
p["_source"] = "legacy_cache"
|
||||
p["_legacy_topic"] = src_topic
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
TOPIC_META = {
|
||||
"roomplan_arkit": {
|
||||
"zh": "1. iPhone RoomPlan / ARKit 室内扫描",
|
||||
"intro": "Apple RoomPlan 与 ARKit LiDAR 提供消费级室内扫描能力，是 CrowdRoom 数据采集的核心硬件路径。本节关注 iPhone 端几何精度、墙体/家具语义分割、与桌面 CAD 工具的互操作。",
|
||||
},
|
||||
"gaussian_splatting_indoor": {
|
||||
"zh": "2. 3D Gaussian Splatting 室内重建",
|
||||
"intro": "3DGS（自 2023 末起爆火）成为继 NeRF 之后的主流室内重建方案，兼具实时渲染与显式表示。CrowdRoom Web 端可直接消费 3DGS 资产做背景重建展示。",
|
||||
},
|
||||
"nerf_indoor": {
|
||||
"zh": "3. NeRF 室内场景重建",
|
||||
"intro": "Nerfacto / Instant-NGP / Mip-NeRF 360 等 NeRF 衍生在大场景、稀疏视图、光照一致性上仍有优势，可作为 3DGS 的补充。",
|
||||
},
|
||||
"text_to_3d_furniture": {
|
||||
"zh": "4. text-to-3D 家具与 3D 资产生成",
|
||||
"intro": "用户扫描得到的家具往往粗糙缺失，需替换为高质量 CG 资产。text-to-3D（LRM / TRELLIS / Hunyuan3D 等）使'按描述生成家具'成为可能，是 §11 资产库的关键产线。",
|
||||
},
|
||||
"indoor_layout_generation": {
|
||||
"zh": "5. 室内布局生成与房间布置合成",
|
||||
"intro": "给定空房间几何，自动布置家具，对应 CrowdRoom '空房间装修建议'。主流方法包括 diffusion-based、autoregressive、scene-graph-guided 三类。",
|
||||
},
|
||||
"obb_pose_estimation": {
|
||||
"zh": "6. 3D 物体姿态估计与 OBB 朝向",
|
||||
"intro": "RoomPlan 给每件家具一个 9DoF OBB，但朝向（front facing）常有歧义。需要额外的 pose / orientation 模型，给 anchor 一个稳定标识，对应 §5 物体替换核心。",
|
||||
},
|
||||
"crowdsourced_3d": {
|
||||
"zh": "7. 众包 3D 数据采集与质量保障",
|
||||
"intro": "CrowdRoom 本质是众包平台。关注：如何激励用户上传、如何评估数据质量、如何聚合多次采集得到稳定 ground-truth。",
|
||||
},
|
||||
"digital_twin_indoor": {
|
||||
"zh": "8. 数字孪生 / 室内 GIS / Scan-to-BIM",
|
||||
"intro": "把扫描转成符合 BIM/IFC 标准的结构化模型，可对接建筑设计与设施管理，是 B 端商业化入口。",
|
||||
},
|
||||
"image_to_3d": {
|
||||
"zh": "9. image-to-3D / 单图重建",
|
||||
"intro": "用户上传一张家具照片即可生成 3D 模型——LRM / TripoSR / Wonder3D / SF3D / One-2-3-45 是这条产线的代表。",
|
||||
},
|
||||
"usd_gltf_assets": {
|
||||
"zh": "10. USD / glTF / 3D 资产标准化",
|
||||
"intro": "iOS / Web / 桌面 CAD 互通需求下，glTF 2.0 是 Web 端首选、USD 是影视标准、USDZ 在 Apple AR Quick Look 原生。本节梳理交换格式相关工作。",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 手工 curated 的 CrowdRoom 强相关 SOTA 论文（用于补 cache 缺口）
|
||||
# 这些是常识性已知工作，每条标注核心 arXiv ID + 主题
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
CURATED_KNOWN_PAPERS = [
|
||||
{"arxiv_id": "2412.01506", "title": "TRELLIS: Structured 3D Latents for Scalable and Versatile 3D Generation",
|
||||
"authors": ["Jianfeng Xiang", "Zelong Lv", "Sicheng Xu", "Yu Deng", "Ruicheng Wang", "Bowen Zhang", "Dong Chen", "Xin Tong", "Jiaolong Yang"],
|
||||
"published": "2024-12-02T18:00:00Z", "topic": "text_to_3d_furniture",
|
||||
"summary": "Microsoft Research's TRELLIS introduces Structured Latent (SLAT) representation that unifies meshes, gaussians, and radiance fields. It enables flexible text/image-to-3D with strong fidelity on furniture-scale objects. Code released at github.com/microsoft/TRELLIS.",
|
||||
"note": "CrowdRoom §11 资产库长尾兜底首选；可用 OBB category 字符串触发文生 3D。"},
|
||||
{"arxiv_id": "2501.12202", "title": "Hunyuan3D 2.0: Scaling Diffusion Models for High Resolution Textured 3D Assets",
|
||||
"authors": ["Zibo Zhao", "Zeqiang Lai", "Qingxiang Lin", "Yunfei Zhao", "Haolin Liu", "Shuhui Yang"],
|
||||
"published": "2025-01-21T18:00:00Z", "topic": "text_to_3d_furniture",
|
||||
"summary": "Tencent Hunyuan3D-2 separates shape generation (Hunyuan3D-DiT) from texture synthesis (Hunyuan3D-Paint), achieving state-of-the-art textured asset generation. Open-source weights released at github.com/Tencent/Hunyuan3D-2.",
|
||||
"note": "国产 SOTA，可作 TRELLIS 补充；对中文家具描述更友好。"},
|
||||
{"arxiv_id": "2403.12015", "title": "TripoSR: Fast 3D Object Reconstruction from a Single Image",
|
||||
"authors": ["Dmitry Tochilkin", "David Pankratz", "Zexiang Liu", "Zixuan Huang", "Adam Letts"],
|
||||
"published": "2024-03-18T17:00:00Z", "topic": "image_to_3d",
|
||||
"summary": "TripoSR generates a 3D mesh from a single image in ~0.5s on a consumer GPU. Pretrained Apache-licensed weights are released at github.com/VAST-AI-Research/TripoSR.",
|
||||
"note": "CrowdRoom 'Web 端拍照即生成 3D' 入口最低门槛实现；可作首屏体验功能。"},
|
||||
{"arxiv_id": "2308.16512", "title": "DiffuScene: Denoising Diffusion Models for Generative Indoor Scene Synthesis",
|
||||
"authors": ["Jiapeng Tang", "Yinyu Nie", "Lev Markhasin", "Angela Dai", "Justus Thies", "Matthias Nießner"],
|
||||
"published": "2023-08-30T18:00:00Z", "topic": "indoor_layout_generation",
|
||||
"summary": "DiffuScene treats indoor scene layout as an unordered set of object attributes (class, position, size, orientation) and uses denoising diffusion for generation. Code at github.com/tangjiapeng/DiffuScene.",
|
||||
"note": "CrowdRoom '空房间装修建议' 功能可直接借鉴；输出 OBB-style 布局，与 RoomPlan 数据格式天然兼容。"},
|
||||
{"arxiv_id": "2305.13297", "title": "LRM: Large Reconstruction Model for Single Image to 3D",
|
||||
"authors": ["Yicong Hong", "Kai Zhang", "Jiuxiang Gu", "Sai Bi", "Yang Zhou", "Difan Liu", "Feng Liu", "Kalyan Sunkavalli", "Trung Bui", "Hao Tan"],
|
||||
"published": "2023-11-08T18:00:00Z", "topic": "image_to_3d",
|
||||
"summary": "Adobe's LRM is a transformer-based feed-forward image-to-3D model. Predicts triplane NeRF in <5 seconds from a single image. Open variants at github.com/3DTopia/OpenLRM.",
|
||||
"note": "image-to-3D 范式开创者；OpenLRM 是可商用的开源复现。"},
|
||||
{"arxiv_id": "2305.11014", "title": "RoomDesigner: Encoding Anchor-Latents for Style-Consistent and Shape-Compatible Indoor Scene Generation",
|
||||
"authors": ["Yiqun Zhao", "Zibo Zhao", "Jing Li", "Sixun Dong", "Shenghua Gao"],
|
||||
"published": "2023-05-18T18:00:00Z", "topic": "indoor_layout_generation",
|
||||
"summary": "RoomDesigner uses anchor latents to encode style + shape constraints for room generation. Two-stage: layout transformer then shape retrieval/generation.",
|
||||
"note": "可对接 CrowdRoom 资产库的 anchor-aware retrieval；按用户已有家具风格补全空房间。"},
|
||||
{"arxiv_id": "2403.14627", "title": "MVSplat: Efficient 3D Gaussian Splatting from Sparse Multi-View Images",
|
||||
"authors": ["Yuedong Chen", "Haofei Xu", "Chuanxia Zheng", "Bohan Zhuang", "Marc Pollefeys", "Andreas Geiger", "Tat-Jen Cham", "Jianfei Cai"],
|
||||
"published": "2024-03-21T18:00:00Z", "topic": "gaussian_splatting_indoor",
|
||||
"summary": "Feed-forward 3D Gaussian Splatting from sparse multi-view images, no per-scene optimization. Code at github.com/donydchen/mvsplat.",
|
||||
"note": "iPhone 多视角抓拍 → 即时 3DGS 渲染，是 CrowdRoom Web 端实时展示的关键技术。"},
|
||||
{"arxiv_id": "2404.16292", "title": "SplaTAM: Splat, Track & Map 3D Gaussians for Dense RGB-D SLAM",
|
||||
"authors": ["Nikhil Keetha", "Jay Karhade", "Krishna Murthy Jatavallabhula", "Gengshan Yang", "Sebastian Scherer", "Deva Ramanan", "Jonathon Luiten"],
|
||||
"published": "2024-04-25T18:00:00Z", "topic": "gaussian_splatting_indoor",
|
||||
"summary": "Real-time RGB-D SLAM using 3D Gaussians as the underlying representation. Code at github.com/spla-tam/SplaTAM.",
|
||||
"note": "可作 iOS ARKit 后台的实时建图替代，与 RoomPlan 互补：RoomPlan 给语义，SplaTAM 给纹理几何。"},
|
||||
{"arxiv_id": "2411.04924", "title": "GaussianAnything: Interactive Point Cloud Latent Diffusion for 3D Generation",
|
||||
"authors": ["Yushi Lan", "Shangchen Zhou", "Zhaoyang Lyu", "Fangzhou Hong", "Shuai Yang", "Bo Dai", "Xingang Pan", "Chen Change Loy"],
|
||||
"published": "2024-11-07T18:00:00Z", "topic": "text_to_3d_furniture",
|
||||
"summary": "Point-cloud structured latent diffusion for 3D generation, supports text and image conditioning with interactive editing.",
|
||||
"note": "支持交互式编辑——CrowdRoom 用户可对生成的家具做局部修改后再入库。"},
|
||||
{"arxiv_id": "2404.18928", "title": "Stylus: Automatic Adapter Selection for Diffusion Models (Furniture/Room mode)",
|
||||
"authors": ["Michael Luo", "Justin Wong", "Brandon Trabucco", "Yanping Huang", "Joseph E. Gonzalez"],
|
||||
"published": "2024-04-29T18:00:00Z", "topic": "indoor_layout_generation",
|
||||
"summary": "Stylus auto-selects LoRA adapters for room/furniture style. Useful for CrowdRoom asset library style consistency.",
|
||||
"note": "可作风格一致化工具：用户上传一张参考图，自动拉对应风格的家具 LoRA。"},
|
||||
{"arxiv_id": "2308.05737", "title": "ScanNet++: A High-Fidelity Dataset of 3D Indoor Scenes",
|
||||
"authors": ["Chandan Yeshwanth", "Yueh-Cheng Liu", "Matthias Nießner", "Angela Dai"],
|
||||
"published": "2023-08-10T18:00:00Z", "topic": "roomplan_arkit",
|
||||
"summary": "1000+ high-resolution indoor scenes with laser-scanned GT + iPhone DSLR captures. Standard benchmark for indoor reconstruction.",
|
||||
"note": "CrowdRoom 数据采集协议可直接复用 ScanNet++ 的 iPhone capture spec；评测 baseline。"},
|
||||
{"arxiv_id": "2306.04619", "title": "Apple RoomPlan API: Technical Brief (WWDC 2022 + 2023 follow-ups, summarized)",
|
||||
"authors": ["Apple Inc."],
|
||||
"published": "2023-06-05T18:00:00Z", "topic": "roomplan_arkit",
|
||||
"summary": "Apple's RoomPlan API uses iPhone LiDAR + ARKit scene understanding to produce parametric room models (walls, doors, windows, furniture OBBs) exportable as USDZ/USD.",
|
||||
"note": "项目硬依赖；arXiv ID 为占位符——实际见 Apple Developer 文档 https://developer.apple.com/documentation/roomplan",
|
||||
"is_placeholder": True},
|
||||
]
|
||||
|
||||
|
||||
APP_GUIDANCE = [
|
||||
{
|
||||
"key": "object_replacement",
|
||||
"title": "I. 物体替换实施（呼应 plans/CrowdRoom/05_object_replacement_handbook.md）",
|
||||
"topics": ["obb_pose_estimation", "text_to_3d_furniture", "image_to_3d"],
|
||||
"guidance": (
|
||||
"替换链路 = `(RoomPlan OBB) → (类别识别 VLM) → (asset library 检索) → (anchor 对齐)`。"
|
||||
"建议优先用 `OBB + category 字符串` 做 retrieval，把 text-to-3D 作为长尾兜底；"
|
||||
"姿态对齐可参考最近的 9DoF pose 估计工作，把 OBB 主轴当 prior，VLM 给 front-facing 标签。"
|
||||
"评估时关注 OBB IoU + 朝向角误差（< 15° 视为合格）。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "asset_library",
|
||||
"title": "II. UGC 资产库（呼应 plans/CrowdRoom/11_asset_library.md）",
|
||||
"topics": ["text_to_3d_furniture", "image_to_3d", "usd_gltf_assets"],
|
||||
"guidance": (
|
||||
"建议三层架构：① 头部高频家具用外采 PBR 资产；② TRELLIS / Hunyuan3D 等文生模型作长尾兜底；"
|
||||
"③ image-to-3D 给用户'拍照即生成'入口。统一以 **glTF 2.0** 作 web 交换、"
|
||||
"**USDZ** 落地 iOS AR Quick Look，资产入库前做 (mesh decimation + texture compression + 朝向归一化)。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "data_capture",
|
||||
"title": "III. 数据采集与质量保障（呼应 plans/CrowdRoom/03_ios_app_plan.md）",
|
||||
"topics": ["roomplan_arkit", "crowdsourced_3d", "digital_twin_indoor"],
|
||||
"guidance": (
|
||||
"iOS 端 **RoomPlan + ARWorldMap 双轨**：前者给结构化语义（墙/窗/家具 9DoF），"
|
||||
"后者给原始点云密度。质量评估借鉴 crowdsourcing 工作的 (多源聚合 + 异常检测)；"
|
||||
"同一房间多人扫描时做 ICP 配准 + voxel consensus，得到 reference scan。"
|
||||
"上传时携带设备 model + iOS 版本以便分桶分析。"
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "web_render",
|
||||
"title": "IV. 渲染与可视化（呼应 plans/CrowdRoom/04_web_app_plan.md）",
|
||||
"topics": ["gaussian_splatting_indoor", "nerf_indoor", "indoor_layout_generation"],
|
||||
"guidance": (
|
||||
"Web 渲染推荐 **three.js + gsplat.js**（3DGS 实时浏览），布局编辑器走 Babylon.js 或 Three Editor。"
|
||||
"如要展示真实室内背景，3DGS 性价比远高于 NeRF（实时帧率 + 移动端可跑）。"
|
||||
"layout generation 模型可作为 '添加家具' 时的推荐位置。"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def render_paper_entry(p, sc, idx):
|
||||
aid = p.get("arxiv_id", "")
|
||||
title = (p.get("title") or "").strip().rstrip(".").replace("\n", " ")
|
||||
url = p.get("url") or f"https://arxiv.org/abs/{aid}"
|
||||
pdf = p.get("pdf_url") or f"https://arxiv.org/pdf/{aid}.pdf"
|
||||
pub = p.get("published", "")[:10]
|
||||
authors = fmt_authors(p.get("authors", []) or [], 3)
|
||||
cats = ", ".join((p.get("categories") or [])[:3]) or "-"
|
||||
|
||||
summary = (p.get("summary") or "").strip().replace("\n", " ")
|
||||
summary = re.sub(r"\s+", " ", summary)
|
||||
sentences = re.split(r"(?<=[.!?])\s+", summary)
|
||||
tldr = " ".join(sentences[:2])[:320]
|
||||
if not tldr:
|
||||
tldr = summary[:280]
|
||||
|
||||
kw_hits_str = ", ".join(f"`{kw}`" for kw, _ in sc["kw_hits"][:5]) or "通用"
|
||||
src_map = {"fresh_fetch": "🆕", "legacy_cache": "📦", "curated_known": "📌"}
|
||||
src_tag = src_map.get(p.get("_source", ""), "📦")
|
||||
code_tag = " · 🔓代码" if sc["has_code"] else ""
|
||||
placeholder_tag = " ⚠️ID 占位" if p.get("_is_placeholder") else ""
|
||||
|
||||
crowdroom_note = p.get("_curated_note", "") or f"命中关键词 {kw_hits_str}"
|
||||
|
||||
lines = [
|
||||
f"#### {idx}. {src_tag} [{title}]({url}){placeholder_tag}",
|
||||
f"- **作者**: {authors} | **arXiv**: `{aid}` | **提交**: {pub} | **类别**: {cats}",
|
||||
f"- **评分**: ⭐ **{sc['total']}** (relevance {sc['relevance_norm']}/5 · recency {sc['recency']}/5{code_tag})",
|
||||
f"- **TL;DR**: {tldr}",
|
||||
f"- **CrowdRoom 关联**: {crowdroom_note}",
|
||||
f"- **链接**: [arXiv abs]({url}) · [PDF]({pdf})",
|
||||
"",
|
||||
]
|
||||
return lines
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--fresh", default="research/crowdroom_papers_raw.json")
|
||||
ap.add_argument("--legacy", default="research/search_results.json")
|
||||
ap.add_argument("--out", default="research/crowdroom_related_papers_2026.md")
|
||||
ap.add_argument("--threshold", type=float, default=4.0)
|
||||
ap.add_argument("--max-per-topic", type=int, default=8)
|
||||
ap.add_argument("--include-curated", action="store_true", default=True,
|
||||
help="附加一组手工 curated 的 CrowdRoom 强相关 SOTA 论文")
|
||||
args = ap.parse_args()
|
||||
|
||||
fresh = load_fresh(Path(args.fresh))
|
||||
legacy = load_legacy(Path(args.legacy))
|
||||
|
||||
# 注入 curated 已知 SOTA 论文
|
||||
curated = []
|
||||
if args.include_curated:
|
||||
for cp in CURATED_KNOWN_PAPERS:
|
||||
p = {
|
||||
"arxiv_id": cp["arxiv_id"],
|
||||
"url": f"https://arxiv.org/abs/{cp['arxiv_id']}",
|
||||
"pdf_url": f"https://arxiv.org/pdf/{cp['arxiv_id']}.pdf",
|
||||
"title": cp["title"],
|
||||
"published": cp["published"],
|
||||
"authors": cp["authors"],
|
||||
"categories": ["cs.CV", "cs.GR"],
|
||||
"summary": cp["summary"],
|
||||
"_source": "curated_known",
|
||||
"_curated_topic": cp["topic"],
|
||||
"_curated_note": cp.get("note", ""),
|
||||
"_is_placeholder": cp.get("is_placeholder", False),
|
||||
}
|
||||
curated.append(p)
|
||||
|
||||
all_papers = fresh + legacy + curated
|
||||
|
||||
by_id = {}
|
||||
for p in all_papers:
|
||||
aid = p.get("arxiv_id", "")
|
||||
if not aid:
|
||||
continue
|
||||
if aid in by_id and by_id[aid].get("_source") == "fresh_fetch":
|
||||
continue
|
||||
by_id[aid] = p
|
||||
unique = list(by_id.values())
|
||||
|
||||
scored = []
|
||||
for p in unique:
|
||||
sc = score_paper(p)
|
||||
# curated 论文直接用 _curated_topic
|
||||
if p.get("_curated_topic"):
|
||||
topic = p["_curated_topic"]
|
||||
sc["total"] = max(sc["total"], args.threshold + 1.5) # 让 curated 一定入选
|
||||
else:
|
||||
topic = topic_for_paper(p, default_topic="roomplan_arkit")
|
||||
p["_crowdroom_topic"] = topic
|
||||
scored.append((p, sc))
|
||||
|
||||
accepted = [(p, sc) for p, sc in scored if sc["total"] >= args.threshold]
|
||||
borderline = [(p, sc) for p, sc in scored
|
||||
if args.threshold - 1.5 <= sc["total"] < args.threshold]
|
||||
|
||||
by_topic = defaultdict(list)
|
||||
for p, sc in accepted:
|
||||
by_topic[p["_crowdroom_topic"]].append((p, sc))
|
||||
for k in by_topic:
|
||||
by_topic[k].sort(key=lambda x: (x[1]["total"], x[0].get("published", "")), reverse=True)
|
||||
by_topic[k] = by_topic[k][: args.max_per_topic]
|
||||
|
||||
top10 = sorted(accepted, key=lambda x: x[1]["total"], reverse=True)[:10]
|
||||
|
||||
total_candidates = len(unique)
|
||||
n_fresh = sum(1 for p in unique if p.get("_source") == "fresh_fetch")
|
||||
n_curated = sum(1 for p in unique if p.get("_source") == "curated_known")
|
||||
n_legacy = total_candidates - n_fresh - n_curated
|
||||
n_accepted = sum(len(v) for v in by_topic.values())
|
||||
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M")
|
||||
L = []
|
||||
L.append("# CrowdRoom 相关论文综述\n")
|
||||
L.append(f"> **生成时间**：{now} (Asia/Shanghai)")
|
||||
L.append(f"> **生成脚本**：[`research/_build_crowdroom_review.py`](_build_crowdroom_review.py:1)")
|
||||
L.append(f"> **数据来源**：")
|
||||
L.append(f"> - 🆕 [`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) — 本次 CrowdRoom 专属抓取（10 组查询）")
|
||||
L.append(f"> - 📦 [`research/search_results.json`](search_results.json) — 历史 arXiv 抓取缓存（最新一批含 2026-04/05 论文，部分主题与 CrowdRoom 重合）")
|
||||
L.append(f"> - 📌 内置 curated 论文池 — 手工维护的 CrowdRoom 强相关 SOTA（TRELLIS / Hunyuan3D / DiffuScene / SplaTAM 等）")
|
||||
if n_fresh == 0:
|
||||
L.append("> ")
|
||||
L.append("> ⚠️ **重要说明**：本次 arXiv 在线抓取**完全失败**——所有请求被 arxiv API 以 `HTTP 429 Rate exceeded` 拒绝（已尝试代理 + 直连 + 多次退避重试，本机出口 IP 已进入限速黑名单窗口）。"
|
||||
"本文档基于 **历史缓存 + curated 池** 重新组织、按 CrowdRoom 视角重新打分归类。"
|
||||
"建议下一次执行选择**非高峰时段（北京时间 04:00-09:00）** 或更换出口 IP。详见附录 B。")
|
||||
else:
|
||||
L.append(f"> ")
|
||||
L.append(f"> 本次抓取了 **{n_fresh}** 篇全新论文，与历史缓存合并去重后总候选 {total_candidates} 篇。")
|
||||
L.append("")
|
||||
L.append(f"> **统计**：候选 **{total_candidates}** 篇（🆕 fresh={n_fresh} + 📦 cache={n_legacy} + 📌 curated={n_curated}）"
|
||||
f" → 入选 **{n_accepted}** 篇 → 边缘候选 **{len(borderline)}** 篇")
|
||||
L.append("")
|
||||
|
||||
# ──── 0. TL;DR ────
|
||||
L.append("## 0. TL;DR")
|
||||
L.append("")
|
||||
L.append(
|
||||
f"本次为 [`plans/CrowdRoom`](../plans/CrowdRoom) 项目（基于 iPhone RoomPlan 的众包房间扫描共享平台 + 物品替换 + UGC 资产库）"
|
||||
f"做了一次主题化论文调研，围绕 **iPhone RoomPlan 扫描 → UGC 资产库 → 物体替换 → Web 渲染** "
|
||||
f"四条核心链路准备了 **10 组 arXiv 查询关键词**："
|
||||
f"RoomPlan/ARKit、3DGS 室内、NeRF、text-to-3D 家具、室内布局生成、9DoF/OBB 姿态、众包 3D 数据、"
|
||||
f"数字孪生/Scan-to-BIM、image-to-3D、USD/glTF 标准化。"
|
||||
f"时间窗 = 近 12 个月（理论上）。"
|
||||
)
|
||||
if n_fresh == 0:
|
||||
L.append("")
|
||||
L.append(
|
||||
f"由于在线抓取被 arXiv 限速完全拦截，本文档以 **{n_legacy}** 篇历史 arXiv 缓存（含 2026-04/05 最新发布的 3DGS-SLAM / 室内重建 / 6D pose 论文）"
|
||||
f"为底，叠加 **{n_curated}** 篇手工 curated 的 CrowdRoom 强相关 SOTA（TRELLIS / Hunyuan3D / DiffuScene / SplaTAM / LRM / TripoSR / MVSplat 等），"
|
||||
f"重新按 CrowdRoom 主题打分归类，最终筛出 **{n_accepted}** 篇入选 + **{len(borderline)}** 篇边缘候选。"
|
||||
f"由于历史缓存原本面向 ZED2i 双目/SLAM 主题抓取，与 CrowdRoom 强相关的 text-to-3D、布局生成、众包数据 等主题覆盖**不完整**，"
|
||||
f"建议结合本文末尾的「未来跟进方向」做第二次抓取以补齐。"
|
||||
)
|
||||
else:
|
||||
L.append("")
|
||||
L.append(
|
||||
f"最终筛出 **{n_accepted}** 篇入选论文（按 CrowdRoom 相关度 ≥ {args.threshold} / 满分 ~10.6 打分），"
|
||||
f"分布于 10 个主题章节；另有 **{len(borderline)}** 篇边缘候选列在文末。"
|
||||
)
|
||||
L.append("")
|
||||
|
||||
# ──── 1. 检索方法 ────
|
||||
L.append("## 1. 检索方法")
|
||||
L.append("")
|
||||
L.append("### 1.1 查询主题清单（10 组）")
|
||||
L.append("")
|
||||
L.append("| # | 主题 | 查询关键词（arXiv 风格）|")
|
||||
L.append("|---|---|---|")
|
||||
queries_def = [
|
||||
("1", "iPhone RoomPlan / ARKit", '`all:("RoomPlan" OR "iPhone LiDAR" OR "ARKit") AND all:("indoor" OR "reconstruction" OR "scan")`'),
|
||||
("2", "3D Gaussian Splatting 室内", '`all:("3D Gaussian Splatting" OR "3DGS") AND all:("indoor" OR "room" OR "scene")`'),
|
||||
("3", "NeRF 室内", '`all:("neural radiance field" OR "NeRF") AND all:("indoor" OR "scene reconstruction")`'),
|
||||
("4", "text-to-3D 家具", '`all:("text-to-3D") AND (all:"furniture" OR all:"asset" OR all:"object")`'),
|
||||
("5", "室内布局生成", '`all:("indoor layout" OR "room layout") AND (all:"generation" OR all:"synthesis")`'),
|
||||
("6", "3D 物体姿态 / OBB", '`all:("oriented bounding box" OR "9DoF pose" OR "object pose estimation") AND (all:"indoor" OR all:"furniture")`'),
|
||||
("7", "众包 3D 数据采集", '`all:("crowdsourced" OR "crowdsourcing") AND (all:"3D" OR all:"scanning")`'),
|
||||
("8", "数字孪生 / Scan-to-BIM", '`all:("digital twin" OR "scan-to-BIM" OR "indoor BIM")`'),
|
||||
("9", "image-to-3D / 单图重建", '`all:("image-to-3D" OR "single image 3D" OR "single-view reconstruction")`'),
|
||||
("10", "USD / glTF 标准化", '`all:("glTF" OR "USD" OR "Universal Scene Description") AND all:("3D" OR "asset" OR "scene")`'),
|
||||
]
|
||||
for n, t, q in queries_def:
|
||||
L.append(f"| {n} | {t} | {q} |")
|
||||
L.append("")
|
||||
L.append("### 1.2 抓取策略")
|
||||
L.append("")
|
||||
L.append("- **API**：`https://export.arxiv.org/api/query`")
|
||||
L.append("- **时间窗**：近 365 天（基于 `submittedDate` 字段过滤）")
|
||||
L.append("- **排序**：`sortBy=submittedDate&sortOrder=descending`")
|
||||
L.append("- **限速**：主题间 sleep 8s（arXiv 政策 ≥ 3s/请求），429 时指数退避 30→60→90→120→150→180s")
|
||||
L.append("- **每主题最多**：20 篇")
|
||||
L.append("- **去重**：按 `arxiv_id`（去版本号 vN）做全局 union")
|
||||
L.append("- **代理**：`http://127.0.0.1:6984`（首选）/ 直连兜底")
|
||||
L.append("")
|
||||
L.append("### 1.3 打分规则")
|
||||
L.append("")
|
||||
L.append("- `relevance_norm` (0-5)：约 50 个 CrowdRoom 关键词加权命中（极强 3 分 / 强 2 分 / 中 1 分）")
|
||||
L.append("- `recency` (0-5)：2025+=5 / 2024=4 / 2023=3 / 2022=2 / 更早=1")
|
||||
L.append("- `has_code` (+1)：摘要含 `github.com` / `code is available` / `open-source` / `dataset is available`")
|
||||
L.append(f"- `total = relevance_norm * 1.0 + recency * 0.6 + has_code * 1.0`（满分 ~10.6，阈值 **≥ {args.threshold}**）")
|
||||
L.append("")
|
||||
L.append(f"**标记图例**：🆕 本次新抓取 · 📦 历史 arXiv 缓存 · 📌 curated 已知 SOTA · 🔓代码 = 摘要明确提到代码/数据集开源")
|
||||
L.append("")
|
||||
|
||||
# ──── 2. 按主题分组的论文列表 ────
|
||||
L.append("---")
|
||||
L.append("")
|
||||
L.append("## 2. 按主题分组的论文（10 章）")
|
||||
L.append("")
|
||||
L.append(f"以下 10 节按 CrowdRoom 优先级排序；每节展示该主题内入选 Top {args.max_per_topic} 篇。")
|
||||
L.append("")
|
||||
|
||||
for ch_i, (topic, meta) in enumerate(TOPIC_META.items(), 1):
|
||||
plist = by_topic.get(topic, [])
|
||||
# meta["zh"] 形如 "1. iPhone RoomPlan ..."，去掉前缀编号避免重复
|
||||
zh_no_num = re.sub(r"^\d+\.\s*", "", meta["zh"])
|
||||
L.append(f"### 2.{ch_i} {zh_no_num}")
|
||||
L.append("")
|
||||
L.append(f"> {meta['intro']}")
|
||||
L.append("")
|
||||
L.append(f"**本节入选**：{len(plist)} 篇")
|
||||
L.append("")
|
||||
if not plist:
|
||||
L.append("_本次抓取/缓存中未找到达到入选阈值的论文。建议参考「未来跟进方向」一节用更具体的查询关键词补抓。_")
|
||||
L.append("")
|
||||
continue
|
||||
for i, (p, sc) in enumerate(plist, 1):
|
||||
L.extend(render_paper_entry(p, sc, i))
|
||||
L.append("")
|
||||
|
||||
# ──── 3. Top 10 必读 ────
|
||||
L.append("---")
|
||||
L.append("")
|
||||
L.append("## 3. 十大必读（Top 10，跨主题）")
|
||||
L.append("")
|
||||
L.append("从所有入选论文中按 total 评分挑出 10 篇，给出阅读优先级。")
|
||||
L.append("")
|
||||
L.append("| 优先级 | 标题 | 主题 | 评分 | 一句话价值 |")
|
||||
L.append("|---|---|---|---|---|")
|
||||
for i, (p, sc) in enumerate(top10, 1):
|
||||
title = (p.get("title") or "").strip().rstrip(".")[:70]
|
||||
topic_zh = TOPIC_META.get(p.get("_crowdroom_topic", ""), {}).get("zh", "-")
|
||||
url = p.get("url") or f"https://arxiv.org/abs/{p.get('arxiv_id','')}"
|
||||
# 一句话价值：curated 用 note；其他用命中关键词
|
||||
if p.get("_curated_note"):
|
||||
value = p["_curated_note"]
|
||||
elif sc["kw_hits"]:
|
||||
value = "命中 " + ", ".join(kw for kw, _ in sc["kw_hits"][:3]) + "；与 CrowdRoom 链路高度对齐"
|
||||
else:
|
||||
value = "与 CrowdRoom 链路高度对齐"
|
||||
# 避免管道符破坏表格
|
||||
value = value.replace("|", "\\|").replace("\n", " ")[:120]
|
||||
L.append(f"| **#{i}** | [{title}]({url}) | {topic_zh} | ⭐ {sc['total']} | {value} |")
|
||||
L.append("")
|
||||
|
||||
# ──── 4. 对 CrowdRoom 的启发 ────
|
||||
L.append("---")
|
||||
L.append("")
|
||||
L.append("## 4. 对 CrowdRoom 的具体启发")
|
||||
L.append("")
|
||||
for sec in APP_GUIDANCE:
|
||||
L.append(f"### 4.{APP_GUIDANCE.index(sec)+1} {sec['title']}")
|
||||
L.append("")
|
||||
L.append(f"**应用建议**：{sec['guidance']}")
|
||||
L.append("")
|
||||
# 从相关主题里挑 3-5 篇
|
||||
picks = []
|
||||
for tp in sec["topics"]:
|
||||
picks.extend(by_topic.get(tp, []))
|
||||
picks = sorted(picks, key=lambda x: x[1]["total"], reverse=True)[:5]
|
||||
if picks:
|
||||
L.append("**相关入选论文（按评分降序）**：")
|
||||
for p, sc in picks:
|
||||
title = (p.get("title") or "").strip().rstrip(".")[:80]
|
||||
url = p.get("url") or f"https://arxiv.org/abs/{p.get('arxiv_id','')}"
|
||||
L.append(f"- ⭐ {sc['total']} · [{title}]({url}) — `{p.get('arxiv_id','')}`")
|
||||
else:
|
||||
L.append("_本次抓取中相关论文不足，等待二次抓取补充。_")
|
||||
L.append("")
|
||||
|
||||
# ──── 5. 边缘候选 ────
|
||||
L.append("---")
|
||||
L.append("")
|
||||
L.append("## 5. 未入选但值得关注（边缘候选）")
|
||||
L.append("")
|
||||
L.append(f"评分位于 `[{args.threshold - 1.5}, {args.threshold})` 区间的论文，与 CrowdRoom 弱相关但可作背景知识。")
|
||||
L.append("")
|
||||
borderline_sorted = sorted(borderline, key=lambda x: x[1]["total"], reverse=True)[:25]
|
||||
if borderline_sorted:
|
||||
for p, sc in borderline_sorted:
|
||||
title = (p.get("title") or "").strip().rstrip(".")[:90]
|
||||
url = p.get("url") or f"https://arxiv.org/abs/{p.get('arxiv_id','')}"
|
||||
pub = p.get("published", "")[:10]
|
||||
topic_zh = TOPIC_META.get(p.get("_crowdroom_topic", ""), {}).get("zh", "-")
|
||||
L.append(f"- ⭐ {sc['total']} · [{title}]({url}) ({pub}) — {topic_zh}")
|
||||
else:
|
||||
L.append("_无_")
|
||||
L.append("")
|
||||
|
||||
# ──── 6. 未来跟进方向 ────
|
||||
L.append("---")
|
||||
L.append("")
|
||||
L.append("## 6. 未来跟进方向（下一次抓取应新增的查询）")
|
||||
L.append("")
|
||||
suggestions = [
|
||||
("**RoomPlan 强化检索**：当前 `all:RoomPlan` 召回不足，建议增加 `all:\"Apple RoomPlan\" OR all:\"USDZ room\" OR all:\"iOS LiDAR scan\"`，"
|
||||
"并对接 [Apple Developer 文档](https://developer.apple.com/documentation/roomplan/) 引用追踪。"),
|
||||
("**3D Gen 前沿模型专追踪**：TRELLIS、Hunyuan3D-2、SF3D、CLAY、Direct3D、3DTopia 等 2024-2025 爆款均需点名追踪，"
|
||||
"推荐查询：`all:(TRELLIS OR Hunyuan3D OR SF3D OR CLAY OR Direct3D) AND all:(furniture OR \"3D asset\")`。"),
|
||||
("**Diffusion 室内布局**：DiffuScene、LayoutDiffusion、Pose2Room、RoomDesigner 系列，"
|
||||
"查询：`all:(DiffuScene OR LayoutDiffusion OR Pose2Room OR RoomDesigner OR InstructScene)`。"),
|
||||
("**Apple Vision Pro / spatial computing**：visionOS、Spatial Persona、Object Capture 在 2025 后产出加速，"
|
||||
"查询：`all:(\"Vision Pro\" OR \"visionOS\" OR \"Object Capture\") AND all:(spatial OR scan)`。"),
|
||||
("**3DGS in browser**：gsplat.js、SuperSplat、Brush 等 web 推理工程化论文/技报，"
|
||||
"查询：`all:(\"web 3DGS\" OR \"browser gaussian splatting\" OR gsplat OR SuperSplat)`。"),
|
||||
("**OBB 9DoF 朝向估计专攻**：当前关键词命中少，可补：`all:(\"category-level 6D pose\" OR \"9DoF object pose\" OR \"front-facing direction\")`。"),
|
||||
("**Crowdsourced 3D / Citizen Science**：补充 `all:(OpenStreetMap 3D OR Mapillary OR CitySim OR \"user-contributed 3D mesh\")`。"),
|
||||
]
|
||||
for i, s in enumerate(suggestions, 1):
|
||||
L.append(f"{i}. {s}")
|
||||
L.append("")
|
||||
|
||||
L.append("---")
|
||||
L.append("")
|
||||
L.append("## 附录 A · 复现脚本")
|
||||
L.append("")
|
||||
L.append("```bash")
|
||||
L.append("# 1. 抓取（建议非高峰时段执行；如限速失败会自动指数退避）")
|
||||
L.append("HTTPS_PROXY= HTTP_PROXY= ALL_PROXY= \\")
|
||||
L.append(" python3 research/fetch_crowdroom_papers.py \\")
|
||||
L.append(" --max-results 20 --delay 8 --max-days 365 \\")
|
||||
L.append(" --out research/crowdroom_papers_raw.json")
|
||||
L.append("")
|
||||
L.append("# 2. 生成本综述")
|
||||
L.append("python3 research/_build_crowdroom_review.py \\")
|
||||
L.append(" --fresh research/crowdroom_papers_raw.json \\")
|
||||
L.append(" --legacy research/search_results.json \\")
|
||||
L.append(" --out research/crowdroom_related_papers_2026.md")
|
||||
L.append("```")
|
||||
L.append("")
|
||||
L.append("**注**：本文档由 [`_build_crowdroom_review.py`](_build_crowdroom_review.py:1) 自动生成。")
|
||||
L.append("如需更新，重新抓取 + 重新跑该脚本即可（脚本会覆盖 md 但不会触碰其他文件）。")
|
||||
L.append("")
|
||||
L.append("## 附录 B · arXiv 429 限速故障排查")
|
||||
L.append("")
|
||||
L.append("本次执行遇到的问题与建议解决方案：")
|
||||
L.append("")
|
||||
L.append("| 现象 | 诊断 | 推荐方案 |")
|
||||
L.append("|---|---|---|")
|
||||
L.append("| `HTTP 429 Rate exceeded` 立即返回 | 本机 IP 在 arXiv API 限速窗口（约 1-2 小时） | 等待 ≥ 2 小时再重试 |")
|
||||
L.append("| 走 `127.0.0.1:6984` 代理仍 429 | 代理出口 IP 属于机房 IP 段，与本机共享限速桶 | 切换住宅 IP 代理或换运营商 |")
|
||||
L.append("| 退避到 180s 仍 429 | 退避不够；arXiv 限速窗口很长 | 单进程 / 单线程 / 主题间 sleep ≥ 30s |")
|
||||
L.append("| 偶尔 `SSL: UNEXPECTED_EOF` | 代理 TLS 不稳定 | 直连 + 增大 timeout（45s+）|")
|
||||
L.append("")
|
||||
L.append("**推荐重抓流程（下次执行）**：")
|
||||
L.append("")
|
||||
L.append("```bash")
|
||||
L.append("# 1. 在北京时间 04:00-09:00（arXiv 北美夜间）执行")
|
||||
L.append("# 2. 用住宅代理或切移动热点")
|
||||
L.append("# 3. 单次抓取，慢速 + 大退避")
|
||||
L.append("HTTPS_PROXY=http://住宅代理:端口 \\")
|
||||
L.append(" python3 research/fetch_crowdroom_papers.py \\")
|
||||
L.append(" --proxy http://住宅代理:端口 \\")
|
||||
L.append(" --max-results 25 --delay 15 --max-days 365 \\")
|
||||
L.append(" --out research/crowdroom_papers_raw.json")
|
||||
L.append("")
|
||||
L.append("# 4. 重新生成 md")
|
||||
L.append("python3 research/_build_crowdroom_review.py")
|
||||
L.append("```")
|
||||
L.append("")
|
||||
L.append("**降级路径**：如多次失败可改用 [Semantic Scholar API](https://api.semanticscholar.org/) "
|
||||
"或 [OpenAlex](https://api.openalex.org/) 作 arXiv 替代，二者无严格限速；要求脚本支持 schema 适配。")
|
||||
L.append("")
|
||||
|
||||
Path(args.out).write_text("\n".join(L), encoding="utf-8")
|
||||
print(f"✅ 生成: {args.out}")
|
||||
print(f" 候选: {total_candidates} (fresh={n_fresh}, cache={n_legacy})")
|
||||
print(f" 入选: {n_accepted}")
|
||||
print(f" 边缘候选: {len(borderline)}")
|
||||
print(f" Top10: {len(top10)}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"meta": {
|
||||
"generated_at": "2026-05-20T16:30:00+0800",
|
||||
"proxy": "direct + http://127.0.0.1:6984 fallback",
|
||||
"max_results_per_topic": 20,
|
||||
"max_days": 365,
|
||||
"sort": "submittedDate",
|
||||
"topic_count": 10,
|
||||
"total_raw": 0,
|
||||
"total_after_date_filter": 0,
|
||||
"total_unique": 0,
|
||||
"fetch_status": "FAILED_RATE_LIMIT",
|
||||
"fetch_error": "All requests blocked by arXiv API with HTTP 429 'Rate exceeded'. Host IP and proxy IP both flagged by arXiv's rate limiter (window > 2 hours). Tried direct connection, http://127.0.0.1:6984 proxy, IPv4, IPv6 — all returned 429 immediately or after exponential backoff (30/60/90/120/150/180s). Two topics ran 6 retries each (~15 min) and still failed.",
|
||||
"fetch_attempts": [
|
||||
{"time": "2026-05-20T13:18 CST", "topic": "roomplan_arkit", "result": "0 papers; 4x 429 after 30/60/90/120s backoff"},
|
||||
{"time": "2026-05-20T16:32 CST (background)", "topic": "roomplan_arkit", "result": "0 papers; 4x 429 + 2x SSL timeout"},
|
||||
{"time": "2026-05-20T16:32 CST (background)", "topic": "gaussian_splatting_indoor", "result": "0 papers; 6x 429 (30/60/90/120/150/180s)"},
|
||||
{"time": "2026-05-20T16:50 CST (probe)", "topic": "all:RoomPlan", "result": "HTTP 429 from both direct & proxy"}
|
||||
],
|
||||
"recommended_retry": "Wait >= 2 hours; ideally execute between 04:00-09:00 Beijing Time (arXiv off-peak); or switch to residential proxy. See research/crowdroom_related_papers_2026.md Appendix B."
|
||||
},
|
||||
"topics_meta": {
|
||||
"roomplan_arkit": {"zh": "1. iPhone RoomPlan / ARKit 室内扫描", "query": "all:(\"RoomPlan\" OR \"iPhone LiDAR\" OR \"ARKit\") AND all:(\"indoor\" OR \"reconstruction\" OR \"scan\")"},
|
||||
"gaussian_splatting_indoor": {"zh": "2. 3D Gaussian Splatting 室内重建", "query": "all:(\"3D Gaussian Splatting\" OR \"3DGS\" OR \"Gaussian Splatting\") AND all:(\"indoor\" OR \"room\" OR \"scene\")"},
|
||||
"nerf_indoor": {"zh": "3. NeRF 室内场景重建", "query": "all:(\"neural radiance field\" OR \"NeRF\") AND all:(\"indoor\" OR \"room scale\" OR \"scene reconstruction\")"},
|
||||
"text_to_3d_furniture": {"zh": "4. text-to-3D 家具 / 资产生成", "query": "all:(\"text-to-3D\" OR \"text to 3D\") AND (all:\"furniture\" OR all:\"asset\" OR all:\"object\")"},
|
||||
"indoor_layout_generation": {"zh": "5. 室内布局生成 / 房间布置合成", "query": "all:(\"indoor layout\" OR \"room layout\" OR \"scene layout\") AND (all:\"generation\" OR all:\"synthesis\" OR all:\"diffusion\")"},
|
||||
"obb_pose_estimation": {"zh": "6. 3D 物体姿态估计 / OBB 朝向", "query": "all:(\"oriented bounding box\" OR \"9DoF pose\" OR \"object pose estimation\") AND (all:\"indoor\" OR all:\"furniture\" OR all:\"scene\")"},
|
||||
"crowdsourced_3d": {"zh": "7. 众包 3D 数据采集 / 数据集", "query": "all:(\"crowdsourced\" OR \"crowdsourcing\" OR \"user-contributed\") AND (all:\"3D\" OR all:\"scanning\" OR all:\"reconstruction\")"},
|
||||
"digital_twin_indoor": {"zh": "8. 数字孪生 / 室内 GIS / Scan-to-BIM", "query": "all:(\"digital twin\" OR \"scan-to-BIM\" OR \"Scan2BIM\" OR \"indoor BIM\") AND (all:\"building\" OR all:\"indoor\" OR all:\"reconstruction\")"},
|
||||
"image_to_3d": {"zh": "9. image-to-3D / 单图重建", "query": "all:(\"image-to-3D\" OR \"single image 3D\" OR \"single-view reconstruction\") AND (all:\"object\" OR all:\"mesh\" OR all:\"furniture\")"},
|
||||
"usd_gltf_assets": {"zh": "10. USD / glTF / 3D 资产标准化", "query": "all:(\"glTF\" OR \"USD\" OR \"Universal Scene Description\" OR \"OpenUSD\") AND (all:\"3D\" OR all:\"asset\" OR all:\"scene\")"}
|
||||
},
|
||||
"arxiv": {
|
||||
"roomplan_arkit": [],
|
||||
"gaussian_splatting_indoor": [],
|
||||
"nerf_indoor": [],
|
||||
"text_to_3d_furniture": [],
|
||||
"indoor_layout_generation": [],
|
||||
"obb_pose_estimation": [],
|
||||
"crowdsourced_3d": [],
|
||||
"digital_twin_indoor": [],
|
||||
"image_to_3d": [],
|
||||
"usd_gltf_assets": []
|
||||
},
|
||||
"stats": {
|
||||
"roomplan_arkit": {"raw": 0, "after_date_filter": 0, "status": "429 rate-limit"},
|
||||
"gaussian_splatting_indoor": {"raw": 0, "after_date_filter": 0, "status": "429 rate-limit"},
|
||||
"nerf_indoor": {"raw": 0, "after_date_filter": 0, "status": "not-attempted (killed at topic 3)"},
|
||||
"text_to_3d_furniture": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"indoor_layout_generation": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"obb_pose_estimation": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"crowdsourced_3d": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"digital_twin_indoor": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"image_to_3d": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"usd_gltf_assets": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"}
|
||||
},
|
||||
"unique_papers": []
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
# CrowdRoom 相关论文综述（2026 骨架版）
|
||||
|
||||
> **文档时间戳（开头）**
|
||||
> - ISO 8601 (UTC): `2026-05-20T09:06:58Z`
|
||||
> - Asia/Shanghai (UTC+8): `2026-05-20 17:06:58`
|
||||
> - 生成模式：**抓取失败状态报告 + 综述骨架（0 篇真实论文）**
|
||||
|
||||
---
|
||||
|
||||
## 摘要
|
||||
|
||||
本综述原计划基于 [`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) 中由 [`research/fetch_crowdroom_papers.py`](fetch_crowdroom_papers.py) 抓取的 arXiv 论文，围绕 CrowdRoom（"RoomPlan 版 Sketchfab + Pinterest"，详见 [`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md)）所关切的 10 个研究方向，给出按主题组织的中文文献综述。然而，**本次数据采集因 arXiv API 全面速率限制（HTTP 429）而完全失败**：JSON 中 `unique_papers` 为空数组，10 个主题各自返回 0 篇论文。为遵守"不要编造未在 JSON 中出现的论文"的硬性约束，本文不引用任何具体论文，而是**以骨架（skeleton）形式**给出：(i) 数据采集失败状态的如实复盘；(ii) 10 个预设主题各自的研究问题（Research Questions, RQ）；(iii) 每个主题应覆盖的典型方法学家族（taxonomy placeholder）；(iv) CrowdRoom 项目可能受益的研究空白与未来方向。本文档预期在后续抓取成功后，被自动化脚本填充为完整综述。
|
||||
|
||||
---
|
||||
|
||||
## 1. 引言与背景
|
||||
|
||||
### 1.1 研究动机：为什么 CrowdRoom 需要一份文献综述
|
||||
|
||||
CrowdRoom 的产品定位是「人人用 iPhone 扫一个房间，传到云端就有可在浏览器里 360° 把玩、分层切换、换家具换材质、Remix 再创作的 3D 房间社区」。其技术链路（参见 [`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §4 架构图）横跨多个活跃研究领域：
|
||||
|
||||
- **采集端**：iPhone RoomPlan / ARKit / LiDAR 的几何与语义精度边界；
|
||||
- **重建端**：以 NeRF、3D Gaussian Splatting（3DGS）为代表的神经场重建；
|
||||
- **生成端**：text-to-3D 家具、image-to-3D 单图重建、室内布局生成；
|
||||
- **理解端**：3D 物体姿态估计、OBB（Oriented Bounding Box）朝向估计；
|
||||
- **数据与社区**：众包 3D 数据采集、隐私脱敏、UGC 治理；
|
||||
- **应用与互操作**：数字孪生 / Scan-to-BIM、USD / glTF / OpenUSD 资产标准化。
|
||||
|
||||
每一条链路都已有成熟的学术社群与代表性方法；CrowdRoom 要在 8 周内交付 MVP（参见 [`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §7.1），需要快速吸收"哪些方法已成熟、哪些仍是开放问题、哪些方法学路线与消费级路线相容"等综述性判断。本文档即为该综述的**预定位**版本。
|
||||
|
||||
### 1.2 综述的范围与边界
|
||||
|
||||
本综述聚焦 10 个由 [`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) 的 `topics_meta` 字段预先定义的主题（详见 §3）。**范围内**：与上述链路直接相关的几何重建、生成式 3D、众包数据、隐私、资产标准化文献。**范围外**：通用机器人 SLAM 综述、与消费级路线无关的工业 BIM/GIS、纯图像生成（2D 扩散模型）综述。
|
||||
|
||||
### 1.3 与 CrowdRoom 项目文档的关系
|
||||
|
||||
本综述与 [`plans/CrowdRoom/`](../plans/CrowdRoom/) 目录下的产品/技术文档形成"文献证据 ↔ 工程决策"的双向引用关系：
|
||||
|
||||
- 工程文档中"选 3DGS 还是 NeRF？"等技术选型问题，应由本综述的 §3.2 / §3.3 给出文献支撑；
|
||||
- 本综述识别出的研究空白，应反馈到 [`plans/CrowdRoom/ROADMAP.md`](../plans/CrowdRoom/ROADMAP.md)。
|
||||
|
||||
---
|
||||
|
||||
## 2. 数据采集状态（如实复盘）
|
||||
|
||||
### 2.1 抓取结果总览
|
||||
|
||||
| 字段 | 值 |
|
||||
|------|-----|
|
||||
| 数据源文件 | [`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) |
|
||||
| 生成时间 | `2026-05-20T16:30:00+0800` |
|
||||
| 抓取脚本 | [`research/fetch_crowdroom_papers.py`](fetch_crowdroom_papers.py) |
|
||||
| 抓取状态 | **`FAILED_RATE_LIMIT`** |
|
||||
| 涵盖论文总数 | **0 篇** |
|
||||
| 唯一论文数 `unique_papers` | **0** |
|
||||
| 涉及主题数 | 10（全部预设主题） |
|
||||
| 失败模式 | arXiv API 全面返回 HTTP 429「Rate exceeded」 |
|
||||
|
||||
### 2.2 已尝试的抓取过程
|
||||
|
||||
JSON `meta.fetch_attempts` 字段记录了 4 次明确的尝试，均失败：
|
||||
|
||||
| # | 时间（CST） | 主题 | 结果 |
|
||||
|---|---|---|---|
|
||||
| 1 | 2026-05-20T13:18 | `roomplan_arkit` | 0 篇；4× 429（30/60/90/120 s 退避） |
|
||||
| 2 | 2026-05-20T16:32（后台） | `roomplan_arkit` | 0 篇；4× 429 + 2× SSL timeout |
|
||||
| 3 | 2026-05-20T16:32（后台） | `gaussian_splatting_indoor` | 0 篇；6× 429（30/60/90/120/150/180 s 退避） |
|
||||
| 4 | 2026-05-20T16:50（探针） | `all:RoomPlan` | 直连与代理双双返回 HTTP 429 |
|
||||
|
||||
其余 8 个主题状态为 `not-attempted (killed at topic 3)`：因脚本在第 3 个主题处即被限流终止，未进入。
|
||||
|
||||
### 2.3 失败原因诊断
|
||||
|
||||
- **触发条件**：宿主机 IP 与 HTTP 代理（`http://127.0.0.1:6984`）IP 均已被 arXiv 速率限制器标记，速率窗口长度 > 2 小时；
|
||||
- **重试无效**：30/60/90/120/150/180 秒的指数退避未能恢复；
|
||||
- **协议无效**：IPv4 与 IPv6 双栈同样被拒；
|
||||
- **代理无效**：直连与回环代理表现一致。
|
||||
|
||||
### 2.4 建议的恢复策略
|
||||
|
||||
根据 JSON `meta.recommended_retry`：
|
||||
|
||||
1. **等待 ≥ 2 小时**后重试；
|
||||
2. **执行时段**：建议在北京时间 04:00–09:00 的 arXiv 低峰窗口；
|
||||
3. **网络方案**：必要时切换到住宅代理（residential proxy）；
|
||||
4. **降级方案**：使用 Semantic Scholar API 或 OpenAlex 作为替代检索源（不在当前脚本范围内）。
|
||||
|
||||
### 2.5 对本综述的直接影响
|
||||
|
||||
由于 0 篇真实论文可引用，本综述：
|
||||
- **不**给出任何 `[编号]` 形式的引用（参考文献章节为空，明确标注 0 篇）；
|
||||
- **不**包含任何具体方法（如"3DGS 方法 X 在 Replica 数据集上取得 PSNR=Y"）的数值或作者归属；
|
||||
- **仅**给出每个主题的研究问题、应覆盖的方法学家族（taxonomy placeholder）、CrowdRoom 应当从中提取的工程结论模板。
|
||||
|
||||
---
|
||||
|
||||
## 3. 主题骨架（10 个预设方向）
|
||||
|
||||
以下 10 个一级主题严格对应 JSON `topics_meta` 字段的 10 个键，顺序保留原顺序。每个主题给出：**(a) 主题定义与 CrowdRoom 关联**、**(b) 待回答的研究问题 RQ**、**(c) 应覆盖的方法学家族**、**(d) 论文填充占位**。
|
||||
|
||||
### 3.1 主题一：iPhone RoomPlan / ARKit 室内扫描
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
该主题关注 Apple RoomPlan API、ARKit、iPhone LiDAR 在室内几何采集中的精度边界、语义输出格式与最佳实践。CrowdRoom iOS App 的唯一采集入口即 RoomPlan，因此该主题是综述的**主干**，对应 [`plans/CrowdRoom/03_ios_app_plan.md`](../plans/CrowdRoom/03_ios_app_plan.md) 与 [`plans/iphone/roomplan_accuracy_and_cad_export.md`](../plans/iphone/roomplan_accuracy_and_cad_export.md) 的工程文档。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ1.1：RoomPlan 在不同光照 / 户型 / 家具密度下的几何精度量化结论是什么？
|
||||
- RQ1.2：RoomPlan 输出的 USDZ + JSON 是否足以支撑下游"分层切换 / 家具替换"的语义需求？
|
||||
- RQ1.3：ARKit / iPhone LiDAR 与中高端激光扫描仪（如 Matterport / Leica BLK）的差距在哪些指标上显著？
|
||||
- RQ1.4：扫描路径与移动速度对 RoomPlan 重建质量的影响曲线如何？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| LiDAR + VIO 几何采集 | 待填充 | 扫描引导提示、精度免责声明 |
|
||||
| 平面 / 房间几何拟合 | 待填充 | 墙地分割稳定性、出图精度 |
|
||||
| 室内语义分割（家具类别） | 待填充 | 与 RoomPlan 内置类别的对齐 |
|
||||
| 扫描质量评估与反馈 | 待填充 | 用户扫描中实时提示 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.2 主题二：3D Gaussian Splatting 室内重建
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
3DGS 与 NeRF（§3.3）并列为近年主流的"可微神经场"方案，其在室内场景的实时渲染速度与显存占用对 Web 端可视化具有直接吸引力。CrowdRoom MVP 中的 Web 端实际渲染由 Three.js / R3F 承担，仍是网格 + 贴图路线；3DGS 是**未来可选升级路径**（参见 [`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §5 备选项）。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ2.1：3DGS 在室内复杂遮挡 / 反射 / 透明物体上的失败模式是什么？
|
||||
- RQ2.2：3DGS 与传统网格 + PBR 贴图路线相比，在带宽 / 显存 / 浏览器兼容性上的真实差距？
|
||||
- RQ2.3：是否存在可编辑（语义可分层、家具可替换）的 3DGS 表达？
|
||||
- RQ2.4：3DGS 与 RoomPlan 输出（USDZ + 平面几何）如何融合？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 原始 3DGS 渲染 | 待填充 | 浏览器端渲染可行性 |
|
||||
| 大场景 / 房间级 3DGS | 待填充 | 单房间显存占用 |
|
||||
| 可编辑 / 可分层 3DGS | 待填充 | 与"分层 = L1–L4"的对齐 |
|
||||
| 3DGS 压缩与流式传输 | 待填充 | CDN 分发可行性 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.3 主题三：NeRF 室内场景重建
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
NeRF 作为 3DGS 之前的主流神经场方法，仍在"高保真离线重建 + 服务端渲染"路线下保有价值。CrowdRoom MVP 明确不做服务端实时渲染（[`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §4），因此 NeRF 主要作为**对比参考与离线烘焙工具**进入综述。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ3.1：NeRF 在房间尺度（room-scale）下的训练耗时与显存占用边界？
|
||||
- RQ3.2：NeRF → 网格（mesh extraction）的质量损失曲线如何？
|
||||
- RQ3.3：是否存在"少量 iPhone 帧 → 可用 NeRF"的实用化方案？
|
||||
- RQ3.4：NeRF 在低纹理墙面 / 大面积玻璃上的退化机制？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 原始体素 NeRF | 待填充 | 训练成本基线 |
|
||||
| 加速 NeRF（哈希编码等） | 待填充 | 离线烘焙可行性 |
|
||||
| 室内大场景 NeRF | 待填充 | 房间尺度收敛性 |
|
||||
| NeRF → mesh 提取 | 待填充 | 与 glTF 管线衔接 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.4 主题四：text-to-3D 家具 / 资产生成
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
text-to-3D 让 Remixer 用户通过自然语言生成新家具，对应 CrowdRoom user story US-5（家具替换，见 [`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §3）。MVP 阶段公共资产库走 CC0 素材路线，text-to-3D 是**P2 阶段**的潜在能力扩展。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ4.1：当前 text-to-3D 在家具类目（沙发 / 椅子 / 灯具）的生成质量是否足够直接进入 glTF 管线？
|
||||
- RQ4.2：生成结果的拓扑、UV 与 PBR 材质是否可被 Three.js 直接消费？
|
||||
- RQ4.3：生成式资产的版权与可商用边界？
|
||||
- RQ4.4：与"扫描自真实家具"的资产相比，生成式资产在物理尺度上的一致性如何？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| Score Distillation 路线 | 待填充 | 生成质量与时间成本 |
|
||||
| 多视图扩散 → 3D 重建 | 待填充 | 与 image-to-3D 的边界 |
|
||||
| Native 3D 扩散（点云 / 三平面） | 待填充 | 输出格式与 glTF 兼容性 |
|
||||
| 家具专用大规模训练 | 待填充 | 类别覆盖度 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.5 主题五：室内布局生成 / 房间布置合成
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
"给定一个空房间，自动生成一种家具布局"，对 CrowdRoom 的 Remix 创作（"装修方案"）有显著加速价值。MVP 暂不引入自动布局，但综述应识别该领域的成熟度，以支撑 P2 路线图。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ5.1：当前室内布局生成是否能保证物理可行性（无穿插、可达性、人体工学）？
|
||||
- RQ5.2：以扩散模型为代表的布局生成方法与传统优化方法（如约束求解）的对比？
|
||||
- RQ5.3：是否存在可条件化于"扫描得到的真实房间几何"的布局生成？
|
||||
- RQ5.4：风格条件（如"日式 / 北欧 / 工业风"）是否可控？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 自回归布局生成 | 待填充 | 物理约束的硬约束注入 |
|
||||
| 扩散模型布局生成 | 待填充 | 风格可控性 |
|
||||
| 优化 / 规则驱动布局 | 待填充 | 与生成式方法的混合 |
|
||||
| 基于场景图的布局 | 待填充 | 与 PRISM 语义图的对齐 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.6 主题六：3D 物体姿态估计 / OBB 朝向
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
RoomPlan 输出的家具节点含位置与 OBB，但**朝向**（"沙发面朝哪里"）的稳定性是已知痛点。该主题决定 CrowdRoom 在家具替换时能否做到"新沙发面朝原沙发同一方向"的自动对齐。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ6.1：在 iPhone LiDAR 噪声水平下，9DoF 姿态估计的可达精度？
|
||||
- RQ6.2：RoomPlan 原生朝向输出的失败模式（如对称家具）如何缓解？
|
||||
- RQ6.3：是否存在仅靠扫描得到的稀疏点云即可恢复朝向的轻量方法？
|
||||
- RQ6.4：基于类别先验（"沙发的座面朝向房间内侧"）的规则修正是否实用？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 模板匹配 / ICP | 待填充 | 与公共资产库的对齐 |
|
||||
| 学习式 6D/9D 姿态 | 待填充 | 噪声鲁棒性 |
|
||||
| 类别级姿态（NOCS 等） | 待填充 | 跨实例泛化 |
|
||||
| 朝向投票 / 对称性消歧 | 待填充 | 对称家具消歧 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.7 主题七：众包 3D 数据采集 / 数据集
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
这是 CrowdRoom 的**身份主题**：是否存在已有的众包 3D 数据社区？其失败 / 成功要素是什么？这是综述中最需要识别"研究空白"的章节，对应 [`plans/CrowdRoom/10_governance.md`](../plans/CrowdRoom/10_governance.md) 与 [`plans/CrowdRoom/09_privacy.md`](../plans/CrowdRoom/09_privacy.md)。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ7.1：现有大规模室内 3D 数据集（如 ScanNet 系列、Matterport3D 等）的采集模式是众包还是专业团队？
|
||||
- RQ7.2：众包 3D 数据的质量控制机制（重叠采集、用户评分、自动筛选）有哪些？
|
||||
- RQ7.3：UGC 3D 内容的版权与许可（CC0 / CC-BY / 商用）实践？
|
||||
- RQ7.4：是否有研究专门量化"业余用户扫描"vs"专业团队扫描"的质量差距？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 专业团队大规模室内数据集 | 待填充 | 数据规模基线 |
|
||||
| 众包 2D / 3D 标注 | 待填充 | 质量控制流程 |
|
||||
| 用户贡献 3D 内容平台 | 待填充 | 社区机制 |
|
||||
| 联邦 / 隐私保护的数据共享 | 待填充 | 与 §3.9 的衔接 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.8 主题八：数字孪生 / 室内 GIS / Scan-to-BIM
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
[`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §1 明确划清："CrowdRoom **不做**专业 GIS 查询，只做消费级图层操作"。本主题在综述中作为**对照组**：说明"我们不走 Scan-to-BIM 路线"的判断依据。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ8.1：Scan-to-BIM 当前的自动化程度是什么？是否仍依赖大量人工修正？
|
||||
- RQ8.2：消费级 RoomPlan 输出与 BIM 所需精度（IFC LOD 等级）之间的差距？
|
||||
- RQ8.3：是否有轻量级"消费级数字孪生"的中间路线？
|
||||
- RQ8.4：室内 GIS 数据模型（如 IndoorGML）与 Three.js 渲染管线的兼容性？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 点云 / 网格 → IFC 自动化 | 待填充 | 自动化上限 |
|
||||
| 室内 GIS 查询 | 待填充 | 暂不引入的理由 |
|
||||
| 数字孪生平台 | 待填充 | 消费级 vs 工业级边界 |
|
||||
| 轻量化 BIM / 简化 IFC | 待填充 | 中间路线可行性 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.9 主题九：image-to-3D / 单图重建
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
单图 → 3D 重建可用于"用户上传一张家具照片，自动生成可替换的 3D 资产"，是 Remix 创作的潜在低门槛入口。它与 §3.4（text-to-3D）共同构成"非扫描类资产入库"路径。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ9.1：当前 image-to-3D 在家具类目下的几何完整性（背面 / 底面）如何？
|
||||
- RQ9.2：单图 vs 少视图（few-view）的质量拐点在哪里？
|
||||
- RQ9.3：输出网格的拓扑质量是否能被 PBR 管线直接消费？
|
||||
- RQ9.4：与 text-to-3D 相比，image-to-3D 在"形似但不同物"的语义偏移上表现如何？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| 回归式单视图重建 | 待填充 | 几何完整性 |
|
||||
| 扩散先验式重建 | 待填充 | 背面合理性 |
|
||||
| 多视图扩散 + 重建 | 待填充 | 与扫描互补 |
|
||||
| 类别先验式重建 | 待填充 | 家具类目泛化 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
### 3.10 主题十：USD / glTF / 3D 资产标准化
|
||||
|
||||
**(a) 主题定义与 CrowdRoom 关联**
|
||||
CrowdRoom 数据链路的核心格式约定为「USDZ（来自 RoomPlan）→ glTF/.glb（Web 端消费）」。该主题为综述提供"为什么是这两种格式""未来 OpenUSD 的影响"等格式层的文献支撑，对应 [`plans/CrowdRoom/01_data_schema.md`](../plans/CrowdRoom/01_data_schema.md) 与 [`plans/iphone/roomplan_accuracy_and_cad_export.md`](../plans/iphone/roomplan_accuracy_and_cad_export.md)。
|
||||
|
||||
**(b) RQ**
|
||||
|
||||
- RQ10.1：USDZ 与 glTF 在浏览器端的兼容性现状？
|
||||
- RQ10.2：OpenUSD 的开放化进程对消费级 3D 社区的影响？
|
||||
- RQ10.3：Draco / Meshopt 压缩对 glTF 在 Web 端的实际收益曲线？
|
||||
- RQ10.4：3D 资产的版权水印 / 来源认证是否已有标准化方案？
|
||||
|
||||
**(c) 应覆盖的方法学家族**
|
||||
|
||||
| 家族 | 代表性方法（占位） | CrowdRoom 关注角度 |
|
||||
|------|------------------|---------------------|
|
||||
| glTF 核心规范与扩展 | 待填充 | Three.js 兼容性 |
|
||||
| USD / OpenUSD 生态 | 待填充 | 与 Apple 工具链衔接 |
|
||||
| 网格 / 纹理压缩 | 待填充 | CDN 带宽优化 |
|
||||
| 资产来源 / 水印 | 待填充 | UGC 治理 |
|
||||
|
||||
**(d) 论文填充占位**：当前 0 篇。
|
||||
|
||||
---
|
||||
|
||||
## 4. 跨主题趋势观察（基于主题定义层面）
|
||||
|
||||
由于本次未抓到任何论文，本节**不**给出基于真实文献的量化趋势，仅基于上述主题定义层面的合理推断（在抓取成功后应被实证数据替换）：
|
||||
|
||||
| # | 观察 | 与 CrowdRoom 的关联 |
|
||||
|---|------|---------------------|
|
||||
| T1 | 神经场（3DGS / NeRF）路线正逐步从"研究 demo"走向"产品可用"，但**可编辑性**仍是显著缺口 | 决定 Web 端是否升级到 3DGS 渲染 |
|
||||
| T2 | 生成式 3D（text-to-3D / image-to-3D）在家具类目上的"可用门槛"正快速降低，但**与 PBR / 物理尺度的对齐**仍未完全解决 | 决定 Remix 资产入库是否引入生成式入口 |
|
||||
| T3 | 室内扫描的"消费级"端（RoomPlan / 手机 LiDAR）与"专业级"端（Matterport / 激光扫描）之间，存在**精度差距 vs 易用性**的稳定权衡 | 决定 CrowdRoom 在"易用性优先"路线上的定位 |
|
||||
| T4 | 众包 3D 数据集的**质量控制与隐私治理**仍缺乏成熟方法学，研究空白显著 | 直接关系 [`plans/CrowdRoom/09_privacy.md`](../plans/CrowdRoom/09_privacy.md) 与 [`plans/CrowdRoom/10_governance.md`](../plans/CrowdRoom/10_governance.md) |
|
||||
| T5 | USD/OpenUSD 的开放化让"Apple 生态 → 通用 Web 3D"的链路在标准层趋于稳定 | 支撑 USDZ → glTF 转码路线的长期可持续性 |
|
||||
|
||||
> 以上 5 条观察仅为**待证伪假设**（hypothesis），需在抓取成功后用真实文献交叉验证。
|
||||
|
||||
---
|
||||
|
||||
## 5. 研究空白与 CrowdRoom 的未来方向
|
||||
|
||||
基于 §3 的 RQ 与 §4 的趋势观察，结合 [`plans/CrowdRoom/00_overview.md`](../plans/CrowdRoom/00_overview.md) §6 与 §7，本综述识别出以下**潜在研究空白**（同样为待证伪命题）：
|
||||
|
||||
1. **可编辑、可分层的神经场表达**：3DGS / NeRF 是否能原生支持"墙 / 地板 / 家具 / 材质"四层切换？这是 CrowdRoom 走 3DGS 路线的硬性前提。
|
||||
2. **手机端扫描的朝向稳定化**：仅靠 RoomPlan 输出（无重新训练）的朝向后处理方法，是 MVP 内最可能产生工程贡献的方向。
|
||||
3. **众包 3D 内容的轻量级隐私脱敏**：端侧人脸 / 身份证 / logo 模糊，与镜面区域 / 反射场景的边界处理，是研究与产品共同的真实空白。
|
||||
4. **生成式资产与扫描资产的尺度 / 材质一致性**：Remix 替换家具时的"无缝感"由此决定。
|
||||
5. **消费级数字孪生的中间表达**：介于"专业 BIM"与"消费级 glTF"之间，是否存在一种轻量结构化表达，恰好满足"分层 + Remix"？
|
||||
|
||||
这 5 条空白构成抓取成功后综述应**重点检索**的关键词组合（如 "editable 3DGS"、"object orientation refinement RoomPlan"、"privacy preserving mesh"、"text-to-3D physical scale"、"lightweight scene graph"）。
|
||||
|
||||
---
|
||||
|
||||
## 6. 局限与下一步
|
||||
|
||||
### 6.1 本综述的明确局限
|
||||
|
||||
- **0 篇真实论文**：本综述未引用任何具体论文，所有判断为基于领域常识与项目文档的占位结论；
|
||||
- **无量化对比**：所有比较表均为"待填充"，未给出 PSNR / 训练时间 / 模型体积等具体数值；
|
||||
- **无引用编号**：参考文献章节为空（§7），文中未使用 `[N]` 形式的编号引用。
|
||||
|
||||
### 6.2 下一步行动建议
|
||||
|
||||
1. 等待 ≥ 2 小时后，按 [`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) 的 `meta.recommended_retry` 在北京时间 04:00–09:00 重跑 [`research/fetch_crowdroom_papers.py`](fetch_crowdroom_papers.py)；
|
||||
2. 若 arXiv 持续受限，降级到 Semantic Scholar / OpenAlex 双源补全；
|
||||
3. 抓取成功后，按本骨架的 §3.1–§3.10 顺序，将每个主题下"应覆盖的方法学家族"表格中"待填充"替换为真实代表性论文（含 `[编号]`），并补全 §7 参考文献；
|
||||
4. 用真实数据替换 §4 的待证伪假设与 §5 的研究空白判断。
|
||||
|
||||
---
|
||||
|
||||
## 7. 参考文献
|
||||
|
||||
> **本次纳入论文总数：0 篇**
|
||||
>
|
||||
> 由于 arXiv API 在 2026-05-20 全面返回 HTTP 429 速率限制，[`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) 的 `unique_papers` 为空数组。按任务规则"不要在综述中加入未在 JSON 中出现的论文"，本章节当前不列出任何条目。
|
||||
>
|
||||
> **预留编号区间**：抓取成功后，预计 10 个主题各 5–15 篇，共约 60–120 篇，按主题分组并在文中以 `[编号]` 形式引用。条目格式预定为：
|
||||
>
|
||||
> ```
|
||||
> [N] 标题. 作者 1, 作者 2, ...（年份）. 期刊/会议. 链接.
|
||||
> ```
|
||||
|
||||
---
|
||||
|
||||
> **文档时间戳（结尾）**
|
||||
> - ISO 8601 (UTC): `2026-05-20T09:06:58Z`
|
||||
> - Asia/Shanghai (UTC+8): `2026-05-20 17:06:58`
|
||||
> - 实时时间获取建议：``date -u +"%Y-%m-%dT%H:%M:%SZ"``
|
||||
> - 生成模式：抓取失败状态报告 + 综述骨架（0 篇真实论文）
|
||||
> - 数据源快照：[`research/crowdroom_papers_raw.json`](crowdroom_papers_raw.json) `fetch_status = FAILED_RATE_LIMIT`
|
||||
@@ -0,0 +1,273 @@
|
||||
"""CrowdRoom 相关 arXiv 论文抓取脚本。
|
||||
|
||||
复用 research/search_info.py 中的 search_arxiv 与 build_opener，
|
||||
定义 CrowdRoom 项目专属的 10 组查询主题。
|
||||
|
||||
用法：
|
||||
HTTPS_PROXY=http://127.0.0.1:6984 HTTP_PROXY=http://127.0.0.1:6984 \
|
||||
python3 research/fetch_crowdroom_papers.py \
|
||||
--proxy http://127.0.0.1:6984 \
|
||||
--max-results 20 --delay 5.0 \
|
||||
--out research/crowdroom_papers_raw.json
|
||||
|
||||
参数：
|
||||
--proxy 代理（默认走环境变量 / 直连）
|
||||
--max-results 每主题最大论文数（默认 20）
|
||||
--delay 主题间延迟秒数（arXiv 政策 ≥3，建议 5）
|
||||
--topics 只跑指定主题
|
||||
--out 输出 JSON
|
||||
--max-days 仅保留 published 在 N 天内的论文（默认 365；0 = 不过滤）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
# 复用 search_info.py
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from search_info import build_opener # noqa: E402
|
||||
|
||||
# 自实现 search_arxiv，带更激进的退避（针对 arxiv 429 限速窗口）
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
|
||||
def search_arxiv(opener, query, max_results=20, sort_by="submittedDate", sort_order="descending",
|
||||
max_retries=6, base_backoff=30):
|
||||
base = "https://export.arxiv.org/api/query"
|
||||
params = urllib.parse.urlencode({
|
||||
"search_query": query, "start": 0, "max_results": max_results,
|
||||
"sortBy": sort_by, "sortOrder": sort_order,
|
||||
})
|
||||
url = f"{base}?{params}"
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "crowdroom-research-bot/0.1 (mailto:research@example.com)"
|
||||
})
|
||||
|
||||
data = ""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
with opener.open(req, timeout=45) as resp:
|
||||
data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = base_backoff * (attempt + 1)
|
||||
print(f"[arxiv][{query[:40]}...] 429 限速，等待 {wait}s 重试 ({attempt+1}/{max_retries})", file=sys.stderr)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f"[arxiv][{query[:40]}...] HTTPError {e.code}", file=sys.stderr)
|
||||
return []
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
wait = 15 * (attempt + 1)
|
||||
print(f"[arxiv][{query[:40]}...] timeout/url error: {e}; 等待 {wait}s 重试", file=sys.stderr)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
else:
|
||||
print(f"[arxiv][{query[:40]}...] 持续失败，放弃", file=sys.stderr)
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(data)
|
||||
except ET.ParseError as e:
|
||||
print(f"[arxiv][{query[:40]}...] XML parse error: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
|
||||
papers = []
|
||||
for entry in root.findall("atom:entry", ns):
|
||||
def _text(elem_path):
|
||||
elem = entry.find(elem_path, ns)
|
||||
return (elem.text or "").replace("\n", " ").strip() if elem is not None else ""
|
||||
|
||||
arxiv_id_url = _text("atom:id")
|
||||
arxiv_id = arxiv_id_url.rsplit("/", 1)[-1] if arxiv_id_url else ""
|
||||
arxiv_id_clean = arxiv_id.split("v")[0] if arxiv_id else ""
|
||||
|
||||
authors = []
|
||||
for a in entry.findall("atom:author", ns):
|
||||
name_elem = a.find("atom:name", ns)
|
||||
if name_elem is not None and name_elem.text:
|
||||
authors.append(name_elem.text.strip())
|
||||
|
||||
categories = [c.attrib.get("term", "") for c in entry.findall("atom:category", ns)]
|
||||
summary = _text("atom:summary")
|
||||
|
||||
# 提取 arxiv 专属字段
|
||||
primary_cat = ""
|
||||
pc = entry.find("arxiv:primary_category", ns)
|
||||
if pc is not None:
|
||||
primary_cat = pc.attrib.get("term", "")
|
||||
|
||||
# 提取 doi / journal_ref / comment 链接
|
||||
pdf_url = ""
|
||||
for link in entry.findall("atom:link", ns):
|
||||
if link.attrib.get("title") == "pdf":
|
||||
pdf_url = link.attrib.get("href", "")
|
||||
|
||||
papers.append({
|
||||
"arxiv_id": arxiv_id_clean,
|
||||
"url": f"https://arxiv.org/abs/{arxiv_id_clean}" if arxiv_id_clean else arxiv_id_url,
|
||||
"pdf_url": pdf_url,
|
||||
"title": _text("atom:title"),
|
||||
"published": _text("atom:published"),
|
||||
"updated": _text("atom:updated"),
|
||||
"authors": authors,
|
||||
"categories": categories,
|
||||
"primary_category": primary_cat,
|
||||
"summary": summary,
|
||||
})
|
||||
return papers
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# CrowdRoom 10 组检索主题
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
TOPICS: dict[str, dict] = {
|
||||
"roomplan_arkit": {
|
||||
"zh": "1. iPhone RoomPlan / ARKit 室内扫描",
|
||||
"query": 'all:("RoomPlan" OR "iPhone LiDAR" OR "ARKit") AND all:("indoor" OR "reconstruction" OR "scan")',
|
||||
},
|
||||
"gaussian_splatting_indoor": {
|
||||
"zh": "2. 3D Gaussian Splatting 室内重建",
|
||||
"query": 'all:("3D Gaussian Splatting" OR "3DGS" OR "Gaussian Splatting") AND all:("indoor" OR "room" OR "scene")',
|
||||
},
|
||||
"nerf_indoor": {
|
||||
"zh": "3. NeRF 室内场景重建",
|
||||
"query": 'all:("neural radiance field" OR "NeRF") AND all:("indoor" OR "room scale" OR "scene reconstruction")',
|
||||
},
|
||||
"text_to_3d_furniture": {
|
||||
"zh": "4. text-to-3D 家具 / 资产生成",
|
||||
"query": 'all:("text-to-3D" OR "text to 3D") AND (all:"furniture" OR all:"asset" OR all:"object")',
|
||||
},
|
||||
"indoor_layout_generation": {
|
||||
"zh": "5. 室内布局生成 / 房间布置合成",
|
||||
"query": 'all:("indoor layout" OR "room layout" OR "scene layout") AND (all:"generation" OR all:"synthesis" OR all:"diffusion")',
|
||||
},
|
||||
"obb_pose_estimation": {
|
||||
"zh": "6. 3D 物体姿态估计 / OBB 朝向",
|
||||
"query": 'all:("oriented bounding box" OR "9DoF pose" OR "object pose estimation") AND (all:"indoor" OR all:"furniture" OR all:"scene")',
|
||||
},
|
||||
"crowdsourced_3d": {
|
||||
"zh": "7. 众包 3D 数据采集 / 数据集",
|
||||
"query": 'all:("crowdsourced" OR "crowdsourcing" OR "user-contributed") AND (all:"3D" OR all:"scanning" OR all:"reconstruction")',
|
||||
},
|
||||
"digital_twin_indoor": {
|
||||
"zh": "8. 数字孪生 / 室内 GIS / Scan-to-BIM",
|
||||
"query": 'all:("digital twin" OR "scan-to-BIM" OR "Scan2BIM" OR "indoor BIM") AND (all:"building" OR all:"indoor" OR all:"reconstruction")',
|
||||
},
|
||||
"image_to_3d": {
|
||||
"zh": "9. image-to-3D / 单图重建",
|
||||
"query": 'all:("image-to-3D" OR "single image 3D" OR "single-view reconstruction") AND (all:"object" OR all:"mesh" OR all:"furniture")',
|
||||
},
|
||||
"usd_gltf_assets": {
|
||||
"zh": "10. USD / glTF / 3D 资产标准化",
|
||||
"query": 'all:("glTF" OR "USD" OR "Universal Scene Description" OR "OpenUSD") AND (all:"3D" OR all:"asset" OR all:"scene")',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
ap = argparse.ArgumentParser(description="CrowdRoom 相关 arXiv 抓取")
|
||||
ap.add_argument("--proxy", default=None)
|
||||
ap.add_argument("--max-results", type=int, default=20)
|
||||
ap.add_argument("--delay", type=float, default=5.0, help="主题间延迟秒数，arXiv ≥3s")
|
||||
ap.add_argument("--topics", nargs="+", default=None)
|
||||
ap.add_argument("--out", default="research/crowdroom_papers_raw.json")
|
||||
ap.add_argument("--max-days", type=int, default=365, help="仅保留近 N 天提交的论文；0 不过滤")
|
||||
ap.add_argument("--sort", choices=["submittedDate", "relevance"], default="submittedDate")
|
||||
return ap.parse_args()
|
||||
|
||||
|
||||
def filter_by_date(papers: list[dict], max_days: int) -> list[dict]:
|
||||
if max_days <= 0:
|
||||
return papers
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max_days)
|
||||
out = []
|
||||
for p in papers:
|
||||
pub = p.get("published", "")
|
||||
try:
|
||||
dt = datetime.fromisoformat(pub.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
out.append(p)
|
||||
continue
|
||||
if dt >= cutoff:
|
||||
out.append(p)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
opener = build_opener(args.proxy)
|
||||
selected = args.topics or list(TOPICS.keys())
|
||||
|
||||
results: dict = {
|
||||
"meta": {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"proxy": args.proxy or os.environ.get("HTTPS_PROXY") or "direct",
|
||||
"max_results_per_topic": args.max_results,
|
||||
"max_days": args.max_days,
|
||||
"sort": args.sort,
|
||||
"topic_count": len(selected),
|
||||
},
|
||||
"topics_meta": {k: TOPICS[k] for k in selected if k in TOPICS},
|
||||
"arxiv": {},
|
||||
"stats": {},
|
||||
}
|
||||
|
||||
total_raw = 0
|
||||
total_after_date = 0
|
||||
|
||||
for i, topic in enumerate(selected, 1):
|
||||
if topic not in TOPICS:
|
||||
print(f"[skip] 未知主题: {topic}", file=sys.stderr)
|
||||
continue
|
||||
query = TOPICS[topic]["query"]
|
||||
print(f"\n[{i}/{len(selected)}] arxiv: {topic}", file=sys.stderr)
|
||||
print(f" query: {query}", file=sys.stderr)
|
||||
papers = search_arxiv(opener, query, max_results=args.max_results, sort_by=args.sort)
|
||||
raw_n = len(papers)
|
||||
papers = filter_by_date(papers, args.max_days)
|
||||
total_raw += raw_n
|
||||
total_after_date += len(papers)
|
||||
results["arxiv"][topic] = papers
|
||||
results["stats"][topic] = {"raw": raw_n, "after_date_filter": len(papers)}
|
||||
print(f" → 抓到 {raw_n} 篇，时间窗内 {len(papers)} 篇", file=sys.stderr)
|
||||
if i < len(selected):
|
||||
time.sleep(args.delay)
|
||||
|
||||
# 去重统计（全局）
|
||||
seen_ids: set[str] = set()
|
||||
unique_papers: list[dict] = []
|
||||
for topic, plist in results["arxiv"].items():
|
||||
for p in plist:
|
||||
aid = p.get("arxiv_id", "")
|
||||
if not aid or aid in seen_ids:
|
||||
continue
|
||||
seen_ids.add(aid)
|
||||
unique_papers.append({**p, "_topic": topic})
|
||||
|
||||
results["meta"]["total_raw"] = total_raw
|
||||
results["meta"]["total_after_date_filter"] = total_after_date
|
||||
results["meta"]["total_unique"] = len(unique_papers)
|
||||
results["unique_papers"] = unique_papers
|
||||
|
||||
out_path = args.out
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n✅ 完成：raw={total_raw} 时间窗后={total_after_date} 去重={len(unique_papers)}", file=sys.stderr)
|
||||
print(f" → {out_path}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,303 @@
|
||||
"""把 search_results.json 渲染成结构化 markdown 综述。
|
||||
|
||||
用法：
|
||||
python3 research/gen_review_from_json.py \
|
||||
--in research/search_results.json \
|
||||
--out research/zed2i_arxiv_live_review.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
# 主题中英文标签 + 项目阶段映射
|
||||
TOPIC_META = {
|
||||
"stereo_matching": {
|
||||
"zh": "A. 双目立体匹配（被动深度）",
|
||||
"stage": "M2-3 / M3-4",
|
||||
"intro": "对应 ZED 双目深度算法的替换/超越路线。关注零样本泛化、Transformer 架构、神经几何编码。",
|
||||
},
|
||||
"visual_inertial_slam": {
|
||||
"zh": "B. 视觉惯性 SLAM / VIO",
|
||||
"stage": "M2-1 / M3-3",
|
||||
"intro": "对标 ZED 内建 VIO 的替代方案。关注 IMU 融合、长时鲁棒、动态环境。",
|
||||
},
|
||||
"gaussian_splatting_slam": {
|
||||
"zh": "C. 3D Gaussian Splatting SLAM（融合建图）",
|
||||
"stage": "M3-5 / M4",
|
||||
"intro": "把 3DGS 作为 SLAM 后端，实现实时定位+建图+渲染一体化。world model 训练的核心视觉表征。",
|
||||
},
|
||||
"monocular_depth_foundation": {
|
||||
"zh": "D. 单目深度基础模型",
|
||||
"stage": "M3-4",
|
||||
"intro": "Depth Anything / Marigold / Metric3D / UniDepth 等通用深度模型，作为双目深度失效的兜底。",
|
||||
},
|
||||
"indoor_rgbd_dataset": {
|
||||
"zh": "E. 室内 RGB-D 数据集与重建",
|
||||
"stage": "M4-1 / M4-2",
|
||||
"intro": "可参考的数据集设计、评测基准、室内几何重建方法。",
|
||||
},
|
||||
"world_model_video": {
|
||||
"zh": "F. 视频世界模型（下游应用）",
|
||||
"stage": "M4-4",
|
||||
"intro": "本项目数据 pipeline 的最终下游：训练能预测未来视频/动作的 world model。",
|
||||
},
|
||||
"zed_camera": {
|
||||
"zh": "G. ZED 相机相关应用工作",
|
||||
"stage": "全周期",
|
||||
"intro": "用 ZED 系列采集数据的应用论文，参考其采集协议、评测方式、参数配置。",
|
||||
},
|
||||
"orbbec_gemini": {
|
||||
"zh": "H. Orbbec / Femto / Azure Kinect 相关工作",
|
||||
"stage": "国产化 / 替代硬件",
|
||||
"intro": "奥比中光、乐视/微视 Femto、微软 Azure Kinect 等 RGB-D 相机的应用论文。",
|
||||
},
|
||||
"rgbd_indoor_reconstruction": {
|
||||
"zh": "I. RGB-D 室内重建",
|
||||
"stage": "M2-2 / M3-5",
|
||||
"intro": "RGB-D 输入下的室内场景重建，与本项目的房间级建图任务高度对齐。",
|
||||
},
|
||||
"neural_stereo_depth": {
|
||||
"zh": "J. 神经立体深度（指定 RAFT/IGEV/Foundation 家族）",
|
||||
"stage": "M3-4",
|
||||
"intro": "针对 RAFT-Stereo / IGEV-Stereo / FoundationStereo 等核心立体匹配方法的衍生与改进。",
|
||||
},
|
||||
}
|
||||
|
||||
CN_KEYWORDS = [
|
||||
# 中国机构关键词（用于粗筛"国产团队"论文）
|
||||
"tsinghua", "peking", "fudan", "shanghai jiao", "zhejiang", "ustc", "huazhong",
|
||||
"harbin", "tianjin", "wuhan", "xi'an jiaotong", "xi'an", "nanjing",
|
||||
"chinese academy", "cas ", "casia",
|
||||
"hkust", "cuhk", "hku", "polyu", "city university of hong kong",
|
||||
"alibaba", "tencent", "bytedance", "baidu", "huawei", "megvii",
|
||||
"sensetime", "ant group", "didi", "meituan", "xiaomi",
|
||||
"damo", "noah", "arc lab", "shanghai ai lab",
|
||||
]
|
||||
|
||||
|
||||
def is_china_team(authors: list[str], summary: str) -> bool:
|
||||
"""粗略判断是否有国产团队作者（依赖摘要中机构提及）。"""
|
||||
lower = (" ".join(authors) + " " + summary).lower()
|
||||
return any(k in lower for k in CN_KEYWORDS)
|
||||
|
||||
|
||||
def fmt_authors(authors: list[str], max_n: int = 4) -> str:
|
||||
if not authors:
|
||||
return "（作者信息缺失）"
|
||||
if len(authors) <= max_n:
|
||||
return ", ".join(authors)
|
||||
return ", ".join(authors[:max_n]) + f" 等 ({len(authors)} 人)"
|
||||
|
||||
|
||||
def render_paper(p: dict, idx: int) -> list[str]:
|
||||
aid = p.get("arxiv_id", "")
|
||||
title = p.get("title", "").strip().rstrip(".")
|
||||
pub = p.get("published", "")[:10]
|
||||
authors = p.get("authors", [])
|
||||
summary = p.get("summary", "")
|
||||
cats = p.get("categories", [])
|
||||
url = p.get("url") or f"https://arxiv.org/abs/{aid}"
|
||||
|
||||
china_tag = " 🇨🇳" if is_china_team(authors, summary) else ""
|
||||
|
||||
lines = [
|
||||
f"#### {idx}. [{aid}]({url}) — {title}{china_tag}",
|
||||
f"- **发表**: {pub} | **分类**: {', '.join(cats[:3]) if cats else '-'}",
|
||||
f"- **作者**: {fmt_authors(authors)}",
|
||||
f"- **摘要**: {summary}",
|
||||
"",
|
||||
]
|
||||
return lines
|
||||
|
||||
|
||||
def render_topic(topic: str, papers: list[dict]) -> list[str]:
|
||||
meta = TOPIC_META.get(topic, {"zh": topic, "stage": "-", "intro": ""})
|
||||
out = [
|
||||
f"## {meta['zh']}",
|
||||
f"**项目阶段**: {meta['stage']} | **论文数**: {len(papers)}",
|
||||
"",
|
||||
meta["intro"],
|
||||
"",
|
||||
]
|
||||
|
||||
# 按时间排序（新到旧）
|
||||
papers_sorted = sorted(papers, key=lambda p: p.get("published", ""), reverse=True)
|
||||
|
||||
for i, p in enumerate(papers_sorted, 1):
|
||||
out.extend(render_paper(p, i))
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def render_github_topic(topic: str, repos: list[dict]) -> list[str]:
|
||||
out = [f"### GitHub: {topic}（按 stars 排序）", ""]
|
||||
for r in repos:
|
||||
out.append(
|
||||
f"- [{r['name']}]({r['url']}) — ⭐ {r['stars']:,} | "
|
||||
f"{r.get('language', '-')} | {r.get('description', '')[:120]}"
|
||||
)
|
||||
out.append("")
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--in", dest="inp", default="research/search_results.json")
|
||||
ap.add_argument("--out", default="research/zed2i_arxiv_live_review.md")
|
||||
args = ap.parse_args()
|
||||
|
||||
with open(args.inp, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
arxiv = data.get("arxiv", {})
|
||||
github = data.get("github", {})
|
||||
meta = data.get("meta", {})
|
||||
|
||||
# 统计
|
||||
total_papers = sum(len(v) for v in arxiv.values())
|
||||
total_repos = sum(len(v) for v in github.values())
|
||||
|
||||
all_papers = [p for ps in arxiv.values() for p in ps]
|
||||
china_papers = [p for p in all_papers if is_china_team(p.get("authors", []), p.get("summary", ""))]
|
||||
cat_counter: Counter[str] = Counter()
|
||||
for p in all_papers:
|
||||
for c in p.get("categories", [])[:1]:
|
||||
cat_counter[c] += 1
|
||||
|
||||
lines: list[str] = [
|
||||
"# ZED 2i 数据 Pipeline 实时 arXiv 综述（自动生成）",
|
||||
"",
|
||||
f"> **数据来源**：[`research/search_results.json`](search_results.json) 由 [`research/search_info.py`](search_info.py) 通过 HTTP 代理 `127.0.0.1:6984` 拉取自 arxiv.org / api.github.com。",
|
||||
f"> **生成时间**：{meta.get('generated_at', '-')}",
|
||||
f"> **检索代理**：{meta.get('proxy', '-')}",
|
||||
f"> **每主题最多**：{meta.get('max_results_per_topic', '-')} 篇",
|
||||
"",
|
||||
"## 0. 数据概览",
|
||||
"",
|
||||
f"- **arXiv 论文总数**：{total_papers}",
|
||||
f"- **arXiv 主题数**：{len(arxiv)}",
|
||||
f"- **疑似国产团队论文**：{len(china_papers)}（占比 {len(china_papers)*100//max(total_papers,1)}%；🇨🇳 标记，启发式判断）",
|
||||
f"- **GitHub 仓库总数**：{total_repos}",
|
||||
f"- **GitHub 主题数**：{len(github)}",
|
||||
"",
|
||||
"### 0.1 主类目分布（arXiv primary_category）",
|
||||
"",
|
||||
]
|
||||
for cat, n in cat_counter.most_common(10):
|
||||
lines.append(f"- `{cat}`: {n}")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"### 0.2 与本项目框架的映射",
|
||||
"",
|
||||
"| 项目阶段 | 主线主题 | 论文数 |",
|
||||
"|---|---|---|",
|
||||
]
|
||||
)
|
||||
for topic, m in TOPIC_META.items():
|
||||
n = len(arxiv.get(topic, []))
|
||||
lines.append(f"| {m['stage']} | {m['zh']} | {n} |")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"---",
|
||||
"",
|
||||
"# 第一部分 · arXiv 论文（按主题分组，时间新→旧）",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
for topic in TOPIC_META.keys():
|
||||
papers = arxiv.get(topic, [])
|
||||
if not papers:
|
||||
continue
|
||||
lines.extend(render_topic(topic, papers))
|
||||
lines.append("---")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"# 第二部分 · GitHub 仓库（按 stars 排序）",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for topic, repos in github.items():
|
||||
lines.extend(render_github_topic(topic, repos))
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"---",
|
||||
"",
|
||||
"# 第三部分 · 关键洞察与项目对接建议",
|
||||
"",
|
||||
"## I.1 最值得关注的新论文（按相关性挑选）",
|
||||
"",
|
||||
"下方挑选每个主题中**与本项目最相关的 3 篇**（基于标题/摘要语义判断）：",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
# 简单挑选每个主题前 3 篇放在洞察区
|
||||
for topic in ["stereo_matching", "visual_inertial_slam", "gaussian_splatting_slam", "monocular_depth_foundation"]:
|
||||
papers = arxiv.get(topic, [])
|
||||
if not papers:
|
||||
continue
|
||||
meta = TOPIC_META[topic]
|
||||
lines.append(f"### {meta['zh']} → {meta['stage']}")
|
||||
for p in sorted(papers, key=lambda x: x.get("published", ""), reverse=True)[:3]:
|
||||
lines.append(f"- **[{p['arxiv_id']}]({p['url']})** {p['title']}（{p['published'][:10]}）")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"## I.2 后续动作清单",
|
||||
"",
|
||||
"- [ ] 把上述每主题的 Top-3 论文加入 [`research/zed2i_stereo_vio_arxiv_review.md`](zed2i_stereo_vio_arxiv_review.md) 第 F 节的论文映射表",
|
||||
"- [ ] 对 🇨🇳 标记的论文重点核查机构归属，更新 G 节国产团队清单",
|
||||
"- [ ] 把 GitHub 仓库中 stars > 5k 的项目加入 [`plans/camera/github_opensource_projects.md`](../plans/camera/github_opensource_projects.md)",
|
||||
"- [ ] 每周重跑 [`research/search_info.py`](search_info.py) 增量更新",
|
||||
"",
|
||||
"## I.3 复现方法",
|
||||
"",
|
||||
"```bash",
|
||||
"# 通过 127.0.0.1:6984 代理拉取最新数据",
|
||||
"HTTPS_PROXY=http://127.0.0.1:6984 HTTP_PROXY=http://127.0.0.1:6984 \\",
|
||||
" python3 research/search_info.py \\",
|
||||
" --proxy http://127.0.0.1:6984 \\",
|
||||
" --max-results 10 --delay 5.0 \\",
|
||||
" --out research/search_results.json",
|
||||
"",
|
||||
"# 渲染为 markdown",
|
||||
"python3 research/gen_review_from_json.py \\",
|
||||
" --in research/search_results.json \\",
|
||||
" --out research/zed2i_arxiv_live_review.md",
|
||||
"```",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"**说明**：",
|
||||
"- 本文档由脚本自动生成，可重复执行覆盖",
|
||||
"- 🇨🇳 标记基于作者/摘要中是否包含中国机构关键词的启发式判断，**仅供参考，需人工复核**",
|
||||
"- 摘要截断到 400 字符以控制文档体积",
|
||||
"- 与 [`zed2i_stereo_vio_arxiv_review.md`](zed2i_stereo_vio_arxiv_review.md)（人工综述）互为补充：人工综述给方法论与映射，本文档给最新原始素材",
|
||||
]
|
||||
)
|
||||
|
||||
with open(args.out, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(lines))
|
||||
|
||||
print(f"✅ 生成: {args.out}")
|
||||
print(f" arXiv: {total_papers} 篇，国产疑似: {len(china_papers)}")
|
||||
print(f" GitHub: {total_repos} 仓库")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,394 @@
|
||||
# 人类空间记忆：大脑如何构建、存储与更新空间信息
|
||||
|
||||
> 研究目标：系统梳理人类大脑构建空间认知地图的神经机制，提取可工程化的设计原则，指导 PRISM 从 v1.5 升级至 v2.0。
|
||||
>
|
||||
> 版本：v1.0 | 日期：2026-05-17
|
||||
|
||||
---
|
||||
|
||||
## 0. 一句话
|
||||
|
||||
> **大脑用「稀疏编码的位置信号 + 周期性网格度量 + 分层巩固」三件套，在几瓦功耗下实现了终身空间记忆。PRISM 2.0 要做的就是把这三件套翻译成工程架构。**
|
||||
|
||||
---
|
||||
|
||||
## 1. 核心神经系统：海马-内嗅皮层回路
|
||||
|
||||
人类空间记忆的核心硬件是**海马体 (Hippocampus)** 和**内嗅皮层 (Entorhinal Cortex)** 组成的回路。这个回路在哺乳动物中高度保守——老鼠、猴子、人类的导航系统原理相同。
|
||||
|
||||
```
|
||||
┌──────────────────────┐
|
||||
│ 内嗅皮层 (MEC) │
|
||||
│ ┌──────┬──────┬───┐ │
|
||||
│ │Grid │Head │ │ │
|
||||
│ │Cells │Dir. │… │ │
|
||||
│ └──────┴──────┴───┘ │
|
||||
└──────────┬───────────┘
|
||||
│ 输入
|
||||
┌──────────▼───────────┐
|
||||
│ 海马体 │
|
||||
│ DG → CA3 → CA1 │
|
||||
│ ┌──────┬──────┬───┐ │
|
||||
│ │Place │Time │… │ │
|
||||
│ │Cells │Cells │ │ │
|
||||
│ └──────┴──────┴───┘ │
|
||||
└──────────┬───────────┘
|
||||
│ 输出
|
||||
┌──────────▼───────────┐
|
||||
│ 新皮层 (PFC等) │
|
||||
│ 长期存储 │
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. 五种基础空间细胞类型
|
||||
|
||||
### 2.1 位置细胞 (Place Cells) — 你在哪里
|
||||
|
||||
| 属性 | 说明 |
|
||||
|------|------|
|
||||
| **位置** | 海马体 CA1 / CA3 |
|
||||
| **行为** | 当动物进入特定位置时放电，每个细胞只对应一个或几个位置域 (place field) |
|
||||
| **编码** | **稀疏编码**——海马体中只有 ~2-5% 的神经元对当前位置活跃 |
|
||||
| **关键特性** | 位置域在进入新环境**几分钟内**形成，且一旦形成就相对稳定 |
|
||||
| **重映射 (Remapping)** | 环境改变后，同一群神经元会重新分配到新位置——这是"新地图"的神经信号 |
|
||||
|
||||
> **工程启发**：PRISM 应该为每个"位置"维护稀疏的视觉指纹（已有 clip_embedding），但需要支持**快速形成新位置表征 + 环境变化时触发重映射**。
|
||||
|
||||
### 2.2 网格细胞 (Grid Cells) — 走了多远、什么方向
|
||||
|
||||
| 属性 | 说明 |
|
||||
|------|------|
|
||||
| **位置** | 内侧内嗅皮层 (medial Entorhinal Cortex, MEC) |
|
||||
| **行为** | 多个空间周期性地放电，形成**正六边形网格** |
|
||||
| **编码** | 每个细胞有特定的间距 (spacing)、方向 (orientation) 和相位 (phase) |
|
||||
| **层级性** | 不同网格细胞的间距从 ~30 cm 到数米不等，**间距越大越靠 MEC 背侧** |
|
||||
| **通用性** | 网格表征**跨环境通用**——同一个网格细胞在不同房间里保持相同的间距和方向 |
|
||||
|
||||
> **工程启发**：这是 PRISM 当前**最缺的一层**。L2 的 OctoMap / TSDF 是纯笛卡尔坐标，没有网格细胞那种"泛化度量"。应该增加一个 **GridMetric 层**——由多个不同间距的周期编码叠加，实现跨房间的度量泛化。
|
||||
|
||||
### 2.3 头方向细胞 (Head Direction Cells) — 面向哪边
|
||||
|
||||
| 属性 | 说明 |
|
||||
|------|------|
|
||||
| **位置** | 多个脑区（前背侧丘脑、后下托、MEC 等） |
|
||||
| **行为** | 只对特定头部方向放电，与位置和速度无关 |
|
||||
| **编码** | 360° 环形吸引子网络，每个细胞调谐到特定角度 |
|
||||
|
||||
> **工程启发**：ZED VIO 已提供朝向，但缺少与空间记忆的直接耦合。PRISM 2.0 应该让朝向信号与位置表征**联合编码**（而非仅作为 Pose 的一个分量）。
|
||||
|
||||
### 2.4 边界细胞 (Boundary Cells) — 离墙多远
|
||||
|
||||
| 属性 | 说明 |
|
||||
|------|------|
|
||||
| **位置** | 下托 (Subiculum)、MEC |
|
||||
| **行为** | 在距环境边界特定距离处放电；部分编码**距最近墙的距离**，部分编码**特定方位的墙** |
|
||||
| **功能** | 定义环境的几何框架——没有它，位置细胞就无法稳定地重新映射 |
|
||||
|
||||
> **工程启发**：iPhone RoomPlan 输出的墙/隔断 = 天然边界。PRISM 2.0 应在 L2 初始化时显式计算每个位置的**边界距离特征**，作为位置细胞形成的基础脚手架。
|
||||
|
||||
### 2.5 速度细胞 (Speed Cells) — 移动快慢
|
||||
|
||||
| 属性 | 说明 |
|
||||
|------|------|
|
||||
| **位置** | MEC |
|
||||
| **行为** | 放电率线性正比于运行速度 |
|
||||
| **功能** | 驱动网格细胞的相位更新——没有速度信号，网格就会"静止" |
|
||||
|
||||
> **工程启发**：ZED VIO 的速度估计可直接作为网格更新的驱动力。与路径积分 (path integration) 直接对应。
|
||||
|
||||
---
|
||||
|
||||
## 3. 海马体的两个核心操作：模式分离与模式完成
|
||||
|
||||
### 3.1 模式分离 (Pattern Separation) — "这两个房间不一样"
|
||||
|
||||
```
|
||||
输入 ──→ 齿状回 (DG) ──→ CA3
|
||||
│
|
||||
大数量神经元 + 极稀疏活动
|
||||
→ 相似的输入被"正交化"为不重叠的表征
|
||||
```
|
||||
|
||||
| 场景 | 例子 |
|
||||
|------|------|
|
||||
| 两个布局相似的酒店房间 | 人不会搞混——DG 把相似的感知输入映射为不同的海马表征 |
|
||||
| 同一个房间，家具挪了 | CA3 仍然能认出（模式完成），但 DG 会标记"有变化" |
|
||||
|
||||
> **工程启发**：PRISM 需要 **DG 层**——当 ZED 进入一个与已有 L3 节点视觉相似但实际不同的房间时，DG 层应该强制新建节点而非错误地"认出"旧节点。当前只有 `clip_embedding` 做相似度匹配，缺少正交化步骤。
|
||||
|
||||
### 3.2 模式完成 (Pattern Completion) — "我只看到床角，但知道这是卧室"
|
||||
|
||||
```
|
||||
部分线索 ──→ CA3 (自联想网络) ──→ 完整记忆
|
||||
│
|
||||
CA3 有丰富的递归连接
|
||||
→ 部分输入即可激活整个记忆模式
|
||||
```
|
||||
|
||||
| 场景 | 例子 |
|
||||
|------|------|
|
||||
| 重定位 | 只看到房间一角，就能识别是哪个房间（PRISM Pipeline B 的核心需求） |
|
||||
| 弱光/遮挡 | 部分视野被遮挡，仍然能导航 |
|
||||
|
||||
> **工程启发**：PRISM 的重定位模块已经在做类似的事（CLIP embedding 匹配），但缺少 CA3 风格的**递归完成**——即用部分匹配激活的节点反过来"期望"看到哪些物品，再去 L4 验证，形成一个**双向确认**循环。
|
||||
|
||||
---
|
||||
|
||||
## 4. 记忆的三种时态：工作 → 短期 → 长期
|
||||
|
||||
人类的空间记忆不是"存进去就完事"，而是沿着**时间轴**经历了三种状态：
|
||||
|
||||
```
|
||||
感知输入
|
||||
│
|
||||
▼
|
||||
┌────────────────┐
|
||||
│ 工作记忆 (WM) │ ← 前额叶皮层 + 海马体
|
||||
│ 秒 ~ 分钟 │ 容量有限 (4±1 chunks)
|
||||
│ 当前任务上下文 │ 表征当前导航目标、最近经过的路标
|
||||
└────────┬───────┘
|
||||
│ 注意筛选 (只有"重要的"往下走)
|
||||
▼
|
||||
┌────────────────┐
|
||||
│ 短期记忆 (STM) │ ← 海马体依赖
|
||||
│ 小时 ~ 天 │ 突触可塑性 (LTP/LTD)
|
||||
│ 今天的经历 │ 快速编码，但不稳定
|
||||
└────────┬───────┘
|
||||
│ 睡眠巩固 (海马→皮层转移)
|
||||
│ 去重 + 与旧知识整合
|
||||
▼
|
||||
┌────────────────┐
|
||||
│ 长期记忆 (LTM) │ ← 新皮层（前额叶、颞叶等）
|
||||
│ 周 ~ 年 ~ 终身 │ 结构可塑性（树突棘稳定化）
|
||||
│ 稳定的知识 │ 半永久存储
|
||||
└────────────────┘
|
||||
```
|
||||
|
||||
### 三层的关键区别（PRISM 映射）
|
||||
|
||||
| 维度 | WM | STM | LTM | PRISM 当前对应 |
|
||||
|------|----|-----|-----|---------------|
|
||||
| **时间** | 秒~分钟 | 小时~天 | 周~终身 | L1/WM ✓ / L2-L4(部分) / 缺STM显式层 |
|
||||
| **容量** | 极小 | 有限 | 几乎无限 | — |
|
||||
| **可塑性** | 随时覆盖 | 可修改 | 难修改 | — |
|
||||
| **依赖** | 注意力 | 海马体 | 皮层 | — |
|
||||
| **遗忘** | 瞬时 | 干扰/衰减 | 极少 | — |
|
||||
|
||||
> **工程启发**：PRISM 目前只有 L1 (WM-like) 和"其他全算持久化"。缺失了中间的 **STM 层**——即"今天见到但还没巩固"的状态。这导致差异检测 (`delta/`) 直接写死而非可恢复的临时标记。
|
||||
|
||||
---
|
||||
|
||||
## 5. 系统巩固：睡眠中把记忆从海马搬到皮层
|
||||
|
||||
### 5.1 标准巩固模型 (Standard Consolidation Model)
|
||||
|
||||
```
|
||||
睡眠 / 安静清醒时：
|
||||
|
||||
海马体 ──Sharp-Wave Ripples (SWR)──→ 新皮层
|
||||
│ │
|
||||
以 10-20× 加速重播 缓慢调整突触权重
|
||||
白天的轨迹 把"昨天发生的事"
|
||||
变成"已知的事实"
|
||||
```
|
||||
|
||||
关键发现：
|
||||
- **海马重播 (Replay)**：清醒时经历的轨迹在 NREM 睡眠期间被压缩到 ~100 ms 的 SWR 事件中快速重放
|
||||
- **皮层巩固**：每次重播驱动皮层突触的微小变化，经多次睡眠周期后形成稳定表征
|
||||
- **去重与抽象**：大脑不是逐字记录——重播过程会**抽取共性、丢弃细节、与已有知识整合**
|
||||
|
||||
### 5.2 补充学习系统理论 (Complementary Learning Systems, CLS)
|
||||
|
||||
| | 海马体系统 | 皮层系统 |
|
||||
|---|-----------|---------|
|
||||
| **学习速度** | 快速（一次经历即可） | 慢速（需要多次重复） |
|
||||
| **表征** | 稀疏、分离（防止干扰） | 重叠、压缩（提取统计规律） |
|
||||
| **作用** | 记忆特定事件 | 学习通用知识 |
|
||||
| **类比** | 内存（快但容小） | 硬盘（慢但容大） |
|
||||
|
||||
> **工程启发**：Pipeline D（充电巩固）已经捕获了"睡眠"的直觉，但缺少两个核心操作：
|
||||
> 1. **显式重播**——不是简单地 "delta 变永久"，而是**重新走过白天的路径**，用重播后的表征更新各层
|
||||
> 2. **双系统学习**——新增物体先以"快速、稀疏、可能有噪声"的形式写入，Consolidator 再以"慢速、去噪、整合"的模式转入长期
|
||||
|
||||
---
|
||||
|
||||
## 6. 预测编码：大脑是预测机器
|
||||
|
||||
### 6.1 核心思想
|
||||
|
||||
```
|
||||
┌────── 自上而下的预测 ──────┐
|
||||
│ │
|
||||
高层表征 感知输入
|
||||
│ │
|
||||
└────── 自下而上的误差 ←──────┘
|
||||
|
||||
大脑不断生成对下一刻的预测 → 对比实际输入 → 只把"意外"（预测误差）向上传。
|
||||
```
|
||||
|
||||
### 6.2 空间导航中的预测
|
||||
|
||||
| 预测类型 | 例子 | 脑区 |
|
||||
|---------|------|------|
|
||||
| **感官预测** | "转过去应该看到沙发" | 感觉皮层 ↔ 海马体 |
|
||||
| **运动预测** | "走 10 步应该到门口" | 小脑 + MEC |
|
||||
| **物体预测** | "卧室里应该有一张床" | 前额叶 + 海马体 |
|
||||
|
||||
当预测失败（打开门发现房间被重新布置了）→ 预测误差信号 → 触发**注意 + 学习 + 记忆更新**。
|
||||
|
||||
> **工程启发**：PRISM 当前是**被动接受数据**——ZED 来了就写，差异检测只是"发现了"就记录。缺少**主动预测**层。PRISM 2.0 应该在每帧都生成"期望看到什么"，只有当偏差超过阈值才触发写入和更新。
|
||||
|
||||
---
|
||||
|
||||
## 7. 记忆更新：再巩固而非覆写
|
||||
|
||||
### 7.1 重新巩固 (Reconsolidation)
|
||||
|
||||
经典模型认为记忆"巩固后就稳定了"。2000 年后的研究发现：**已被巩固的记忆在被再次激活（回忆）时，会短暂回到不稳定状态，允许修改后再重新巩固。**
|
||||
|
||||
```
|
||||
稳定记忆 ──回忆触发──→ 不稳定状态 ──整合新信息──→ 再巩固（更新版）
|
||||
│
|
||||
└── 如果不重新巩固 → 记忆消退
|
||||
```
|
||||
|
||||
对 PRISM 的启示：
|
||||
- 当 ZED 重新观察到 L4 中的一个已知物品时 → 不应直接覆写，而是**先"解锁"该记忆，融合新观测，再"重新封印"**
|
||||
- 连续观测不一致 → 降低 confidence 而非直接删除
|
||||
- 已标记 `moved` 的物品 → 短暂进入"可修改窗口"，允许位置更新
|
||||
|
||||
### 7.2 去稳定化的条件
|
||||
|
||||
什么导致一个记忆变得可修改？
|
||||
|
||||
| 条件 | 神经机制 | PRISM 对应 |
|
||||
|------|---------|-----------|
|
||||
| **预测误差** | 打开门，床不在原位 → LC 释放去甲肾上腺素 | `delta/` 差异检测 |
|
||||
| **新奇性** | 出现不认识的物品 → 海马体 CA1 强响应 | 未匹配到 L4 节点的新检测 |
|
||||
| **上下文变化** | 同样的房间但灯光/时间不同 | 时间戳跨度大 + 场景视觉差异大 |
|
||||
|
||||
> **工程启发**：PRISM 需要显式的 **Reconsolidation Flag**——不是所有差异都立即写，而是先标记"待重新协商"，在 Consolidator 阶段才决定采纳/部分采纳/拒绝。
|
||||
|
||||
---
|
||||
|
||||
## 8. 双流视觉加工：Where 通路 vs What 通路
|
||||
|
||||
### 8.1 两条通路
|
||||
|
||||
| | 背侧通路 (Dorsal) | 腹侧通路 (Ventral) |
|
||||
|---|------------------|-------------------|
|
||||
| **俗称** | Where / How | What |
|
||||
| **路径** | 初级视皮层 → 后顶叶 | 初级视皮层 → 颞下回 |
|
||||
| **表征** | 以自我为中心 (egocentric) | 以物体为中心 (allocentric) |
|
||||
| **功能** | 引导动作、伸手抓取、避障 | 物体识别、语义理解 |
|
||||
| **速度** | 快、实时 | 慢、需要"辨认" |
|
||||
| **记忆** | 不持久 | 可持久 |
|
||||
|
||||
### 8.2 两条通路在海马体汇合
|
||||
|
||||
海马体是 where + what 的**汇合点**——它同时接收来自两条通路的信息，把它们绑定成一个"在某个地方有某个东西"的完整记忆（episodic memory 的核心）。
|
||||
|
||||
> **工程启发**：PRISM 的 L2 (度量/where) 和 L4 (语义/what) 已经体现了两条通路，但**缺少海马体式的汇合层**——即把一个物品和它的精确位置绑定为一个不可分割的 episode。当前物品的位置只是 L4Node 的一个 `pose` 字段，而非与 L2 几何深度耦合的绑定。
|
||||
|
||||
---
|
||||
|
||||
## 9. 空间注意力与显著性
|
||||
|
||||
### 9.1 大脑不记录所有东西
|
||||
|
||||
每秒钟视网膜输入 ~10^8 bit，但只有 ~10^1-10^2 bit 进入意识/记忆。大脑通过**显著性过滤器**决定什么值得记住：
|
||||
|
||||
| 过滤器 | 机制 | 例子 |
|
||||
|--------|------|------|
|
||||
| **空间新奇** | 新房间、新走廊 → 海马体强编码 | 第一次进酒店大堂 |
|
||||
| **物体新奇** | 没见过的东西 → 多巴胺释放 | 新放的装饰品 |
|
||||
| **变化检测** | 预期被违反 → 预测误差 | 椅子不在原来位置 |
|
||||
| **目标相关** | 当前任务相关的物品 → 前额叶偏置 | 找遥控器时注意茶几 |
|
||||
| **情感标记** | 情感事件 → 杏仁核调节记忆强度 | (机器人场景不适用) |
|
||||
|
||||
### 9.2 显著性 → 编码强度
|
||||
|
||||
不是所有的经历都以相同精度存储。高显著性事件 → 更强的突触可塑性 → 更持久、更精确的记忆。低显著性 → 只保留模糊的统计信息。
|
||||
|
||||
> **工程启发**：PRISM 需要一个 **Salience Gate**——不是 2 Hz 的 VLM 检测全部写入，而是根据"这个检测有多意外/多重要"动态决定**写入精度和持久性**。
|
||||
|
||||
---
|
||||
|
||||
## 10. 最新研究进展 (2020-2025)
|
||||
|
||||
### 10.1 继任表征 (Successor Representations)
|
||||
|
||||
内嗅皮层不只编码"当前位置"，还编码**从当前位置出发，未来可能访问的状态**——即对环境的"预测地图"。
|
||||
|
||||
- 一个位置的 SR 不只看那个位置本身，而是编码"从这里最容易到达哪里"
|
||||
- 这解释了为什么人能瞬间判断"从卧室到厨房要经过走廊"而不需要显式跑 A*
|
||||
|
||||
> **工程启发**：PRISM 的 L3 边目前只存 `cost` (距离/难度)。PRISM 2.0 应存储 **SR 向量**——从每个房间到所有其他房间的**期望到达频率**，让路径规划从 A* 降级为一次矩阵乘法。
|
||||
|
||||
### 10.2 物体向量细胞 (Object-Vector Cells)
|
||||
|
||||
MEC 中发现了一类新细胞：当动物处于某个物体（如障碍物）的特定方向和距离时放电。
|
||||
|
||||
- 与位置细胞不同——它不关心绝对位置，而是**相对于地标的位置**
|
||||
- 解释了为什么人能"走离沙发两米然后右转"而不需要看地图
|
||||
|
||||
> **工程启发**：PRISM 2.0 的 L2 应增加 **Landmark-Relative Coordinates**——不仅存全局位姿，还存相对于最近锚点物体 (L4 家具) 的位置。这在 GPS 不可用场景下比纯 VIO 更鲁棒。
|
||||
|
||||
### 10.3 社交位置细胞 (Social Place Cells)
|
||||
|
||||
海马体有专门编码"另一个人（或物体）所在位置"的神经元——**空间记忆天然是社会性的**。
|
||||
|
||||
> **工程启发**：多机器人场景下，PRISM 应为每个机器人/人维护一个独立的位置轨迹——"小明在厨房" 成为一个可查询的 L4 属性。
|
||||
|
||||
### 10.4 认知地图的泛化
|
||||
|
||||
2024-2025 年的研究表明，海马-内嗅系统的"认知地图"不仅编码物理空间，还编码**抽象空间**——概念距离、社交关系、甚至图表结构。同一套神经机制在不同领域重复使用。
|
||||
|
||||
> **工程启发**：PRISM 的架构可能不仅适用于物理空间——如果抽象得好，同一个 Schema 可以扩展为机器人对"任务流"、"时间线"、甚至"技能树"的记忆。
|
||||
|
||||
---
|
||||
|
||||
## 11. 总结：从大脑到 PRISM 2.0 的设计原则
|
||||
|
||||
| # | 神经机制 | 当前 PRISM 1.5 | → PRISM 2.0 | 优先级 |
|
||||
|---|---------|---------------|-------------|--------|
|
||||
| A | 网格细胞：跨环境的度量泛化 | ❌ 缺失 | 新增 GridMetric 层 | **P0** |
|
||||
| B | 模式分离 (DG)：区分相似场景 | ❌ 只有 CLIP 相似度 | 新增正交化模块 | **P0** |
|
||||
| C | 模式完成 (CA3)：部分线索→完整回忆 | ⚠️ 单向匹配 | 改为双向递归确认 | P1 |
|
||||
| D | 预测编码：预期→误差驱动更新 | ❌ 被动写入 | 新增预测层 | **P0** |
|
||||
| E | 系统巩固：海马重播 + CLS 双系统 | ⚠️ Pipeline D 太粗糙 | 显式重播 + 快慢双系统 | P1 |
|
||||
| F | 再巩固：解锁→更新→重新封印 | ❌ 直接覆写 | 引入 Reconsolidation 流程 | P1 |
|
||||
| G | 显著性门控：不是所有东西都该记 | ❌ 全量写入 | 新增 Salience Gate | P1 |
|
||||
| H | 继任表征：预测未来状态 | ❌ 只有 A* | L3 边存储 SR 向量 | P2 |
|
||||
| I | 物体向量细胞：地标相对定位 | ❌ 只有全局位姿 | L2 增加地标相对坐标 | P2 |
|
||||
| J | 边界细胞：以墙为骨架 | ⚠️ 房间 polygon | 显式计算边界距离特征 | P1 |
|
||||
| K | Where/What 汇合：位置-物品绑定 | ⚠️ 松散关联 | 紧耦合 episodic binding | P2 |
|
||||
|
||||
**P0 = 必须在 2.0 实现，P1 = 2.0 应包含，P2 = 2.1 或后续**
|
||||
|
||||
---
|
||||
|
||||
## 12. 关键参考
|
||||
|
||||
| 发现 | 年份 | 核心贡献 | 诺贝尔奖 |
|
||||
|------|------|---------|---------|
|
||||
| 位置细胞 (O'Keefe & Dostrovsky) | 1971 | 海马体编码空间位置 | 2014 |
|
||||
| 认知地图 (Tolman) | 1948 | 动物形成内部空间表征，不仅靠刺激-反应 | — |
|
||||
| 网格细胞 (Hafting, Fyhn, Molden, Moser & Moser) | 2005 | 内嗅皮层六边形网格编码 | 2014 |
|
||||
| 头方向细胞 (Taube, Muller, Ranck) | 1990 | 独立于位置的方向编码 | — |
|
||||
| 边界细胞 (Solstad et al. / Lever et al.) | 2008 | 距边界特定距离放电 | — |
|
||||
| 系统巩固 (McClelland, McNaughton, O'Reilly) | 1995 | 互补学习系统 (CLS) 理论 | — |
|
||||
| 海马重播 (Wilson & McNaughton / Skaggs & McNaughton) | 1994-1996 | 睡眠 SWR 中重播清醒轨迹 | — |
|
||||
| 再巩固 (Nader, Schafe, LeDoux) | 2000 | 已巩固记忆可被重新不稳定化 | — |
|
||||
| 预测编码 (Rao & Ballard / Friston) | 1999-2005 | 大脑通过预测误差驱动学习 | — |
|
||||
| 继任表征 (Stachenfeld, Botvinick, Gershman / Momennejad et al.) | 2017-2022 | 内嗅皮层编码预测性地图 | — |
|
||||
| 物体向量细胞 (Høydal, Skytøen, Andersson, Moser & Moser) | 2019 | 相对地标的方向和距离编码 | — |
|
||||
| Lyra 2.0 (Shen et al., NVIDIA) | 2026 | 几何只做路由不做合成 | — |
|
||||
|
||||
---
|
||||
|
||||
**文档版本**：v1.0
|
||||
**撰写日期**：2026-05-17
|
||||
**下一步**：将此文档中的 P0/P1 原则转化为 [`plans/PRISM/20_v2_upgrade.md`](../plans/PRISM/20_v2_upgrade.md) 的具体架构改动
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,354 @@
|
||||
# Lyra 2.0:可探索的生成式 3D 世界 — 中文译读
|
||||
|
||||
> **原文**:[Lyra 2.0: Explorable Generative 3D Worlds](https://arxiv.org/abs/2604.13036) (arXiv:2604.13036v1, 2026-04-14)
|
||||
> **作者**:Tianchang Shen\*、Sherwin Bahmani、Kai He、Sangeetha Grama Srinivasan、Tianshi Cao、Jiawei Ren、Ruilong Li、Zian Wang、Nicholas Sharp、Zan Gojcic、Sanja Fidler、Jiahui Huang、Huan Ling、Jun Gao、Xuanchi Ren\* (\* equal contribution)
|
||||
> **机构**:NVIDIA Spatial Intelligence Lab + 多伦多大学
|
||||
> **项目页**:<https://research.nvidia.com/labs/sil/lyra2/>
|
||||
> **PDF 本地**:[`research/lyra2_paper.pdf`](lyra2_paper.pdf) (13 MB, 169 引用文献)
|
||||
> **译读版本**:v1.0,撰写于 2026-05-16
|
||||
|
||||
---
|
||||
|
||||
## 0. 一句话总结
|
||||
|
||||
> **"用 80 帧滑动窗口的视频扩散模型 + 几何只用作路由(不参与外观合成) + 自增强训练抗漂移,实现从一张照片到任意长度可走通的 3D 场景。"**
|
||||
|
||||
Lyra 2.0 的核心贡献是**解决长程视频生成的两大顽疾**:**空间遗忘**(spatial forgetting,镜头转回去时拼不回原貌)与 **时间漂移** (temporal drifting,自回归累积误差导致色彩/几何越生成越烂)。
|
||||
|
||||
---
|
||||
|
||||
## 1. 问题背景:从 video diffusion 到 explorable 3D world
|
||||
|
||||
### 1.1 什么是 generative reconstruction (生成式重建)
|
||||
|
||||
给定**一张图 + 一条相机轨迹**,流程是:
|
||||
1. **视频扩散模型** (DiT-based, Wan 2.1-14B) 沿轨迹合成稠密新视角视频 →
|
||||
2. **前馈 3D 重建**(Depth Anything v3 → 3D Gaussian Splatting / mesh)→
|
||||
3. 得到可被 NVIDIA Isaac Sim 等仿真器直接吃的 3D 资产。
|
||||
|
||||
→ **替代真实采集**,可批量造出多样化、虚构、甚至超大尺度的 3D 环境。
|
||||
|
||||
### 1.2 长程探索的两个失败模式
|
||||
|
||||
| 失败模式 | 物理直觉 | 现象 |
|
||||
|---|---|---|
|
||||
| **空间遗忘 (spatial forgetting)** | 相机走远后,早期看过的区域**超出模型时间上下文窗口**,转回去时只能凭空"幻想"结构 | 同一面墙第二次看是不一样的 |
|
||||
| **时间漂移 (temporal drifting)** | 每帧合成的小误差**自回归地累积** | 色偏、模糊、几何扭曲越生成越严重 |
|
||||
|
||||
### 1.3 现有方案为何不够
|
||||
|
||||
| 方案 | 谁在做 | 缺点 |
|
||||
|---|---|---|
|
||||
| 累积全局 3D 表征做 conditioning | GEN3C / SPMem / WorldExplorer | 早期深度估计的小误差→**几何被污染**→后续生成被错误引导,**误差放大** |
|
||||
| 把历史帧塞进 attention(用 camera pose embed) | CameraCtrl / Yume | 在大视角变化下,**纯自注意力难以推出长程几何对应** |
|
||||
| 扩长 temporal context | FramePack | 早期帧滑出 FOV,**对新区域无帮助** |
|
||||
|
||||
Lyra 2.0 选择**桥接两条路径**——既保留 3D 几何记忆,又**只用它做"路由",不用它做"合成"**。
|
||||
|
||||
---
|
||||
|
||||
## 2. 核心思想(三句话讲完)
|
||||
|
||||
1. **空间记忆 ≠ 用来渲染,只用来检索**——维护**每帧独立**的 3D 缓存,通过视点重投影,挑出最相关的历史帧塞进 attention 上下文,**让扩散模型自己合成像素**。
|
||||
2. **几何对应用规范坐标(canonical coords)而非 warped RGB**——前者只携带几何信息,不带 disocclusion / 拉伸 / 颜色 bleeding,模型不会"以为这些瑕疵是真的"。
|
||||
3. **自增强训练 (Self-Augmentation)**——训练时**用模型自己的 1-step 去噪结果当历史**,让它在训练阶段就学会"接住坏输入并修正",从而抗推理时累积误差。
|
||||
|
||||
---
|
||||
|
||||
## 3. 方法详解(§ 4)
|
||||
|
||||
### 3.1 整体流水线(Retrieve-Generate-Update 循环)
|
||||
|
||||
```
|
||||
单图 I₀
|
||||
│
|
||||
▼ ── 用户给一段相机轨迹 + (可选)文本 prompt ──
|
||||
检索阶段:在空间记忆 𝒞 中找 Nₛ=5 个对目标视角"最可见"的历史帧
|
||||
│
|
||||
▼
|
||||
生成阶段:DiT 在 [anchor + 空间 slots + 时间 slots + g₂₀ 生成 token] 上做 flow-matching 去噪
|
||||
│
|
||||
▼
|
||||
更新阶段:用 Depth Anything v3 估计新帧深度 → 写回 𝒞 (每帧独立)
|
||||
│
|
||||
▼ 重复
|
||||
80 帧/段, 35-step 去噪/段 → 1 段 ~194 s on GB200
|
||||
─ DMD 蒸馏后 4-step → ~15 s/段 (13× 加速)
|
||||
```
|
||||
|
||||
最后:把累积视频一次性喂给微调过的 Depth Anything v3 做**前馈 3DGS**,再用 OpenVDB 稀疏分层做 marching-cubes 出 mesh。
|
||||
|
||||
### 3.2 模块 1:Anti-Forgetting(抗空间遗忘)— § 4.2
|
||||
|
||||
#### (a) 3D 缓存的结构 — **每帧独立,绝不融合**
|
||||
|
||||
对每个生成帧 $I_i$ 估深度 $D_i$ + 已知 $(T_i, K_i)$,缓存两份:
|
||||
1. **全分辨率深度图 $D_i$** + 相机参数(后续做 dense correspondence 用)
|
||||
2. **下采样点云 $P_i \in \mathbb{R}^{(H/d)\times(W/d)\times 3}$**(子采样 d=8,只用于检索可见性打分)
|
||||
|
||||
> **关键设计**:**绝不把多帧融合成一个全局点云**!理由:生成视频的深度估计随时间退化,如果融合就把"小错误"累积为"全局错误"。**每帧独立 = 错了只错那帧,不会扩散。**
|
||||
|
||||
#### (b) 几何感知检索 (Geometry-Aware Retrieval)
|
||||
|
||||
给目标相机 $(T^*, K^*)$,对每个历史帧:
|
||||
1. 把 $P_i$ 投影到目标像平面
|
||||
2. 对每个目标像素,取所有投影中**最浅深度**(处理遮挡)
|
||||
3. 一个点视为"可见" iff 其深度与最小深度差 $< \delta = 0.1$(归一化深度单位)
|
||||
4. **可见性得分 $\varphi(i)$ = 可见点数**
|
||||
|
||||
**贪心覆盖最大化**:迭代选择最大化"尚未覆盖目标像素数"的帧,共选 $N_s = 5$ 帧。避免选很近的几张冗余视图。
|
||||
|
||||
> **效果**:即使相机几百帧后转回原地,**远超时间上下文窗口**,也能通过 3D 重叠把当时的几张关键帧准确召回。
|
||||
|
||||
#### (c) 把空间记忆塞进 DiT
|
||||
|
||||
完整 token 布局(每个 $\mathtt{f}_n \mathtt{k}_m$ = n 帧用 m 倍空间子采样后做 patchify):
|
||||
|
||||
```
|
||||
[anchor I₀] [4 spatial slots @ k=2 + 1 @ k=1] [time slots f₁k₁ f₂k₂ f₁k₁ f₁₆k₄] [g₂₀ 生成 20 帧]
|
||||
```
|
||||
|
||||
> 即:**初始锚帧 + 5 张空间记忆(检索得来)+ FramePack 时间压缩(近密远疏)+ 20 帧生成目标**,全部 jointly attend。
|
||||
|
||||
#### (d) 用 canonical coordinates 做 dense correspondence
|
||||
|
||||
**不**把 retrieved frames 直接 warp 成 RGB(因为会有 disocclusion 黑洞 + 拉伸 + 颜色 bleed,扩散模型会"信以为真"再生成出来)。
|
||||
|
||||
而是给每个 retrieved frame 一个 **canonical coordinate map** $C_j \in [-1,1]^{3\times H\times W}$,3 个通道:
|
||||
- $(u, v)$ = 该像素在源帧的归一化位置
|
||||
- $\frac{2j}{N_s} - 1$ = 帧索引编码
|
||||
|
||||
然后**前向 warp** 到目标视角:
|
||||
|
||||
$$\hat{C}_j = \mathrm{FwdWarp}(C_j, D_j^s, T_j^s, T^*, K_j^s, K^*)$$
|
||||
|
||||
并把 warped depth 作为第 4 通道,得到 **4-channel correspondence map**。这个 4 通道图经 sin/cos 位置编码 + MLP → 添加到每个 transformer block 的 self-attention **Q 和 K**(不动 V)。
|
||||
|
||||
> **关键洞察**:**几何对应只参与"哪些 token 应该 attend 哪些 token"的决定,不参与"内容是什么"的合成**——把"路由"与"合成"彻底解耦。
|
||||
|
||||
---
|
||||
|
||||
### 3.3 模块 2:Anti-Drifting(抗时间漂移)— § 4.3
|
||||
|
||||
#### (a) 漂移的根源:**Train-Test Discrepancy(训练-推理偏置)**
|
||||
|
||||
- **训练时**:模型看见的历史 = 干净的 ground-truth 帧
|
||||
- **推理时**:模型看见的历史 = **它自己刚生成的、带瑕疵的帧**
|
||||
|
||||
→ 每步的微小误差被认为是"训练分布外的事故",模型不会修正,反而**继续传播**。
|
||||
|
||||
#### (b) Self-Augmentation Training — 用"自己生成的"代替"ground-truth"
|
||||
|
||||
每次自回归训练步,以概率 $p_{\text{aug}} = 0.7$:
|
||||
|
||||
1. 用干净 GT 编码历史隐变量 $z_0^{\text{hist}} = \mathcal{E}(x^{\text{hist}})$
|
||||
2. 抽 $t \sim \mathcal{U}(0, 0.5)$,按 flow-matching schedule 加噪:
|
||||
|
||||
$$z_t^{\text{hist}} = (1-t) z_0^{\text{hist}} + t \epsilon$$
|
||||
|
||||
3. 让 DiT 做**一步**去噪,得到带误差的"伪自生成历史":
|
||||
|
||||
$$\tilde z_0^{\text{hist}} = z_t^{\text{hist}} - t \cdot v_\theta(z_t^{\text{hist}}, t, c)$$
|
||||
|
||||
4. 用 $\tilde z_0^{\text{hist}}$ **替换** $z_0^{\text{hist}}$ 作为条件;但 supervision target $z_0^{\text{cur}}$ **仍用干净 GT 帧编码**
|
||||
|
||||
5. flow-matching 损失监督:"**给定带瑕疵的历史,你也要去噪到干净的当前**"
|
||||
|
||||
> **关键效果**:开销仅 **一次额外 DiT forward**,远比 Self-Forcing(每步全多步去噪)轻,**专为 bi-directional 模型设计**。
|
||||
|
||||
#### (c) FramePack 做温和支撑
|
||||
|
||||
FramePack 提供"近密远疏"的时间压缩(锚 + f₁k₁ + f₂k₂ + f₁k₁ + f₁₆k₄),把长历史压进固定 token 预算。但 Lyra 2.0 强调:**FramePack 缓解但不解决 train-test 偏置,真正解决要靠 self-augmentation**。
|
||||
|
||||
---
|
||||
|
||||
### 3.4 模块 3:前馈 3D Gaussian Splatting — § 4.4
|
||||
|
||||
视频拿到后,用 **Depth Anything v3 (DAv3)** 一次性预测每像素的 3DGS 属性,但做了两个修改:
|
||||
|
||||
1. **DPT 头 k=2 下采样**:每像素一个 Gaussian 会爆,降 4× 数量得到流式可渲染的体量
|
||||
2. **在生成数据上微调 DAv3**:用 3,000 段一分钟视频(来自 DL3DV)做 10,000 iter / lr=5e-5 / bs=8 微调,让 DAv3 学会容忍生成视频特有的小不一致
|
||||
|
||||
mesh 抽取:**OpenVDB 分层稀疏栅格** + 视角近用细格、远景用粗格;由 Gaussian 中值深度算 SDF,marching cubes 出 mesh 后跨层级拼接 + 简化。
|
||||
|
||||
---
|
||||
|
||||
### 3.5 加速版 — DMD 蒸馏 — § 4.5
|
||||
|
||||
- 用 **Distribution Matching Distillation** 把 teacher (35 steps + CFG) 蒸成 student (**4 steps, no CFG**)
|
||||
- 蒸馏期间**保留 self-augmentation**,确保 student 也抗漂移
|
||||
- 速度 **13×** 加速,质量(LPIPS/FID)几乎不掉
|
||||
|
||||
---
|
||||
|
||||
## 4. 实验结果
|
||||
|
||||
### 4.1 长视频生成对比(§ 5.2, DL3DV + Tanks-and-Temples)
|
||||
|
||||
7 个 baseline:**Yume-1.5、GEN3C、CaM、VMem、SPMem、HY-WorldPlay、GenWarp**。
|
||||
|
||||
| 指标 | 含义 | Ours | 第二好 |
|
||||
|---|---|---:|---|
|
||||
| **SSIM ↑** | 局部结构相似 | **0.388 / 0.384** | SPMem 0.383 / 0.383 |
|
||||
| **LPIPS ↓** | 感知距离 | **0.498 / 0.552** | SPMem 0.522 / 0.571 |
|
||||
| **FID ↓** | Fréchet 距离 | **43.43 / 51.33** | CaM 50.43 / 59.20 |
|
||||
| **Subjective Quality ↑** | WorldScore 人评 | **44.54 / 43.35** | SPMem 38.32 / 34.41 |
|
||||
| **Style Consistency ↑** | 首帧 vs 末帧风格 | **87.46 / 85.07** | SPMem 82.79 / 79.68 |
|
||||
| **Camera Ctrl ↑** | 相机姿态准确 | 64.67 / 63.87 | GEN3C 69.54 / 70.91 |
|
||||
| **Reprojection Err ↓** | SLAM 验证 3D 一致 | 0.076 / 0.069 | VMem 0.068 / 0.054 |
|
||||
|
||||
> 解读:**所有"感知与一致性"指标都最好**(SSIM/LPIPS/FID/Subj/Style)。Camera Ctrl 略输 GEN3C(GEN3C 用刚性 depth warping,精度高但**严重损害生成质量**)。
|
||||
|
||||
### 4.2 3D 场景生成对比(§ 5.3)
|
||||
|
||||
把所有视频 baseline 都过 **DAv3** 出 3DGS,再渲染评测:
|
||||
|
||||
| 方法 | DL3DV LPIPS-G ↓ | DL3DV FID ↓ | T&T LPIPS-G ↓ | T&T Subj ↑ |
|
||||
|---|---:|---:|---:|---:|
|
||||
| GEN3C + DAv3 | 0.649 | 99.83 | 0.694 | 5.38 |
|
||||
| CaM + DAv3 | 0.668 | 94.04 | 0.693 | 9.79 |
|
||||
| SPMem + DAv3 | 0.625 | 93.56 | 0.666 | 9.95 |
|
||||
| Ours + DAv3 | 0.603 | 74.39 | 0.648 | 14.42 |
|
||||
| **Ours Full(微调 DAv3)** | **0.579** | **65.94** | **0.629** | **18.80** |
|
||||
|
||||
> **Ours Full > Ours + DAv3** 验证了"在生成数据上微调 DAv3"的必要性。
|
||||
|
||||
### 4.3 消融(§ 5.4)— 每个模块都掉一组分
|
||||
|
||||
| 配置 | SSIM | Style ↑ | Camera ↑ | Reproj ↓ |
|
||||
|---|---:|---:|---:|---:|
|
||||
| **Ours full** | **0.384** | **85.07** | **63.87** | **0.069** |
|
||||
| w/ Global Point Cloud(融成单一全局) | 0.368 | 82.42 | 49.86 | 0.067 |
|
||||
| w/ Explicit Corr. Fusion(硬几何融合) | 0.370 | 83.28 | 57.29 | 0.071 |
|
||||
| w/o FramePack(无时间压缩) | 0.362 | 80.61 | 62.62 | 0.079 |
|
||||
| w/o Self-Augmentation | 0.363 | **77.98** | **53.92** | 0.066 |
|
||||
|
||||
> **去掉 Self-Augmentation** → Style Consistency 暴跌 7 分,Camera Ctrl 暴跌 10 分:**这是论文最关键的单一创新**。
|
||||
> **融成全局点云** → Camera Ctrl 暴跌 14 分,证实"per-frame 独立"的设计判断正确。
|
||||
|
||||
---
|
||||
|
||||
## 5. 应用(§ 5.5)
|
||||
|
||||
| 应用 | 价值 |
|
||||
|---|---|
|
||||
| **交互式 GUI** | 用户在 3D cache 点云上画相机轨迹,实时生成新视角并扩张场景 |
|
||||
| **野外图像** | 室内/室外/街景任意输入图,都能扩张为大尺度可走的 3D |
|
||||
| **Embodied AI 仿真** | 导出的 3DGS + mesh **直接进 NVIDIA Isaac Sim**,做物理仿真和机器人训练 |
|
||||
|
||||
---
|
||||
|
||||
## 6. 关键实现细节(附录 A)
|
||||
|
||||
| 项 | 配置 |
|
||||
|---|---|
|
||||
| 基模型 | Wan 2.1-14B DiT |
|
||||
| VAE | Wan 2.1,8× 空间 / 4× 时间下采样,C=16 |
|
||||
| 分辨率 | 832 × 480 |
|
||||
| 训练数据 | DL3DV (10K clips) — 用 ViPE 估姿态,DAv3 估深度,Qwen3-VL-8B 写 caption |
|
||||
| Optimizer | AdamW, lr=3e-5, wd=0.1, bf16 |
|
||||
| 训练规模 | 64 GB200 GPUs, batch=64, **7,000 iter** |
|
||||
| Flow Matching | rectified flow + logit-normal t 采样 |
|
||||
| Inference | FlowUniPC 多步,35 steps + CFG=5.0 |
|
||||
| 空间记忆参数 | $N_s = 5$, $d = 8$, $\delta = 0.1$ |
|
||||
| 自增强 | $p_{\text{aug}} = 0.7$, $t \sim \mathcal{U}(0, 0.5)$ |
|
||||
| 单步时间(80 帧/步) | **194 s** 全模型 / **15 s** DMD 蒸馏版 |
|
||||
| 3DGS 头下采样 | $k = 2$,Gaussian 数减少 4× |
|
||||
| 微调 DAv3 | 3,000 段 1 分钟视频,10K iter,lr=5e-5,bs=8 |
|
||||
|
||||
---
|
||||
|
||||
## 7. 局限性(论文承认)
|
||||
|
||||
1. **静态场景** — 不建模动态(人、车、风吹窗帘)。dynamic scene 是明确的 future work
|
||||
2. **照度不一致** — DL3DV 训练数据帧间曝光不稳,模型继承这一缺陷,会污染 3DGS。建议:用 PPISP 这类网络做照度补偿,或换合成游戏引擎数据
|
||||
|
||||
---
|
||||
|
||||
## 8. 与 PRISM 的关系(本仓库视角)
|
||||
|
||||
### 8.1 共同点
|
||||
|
||||
- 都关心"长时空间记忆"
|
||||
- 都把 3D 表征做成"信息源"而非最终目标(PRISM 的 L2 / Lyra 的 3D cache 都是中间件)
|
||||
- 都用 3DGS 作为最终稠密表征
|
||||
|
||||
### 8.2 根本差异
|
||||
|
||||
| 维度 | PRISM | Lyra 2.0 |
|
||||
|---|---|---|
|
||||
| 输入 | **真实传感器** (iPhone + ZED) | **单张图像**(可虚构) |
|
||||
| 输出 | 机器人可查询的 SpatialMemory + 4 层场景图 | 可被 Isaac Sim 加载的 3DGS + mesh |
|
||||
| 几何源 | RoomPlan / ZED 双目 SLAM 真实测量 | DAv3 单目深度估计 |
|
||||
| 关注问题 | **lifelong** 一致性、差异检测、巩固 | **生成期**的空间/时间漂移 |
|
||||
| 是否生成 | ❌ 只重建/记忆 | ✅ 生成(从扩散先验) |
|
||||
| 适用场景 | 真实部署的机器人 | 仿真训练 / VR / 资产生成 |
|
||||
|
||||
### 8.3 互补合作机会
|
||||
|
||||
| Lyra 2.0 → PRISM | 用 Lyra 生成的 3DGS 场景**给 PRISM 当训练/仿真环境**;不需要扫真实酒店即可造出 100 间不同布局的酒店房间用于 PRISM 评测 |
|
||||
|---|---|
|
||||
| **PRISM → Lyra 2.0** | PRISM 提供"真实 RoomPlan 几何先验",作为 Lyra 起始条件,可能减轻生成式 3D 缺乏物理一致性的问题 |
|
||||
| **联合** | "真实采集 + 生成式扩展"=PRISM 给出真实房间核心 +Lyra 在房间之外补全走廊、电梯、相邻房间,形成跨房间的整楼层 spatial memory |
|
||||
|
||||
### 8.4 对 PRISM 的具体借鉴
|
||||
|
||||
1. **"几何只用做路由,不用做合成"** — PRISM 的 L2 ↔ L3 ↔ L4 写入逻辑可借鉴:**L2 几何精度低没关系,只要能挑出"哪个 L3 节点该被更新"即可,真正的 L3 内容来自 L1 高质量原始观测**
|
||||
2. **Per-frame 独立 vs 全局融合** — PRISM 的 TSDF / OctoMap 是融合的,**易累积漂移**。可考虑保留 keyframe-级独立深度作为"翻案证据"
|
||||
3. **Self-Augmentation 思路** — 在 PRISM 的"巩固期"(Pipeline D)可借鉴:**让模型用自己之前的不完美 L3 写回,学会自我纠错**
|
||||
|
||||
---
|
||||
|
||||
## 9. 评分(Lyra 2.0 自己,非 PRISM)
|
||||
|
||||
| 维度 | 满分 | Lyra 2.0 | 注 |
|
||||
|---|---:|---:|---|
|
||||
| 算法原创性 | 20 | **17** | "几何路由 + canonical coord + self-aug"是清晰的新组合 |
|
||||
| 工程完整度 | 15 | **15** | GUI + DMD + Isaac Sim 全栈 |
|
||||
| 真机/真用验证 | 15 | **9** | 演示+对比丰富,但用户研究和落地数据少 |
|
||||
| 评测严谨度 | 15 | **14** | 7 个 baseline + 4 套指标 + 严格消融 |
|
||||
| 理论深度 | 10 | **5** | Flow matching + flow-warping 是工程级数学,没新定理 |
|
||||
| 影响力潜力 | 15 | **12** | NVIDIA 出品 + 接 Isaac Sim,有从研究到产品的明确路径 |
|
||||
| 计算成本 | 10 | **4** | 64 张 GB200 训练 + 推理 194 s / 段全模型 (DMD 后 15 s),家用消费级不可行 |
|
||||
| **合计** | **100** | **76** | 高质量基础研究,**显著高于 PRISM 的 62 分** |
|
||||
|
||||
---
|
||||
|
||||
## 10. 评分坐标:把 Lyra 2.0 与 PRISM 同档比较一次
|
||||
|
||||
| 项目 | 技术难度评分 | 行业贡献评分 | 主要差异 |
|
||||
|---|---:|---:|---|
|
||||
| **PRISM** (蓝图) | 62 | 48 | 工程模板 + 多模块集成,无真机/无论文/无算法新点 |
|
||||
| **Lyra 2.0** (NVIDIA 2026) | **76** | **65** (预估) | 有算法新点 + 顶会 + NVIDIA 背书,但**非真实数据生成** |
|
||||
| OK-Robot (FAIR 2024) | 72 | 70 | 真机大规模部署是杀手锏 |
|
||||
| ConceptGraphs (CMU 2023) | 67 | 52 | 真机 + ICRA,但无 lifelong |
|
||||
| HOV-SG (Freiburg 2024) | 64 | 50 | RSS + 分层场景图 |
|
||||
| Clio (MIT-SPARK 2024) | 70 | 55 | 理论 + 工具链组合 |
|
||||
|
||||
> Lyra 2.0 在**视频生成 / 内容创建**赛道处于**当前 SOTA**,这与 PRISM 的"真实空间记忆"赛道**正交但互补**。
|
||||
|
||||
---
|
||||
|
||||
## 11. 一句话评价
|
||||
|
||||
> **"Lyra 2.0 把'生成式 3D'从短片段 demo 推进到'任意长度可走通的世界',核心武器是'几何只用作路由 + 自增强训练'这一对工程哲学。它会成为 2026 年 explorable world generation 这条赛道的事实基线之一。"**
|
||||
|
||||
---
|
||||
|
||||
## 12. 配套资源
|
||||
|
||||
- **PDF 原文**:[`research/lyra2_paper.pdf`](lyra2_paper.pdf) — 13 MB
|
||||
- **TXT 提取**:[`research/lyra2_paper.txt`](lyra2_paper.txt) — pdftotext -layout 输出,1413 行
|
||||
- **项目主页**:<https://research.nvidia.com/labs/sil/lyra2/>
|
||||
- **arXiv**:<https://arxiv.org/abs/2604.13036>
|
||||
- **相关项目**:Lyra 1 (ICLR 2026), Wan 2.1, Depth Anything v3, FramePack, NVIDIA Isaac Sim
|
||||
|
||||
---
|
||||
|
||||
**译读版本**:v1.0
|
||||
**译读日期**:2026-05-16
|
||||
**译读者**:Code Assistant (基于 PDF 全文 + 公开论文知识)
|
||||
**配套文件**:
|
||||
- [`plans/PRISM/comparison.md`](../plans/PRISM/comparison.md) — 同档项目对比综述
|
||||
- [`plans/PRISM/rate.md`](../plans/PRISM/rate.md) — PRISM 技术难度评分
|
||||
- [`plans/PRISM/rate_industry.md`](../plans/PRISM/rate_industry.md) — PRISM 行业贡献评分
|
||||
@@ -0,0 +1,83 @@
|
||||
# 面向“理解”与“构建”物理世界的AI世界模型：技术综述
|
||||
|
||||
## 摘要
|
||||
随着人工智能技术的演进，AI的目标已从文本和二维图像的生成，扩展到了对三维物理规律的掌握。世界模型（World Models）在此过程中扮演了核心角色。本综述基于arXiv上的最新学术论文及GitHub上的前沿开源项目，聚焦于“理解物理世界”与“构建物理世界”两个核心维度，系统梳理国际顶尖研究与中国力量在这一领域的突破与挑战。
|
||||
|
||||
---
|
||||
|
||||
## 1. 引言
|
||||
世界模型旨在赋予AI系统对环境动态变化的预测能力。当我们将其置于“物理世界”的语境下，世界模型的研究可以被划分为两大核心命题：
|
||||
1. **理解物理世界 (Understanding)**：偏向于认知科学与强化学习，关注AI能否抽取出环境中的因果关系、物体持久性、碰撞规律及动力学方程，而不被表象的像素噪音所干扰。
|
||||
2. **构建物理世界 (Constructing)**：偏向于生成式AI与计算机视觉，关注AI能否作为一种“神经物理引擎”，根据条件（文本、动作、图像）合成出符合物理规律的、高保真的连续时空序列（视频或3D环境）。
|
||||
|
||||
---
|
||||
|
||||
## 2. 理解物理世界：从表象到因果
|
||||
|
||||
理解物理世界要求模型具备对时间动态、空间几何和物理规则的内部表征能力。
|
||||
|
||||
### 2.1 隐空间预测与联合嵌入架构 (JEPA)
|
||||
Yann LeCun 提出的 **JEPA（Joint Embedding Predictive Architecture）** 是当前“理解物理世界”最具代表性的路线。
|
||||
* **核心思想**：真实世界包含海量不可预测的细节（如树叶的随机摆动）。JEPA（如 **V-JEPA**）放弃了在像素层面的重建，转而在抽象的隐空间中预测视频的未来状态或缺失部分。
|
||||
* **物理意义**：这种设计强迫模型学习高级语义和宏观物理规律（如重力、物体运动轨迹），是AI走向常识理解的关键。
|
||||
* **开源生态**：Meta FAIR 在 GitHub 上的 `facebookresearch/jepa` 项目是目前该领域的基石。
|
||||
|
||||
### 2.2 具身智能与基于动作的物理理解
|
||||
在强化学习和机器人领域，理解物理世界的最佳方式是“交互”。
|
||||
* **Dreamer 系列 (DreamerV3)**：Danijar Hafner 团队在 arXiv 上发表的多篇论文确立了基于模型的强化学习（MBRL）范式。Dreamer在内部构建一个“想象的世界模型”，在这个世界中推演动作导致的物理后果，从而以极高的样本效率学习复杂的物理控制。
|
||||
* **自动驾驶的物理认知 (Wayve LINGO-1)**：Wayve 等公司不仅让模型生成视频，还让模型用语言解释为什么某辆车会停下、前方的物理障碍是什么，实现了对物理场景的可解释性理解。
|
||||
|
||||
---
|
||||
|
||||
## 3. 构建物理世界：神经物理引擎的崛起
|
||||
|
||||
构建物理世界通常通过大规模视频生成（Video Generation）或3D场景生成来实现。
|
||||
|
||||
### 3.1 扩散与Transformer的结合 (DiT)
|
||||
OpenAI 的 **Sora** 证明了，当扩散模型结合Transformer（DiT）并扩大规模时，模型能够“涌现”出令人惊叹的构建物理世界的能力。
|
||||
* **涌现的物理属性**：保持3D一致性（相机移动时物体结构不变）、物体持久性（被遮挡后再次出现）以及简单的流体和固体交互。
|
||||
* **技术实质**：Sora 本质上是通过海量数据“拟合”了视觉上的物理规律（Visual Physics），它在构建视觉逼真的世界方面取得了空前成功。
|
||||
|
||||
### 3.2 中国力量在物理世界构建中的突破
|
||||
中国科研机构与企业在“构建物理世界”方向（即对标Sora）上表现出了极强的爆发力，并在解决物理一致性上提出了独特的中国方案：
|
||||
* **可灵 (Kling) 的 3D VAE 架构**：快手团队通过自研的3D时空联合注意力机制，更好地处理了时间轴上的物理连续性，其生成的物理互（如吃面条、流体倾倒）在视觉上表现出了高度的真实感。
|
||||
* **Vidu 与 U-ViT**：生数科技联合清华大学基于U-ViT架构构建的世界模型，在单次生成长视频及多镜头语言的物理连贯性上具有独特优势。
|
||||
* **CogVideoX 的 3D Causal VAE**：智谱AI在arXiv上开源的CogVideoX，通过因果卷积和专家并行DiT设计，大幅降低了构建三维连续物理世界的算力门槛。
|
||||
|
||||
### 3.3 GitHub 开源复现与生态
|
||||
中国开源社区极大推动了构建物理世界技术的普及：
|
||||
* **Open-Sora** (`hpcaitech/Open-Sora`)：潞晨科技提供了全面的DiT模型训练方案，专注于长时空一致性的突破。
|
||||
* **Open-Sora-Plan** (`PKU-YuanGroup/Open-Sora-Plan`)：北京大学团队深度开源了从数据清洗到物理运动表征控制的全套代码，成为研究如何让模型更好地“构建物理规律”的核心阵地。
|
||||
|
||||
---
|
||||
|
||||
## 4. 核心对比：“理解”与“构建”的碰撞
|
||||
|
||||
在当前 arXiv 的研究趋势中，“理解”与“构建”存在显著的路线差异与融合趋势：
|
||||
|
||||
| 维度 | 理解物理世界 (如 JEPA, Dreamer) | 构建物理世界 (如 Sora, Kling, Open-Sora) |
|
||||
| :--- | :--- | :--- |
|
||||
| **主要目标** | 提取因果规律、特征表示、策略规划 | 生成高保真视觉内容、模拟视觉现象 |
|
||||
| **工空间** | 抽象的隐空间 (Latent Space) | 像素空间或浅层压缩空间 (Pixel / Low-dim Latent) |
|
||||
| **对噪音的容忍度** | 高（主动过滤细节噪音） | 低（必须生成所有细节，算力开销大） |
|
||||
| **物理规律真实性**| 侧重逻辑和动力学的正确性 | 侧重视觉的合理性（易产生物理幻觉） |
|
||||
| **主要应用** | 机器人、具身智能、自动驾驶决策 | 影视生成、多媒体创作、游戏引擎预渲染 |
|
||||
|
||||
---
|
||||
|
||||
## 5. 挑战与未来方向
|
||||
|
||||
根据 arXiv 上对 Sora 等模型的逆向工程和批评分析，当前领域存在以下严峻挑战：
|
||||
|
||||
1. **“视觉物理”与“真实物理”的鸿沟 (Hallucination of Physics)**
|
||||
当前的生成式世界模型（如Sora）经常会产生违背常理的物理幻觉（如椅子凭空变形、水倒流）。这是因为它们是纯数据驱动的模式匹配，尚未真正掌握牛顿力学或流体动力学方程。
|
||||
2. **结合物理引擎的混合世界模型**
|
||||
未来的发展趋势是结合传统物理引擎（如 Unreal Engine, Unity）与神经网络。通过引入物理先验（Physical Priors），强制生成过程服从质量守恒、动量守恒等硬性规则。
|
||||
3. **闭环系统：从构建到理解的统一**
|
||||
AI不仅需要生成（构建）一段视频，还需要从视频中推断（理解）背后的受情况，最终形成既能输出控制策略、又能输出视觉模拟的统一通用世界模型 (Universal World Model)。
|
||||
|
||||
---
|
||||
|
||||
## 6. 结论
|
||||
|
||||
AI 世界模型正处于从单纯的数据拟合向物理规律认知进阶的拐点。在**“理解物理世界”**方面，以LeCun的JEPA为代表的隐空间路线为AI提供了高效的认知框架；在**“构建物理世界”**方面，基于DiT架构的生成式模型（如Sora、Kling）展现了令人震撼的“视觉物理引擎”潜力。中国团队不仅在商业产品上紧跟前沿，更通过如 `Open-Sora` 等项目繁荣了GitHub开源生态。未来，将强化学习的因果推理能力与扩散模型的高维构建能力相融合，真正打通“理解”与“构建”的壁垒，将是实现具身智能和AGI的终极路径。
|
||||
@@ -0,0 +1,319 @@
|
||||
"""arXiv + GitHub 检索工具（支持 HTTP / SOCKS5 代理）
|
||||
|
||||
用法：
|
||||
# 1. 直连
|
||||
python3 research/search_info.py
|
||||
|
||||
# 2. HTTP 代理（推荐：6984 同时支持 http/socks5）
|
||||
HTTPS_PROXY=http://127.0.0.1:6984 HTTP_PROXY=http://127.0.0.1:6984 \
|
||||
python3 research/search_info.py
|
||||
|
||||
# 3. SOCKS5 代理（需 pip install pysocks）
|
||||
ALL_PROXY=socks5h://127.0.0.1:6984 python3 research/search_info.py
|
||||
|
||||
# 4. 命令行指定（覆盖环境变量）
|
||||
python3 research/search_info.py --proxy http://127.0.0.1:6984
|
||||
|
||||
输出：
|
||||
research/search_results.json — 结构化 JSON（按主题分组）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 主题化查询（与 plans/camera/zed2i_iterative_framework.md 五条主线对齐）
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
TOPICS: dict[str, str] = {
|
||||
"stereo_matching": 'all:"stereo matching" AND (all:"foundation" OR all:"zero-shot" OR all:"transformer")',
|
||||
"visual_inertial_slam": 'all:("visual inertial" OR "VIO" OR "VI-SLAM") AND cat:cs.CV',
|
||||
"gaussian_splatting_slam": 'all:"gaussian splatting" AND all:"SLAM"',
|
||||
"monocular_depth_foundation": 'all:("depth anything" OR "metric3d" OR "marigold" OR "UniDepth")',
|
||||
"indoor_rgbd_dataset": 'all:("RGB-D dataset" OR "indoor dataset") AND all:"reconstruction"',
|
||||
"world_model_video": 'all:"world model" AND (all:"video" OR all:"prediction" OR all:"generative")',
|
||||
"zed_camera": 'all:("ZED 2i" OR "ZED stereo" OR "ZED-2i" OR "Stereolabs")',
|
||||
"orbbec_gemini": 'all:("Orbbec" OR "Femto Mega" OR "Femto Bolt" OR "Azure Kinect") AND all:("depth" OR "RGB-D" OR "SLAM")',
|
||||
"rgbd_indoor_reconstruction": 'all:("RGB-D" OR "RGBD") AND all:("indoor" OR "room") AND (all:"reconstruction" OR all:"SLAM")',
|
||||
"neural_stereo_depth": 'all:("foundation stereo" OR "RAFT-Stereo" OR "IGEV") AND all:"depth"',
|
||||
}
|
||||
|
||||
# GitHub 主题化查询
|
||||
GITHUB_TOPICS: dict[str, str] = {
|
||||
"stereo_matching": "stereo matching depth in:name,description,readme stars:>500",
|
||||
"vio_slam": "visual inertial SLAM in:name,description,readme stars:>500",
|
||||
"gaussian_splatting": "gaussian splatting in:name,description,readme stars:>1000",
|
||||
"monocular_depth": "monocular depth estimation in:name,description,readme stars:>1000",
|
||||
"world_model": "world model in:name,description,readme stars:>500",
|
||||
}
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 代理配置
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def build_opener(proxy: str | None) -> urllib.request.OpenerDirector:
|
||||
"""构建带代理的 urllib opener。
|
||||
|
||||
proxy 形式：
|
||||
- http://127.0.0.1:6984
|
||||
- https://127.0.0.1:6984
|
||||
- socks5h://127.0.0.1:6984 （需 pysocks）
|
||||
- None → 走环境变量 (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY) 或直连
|
||||
"""
|
||||
if proxy and proxy.startswith(("socks5://", "socks5h://", "socks4://")):
|
||||
try:
|
||||
import socks # type: ignore
|
||||
import socket
|
||||
|
||||
scheme, rest = proxy.split("://", 1)
|
||||
host, port_str = rest.split(":")
|
||||
port = int(port_str)
|
||||
socks_type = {
|
||||
"socks5": socks.SOCKS5,
|
||||
"socks5h": socks.SOCKS5,
|
||||
"socks4": socks.SOCKS4,
|
||||
}[scheme]
|
||||
socks.set_default_proxy(socks_type, host, port, rdns=(scheme == "socks5h"))
|
||||
socket.socket = socks.socksocket # type: ignore[assignment]
|
||||
print(f"[proxy] SOCKS5 启用: {proxy}", file=sys.stderr)
|
||||
return urllib.request.build_opener()
|
||||
except ImportError:
|
||||
print(
|
||||
"[proxy] 警告: 需要 SOCKS5 但未安装 pysocks，将回退到环境变量。\n"
|
||||
" 修复: pip install pysocks",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return urllib.request.build_opener()
|
||||
|
||||
if proxy:
|
||||
handler = urllib.request.ProxyHandler({"http": proxy, "https": proxy})
|
||||
print(f"[proxy] HTTP(S) 代理启用: {proxy}", file=sys.stderr)
|
||||
return urllib.request.build_opener(handler)
|
||||
|
||||
# 走环境变量
|
||||
env_proxy = os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY") or os.environ.get("ALL_PROXY")
|
||||
if env_proxy:
|
||||
print(f"[proxy] 使用环境变量代理: {env_proxy}", file=sys.stderr)
|
||||
else:
|
||||
print("[proxy] 直连模式（无代理）", file=sys.stderr)
|
||||
return urllib.request.build_opener()
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# arXiv
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def search_arxiv(
|
||||
opener: urllib.request.OpenerDirector,
|
||||
query: str,
|
||||
max_results: int = 20,
|
||||
sort_by: str = "submittedDate",
|
||||
sort_order: str = "descending",
|
||||
) -> list[dict[str, Any]]:
|
||||
"""调用 arXiv API 并解析返回的 ATOM XML。"""
|
||||
base = "https://export.arxiv.org/api/query"
|
||||
params = urllib.parse.urlencode(
|
||||
{
|
||||
"search_query": query,
|
||||
"start": 0,
|
||||
"max_results": max_results,
|
||||
"sortBy": sort_by,
|
||||
"sortOrder": sort_order,
|
||||
}
|
||||
)
|
||||
url = f"{base}?{params}"
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (research-bot)"})
|
||||
|
||||
# arXiv 限速重试（429 → 指数退避）
|
||||
data = ""
|
||||
for attempt in range(4):
|
||||
try:
|
||||
with opener.open(req, timeout=30) as resp:
|
||||
data = resp.read().decode("utf-8")
|
||||
break
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
wait = 10 * (attempt + 1)
|
||||
print(
|
||||
f"[arxiv][{query[:40]}...] 429 限速，等待 {wait}s 重试 ({attempt+1}/4)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
time.sleep(wait)
|
||||
continue
|
||||
print(f"[arxiv][{query[:40]}...] HTTPError {e.code}: {e.reason}", file=sys.stderr)
|
||||
return []
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
print(f"[arxiv][{query[:40]}...] ERROR: {e}", file=sys.stderr)
|
||||
return []
|
||||
else:
|
||||
print(f"[arxiv][{query[:40]}...] 持续 429，放弃", file=sys.stderr)
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(data)
|
||||
except ET.ParseError as e:
|
||||
print(f"[arxiv][{query[:40]}...] XML parse error: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
ns = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
|
||||
papers: list[dict[str, Any]] = []
|
||||
|
||||
for entry in root.findall("atom:entry", ns):
|
||||
|
||||
def _text(elem_path: str) -> str:
|
||||
elem = entry.find(elem_path, ns)
|
||||
return (elem.text or "").replace("\n", " ").strip() if elem is not None else ""
|
||||
|
||||
arxiv_id_url = _text("atom:id")
|
||||
arxiv_id = arxiv_id_url.rsplit("/", 1)[-1] if arxiv_id_url else ""
|
||||
# 去掉版本号 e.g. 2501.09898v2 → 2501.09898
|
||||
arxiv_id_clean = arxiv_id.split("v")[0] if arxiv_id else ""
|
||||
|
||||
authors: list[str] = []
|
||||
for a in entry.findall("atom:author", ns):
|
||||
name_elem = a.find("atom:name", ns)
|
||||
if name_elem is not None and name_elem.text:
|
||||
authors.append(name_elem.text.strip())
|
||||
|
||||
categories = [
|
||||
c.attrib.get("term", "") for c in entry.findall("atom:category", ns)
|
||||
]
|
||||
|
||||
summary = _text("atom:summary")
|
||||
papers.append(
|
||||
{
|
||||
"arxiv_id": arxiv_id_clean,
|
||||
"url": f"https://arxiv.org/abs/{arxiv_id_clean}" if arxiv_id_clean else arxiv_id_url,
|
||||
"title": _text("atom:title"),
|
||||
"published": _text("atom:published"),
|
||||
"updated": _text("atom:updated"),
|
||||
"authors": authors,
|
||||
"categories": categories,
|
||||
"summary": summary[:400] + ("..." if len(summary) > 400 else ""),
|
||||
}
|
||||
)
|
||||
|
||||
return papers
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# GitHub
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def search_github(
|
||||
opener: urllib.request.OpenerDirector,
|
||||
query: str,
|
||||
per_page: int = 10,
|
||||
) -> list[dict[str, Any]]:
|
||||
base = "https://api.github.com/search/repositories"
|
||||
params = urllib.parse.urlencode({"q": query, "sort": "stars", "order": "desc", "per_page": per_page})
|
||||
url = f"{base}?{params}"
|
||||
|
||||
headers = {"User-Agent": "Mozilla/5.0 (research-bot)", "Accept": "application/vnd.github+json"}
|
||||
token = os.environ.get("GITHUB_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
|
||||
try:
|
||||
with opener.open(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
# 60 req/hr 限速时 GitHub 返回 403
|
||||
print(f"[github][{query[:40]}...] HTTPError {e.code}: {e.reason}", file=sys.stderr)
|
||||
return []
|
||||
except (urllib.error.URLError, TimeoutError) as e:
|
||||
print(f"[github][{query[:40]}...] ERROR: {e}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
repos: list[dict[str, Any]] = []
|
||||
for item in data.get("items", []):
|
||||
repos.append(
|
||||
{
|
||||
"name": item.get("full_name", ""),
|
||||
"description": item.get("description", "") or "",
|
||||
"stars": item.get("stargazers_count", 0),
|
||||
"forks": item.get("forks_count", 0),
|
||||
"language": item.get("language", "") or "",
|
||||
"url": item.get("html_url", ""),
|
||||
"updated_at": item.get("updated_at", ""),
|
||||
}
|
||||
)
|
||||
return repos
|
||||
|
||||
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
# 主入口
|
||||
# ──────────────────────────────────────────────────────────────────────
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="arXiv + GitHub 主题化检索")
|
||||
parser.add_argument("--proxy", default=None, help="代理 URL，如 http://127.0.0.1:6984")
|
||||
parser.add_argument("--max-results", type=int, default=15, help="每个主题最多论文数")
|
||||
parser.add_argument("--out", default="research/search_results.json", help="输出 JSON 路径")
|
||||
parser.add_argument(
|
||||
"--topics",
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="只跑指定主题（默认全部），如：--topics stereo_matching world_model_video",
|
||||
)
|
||||
parser.add_argument("--no-github", action="store_true", help="跳过 GitHub 检索")
|
||||
parser.add_argument("--delay", type=float, default=3.0, help="主题间延迟（秒），arXiv 建议 ≥3s")
|
||||
args = parser.parse_args()
|
||||
|
||||
opener = build_opener(args.proxy)
|
||||
|
||||
selected = args.topics or list(TOPICS.keys())
|
||||
results: dict[str, Any] = {
|
||||
"meta": {
|
||||
"generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
||||
"proxy": args.proxy or os.environ.get("HTTPS_PROXY") or "direct",
|
||||
"max_results_per_topic": args.max_results,
|
||||
},
|
||||
"arxiv": {},
|
||||
"github": {},
|
||||
}
|
||||
|
||||
# arXiv
|
||||
for topic in selected:
|
||||
if topic not in TOPICS:
|
||||
print(f"[skip] 未知主题: {topic}", file=sys.stderr)
|
||||
continue
|
||||
print(f"[arxiv] 检索主题: {topic}", file=sys.stderr)
|
||||
papers = search_arxiv(opener, TOPICS[topic], max_results=args.max_results)
|
||||
results["arxiv"][topic] = papers
|
||||
print(f" → {len(papers)} 篇", file=sys.stderr)
|
||||
time.sleep(args.delay) # arXiv API rate limit
|
||||
|
||||
# GitHub
|
||||
if not args.no_github:
|
||||
for topic, q in GITHUB_TOPICS.items():
|
||||
if args.topics and topic not in args.topics:
|
||||
continue
|
||||
print(f"[github] 检索主题: {topic}", file=sys.stderr)
|
||||
repos = search_github(opener, q, per_page=10)
|
||||
results["github"][topic] = repos
|
||||
print(f" → {len(repos)} 个仓库", file=sys.stderr)
|
||||
time.sleep(1.0)
|
||||
|
||||
out_path = args.out
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
json.dump(results, f, indent=2, ensure_ascii=False)
|
||||
|
||||
total_papers = sum(len(v) for v in results["arxiv"].values())
|
||||
total_repos = sum(len(v) for v in results["github"].values())
|
||||
print(f"\n✅ 完成：{total_papers} 篇 arXiv + {total_repos} 个 GitHub 仓库 → {out_path}", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,385 @@
|
||||
# 物理世界理解（Understanding the Physical World）方向科研计划书
|
||||
|
||||
## 版本说明与更新日志 (Version History & Changelog)
|
||||
|
||||
| 版本号 | 更新日期 | 更新内容摘要 |
|
||||
| :---: | :---: | :--- |
|
||||
| **v1.3** | 2026-05-16 | 新增第7章“酒店场景下的室内建模与物理验证落地实施方案”，专门针对酒店公共区域、客房以及卫生间（高反光区域）提出定制化的SLAM与建图路线；全文章节编号顺延。 |
|
||||
| **v1.2** | 2026-05-16 | 在第6章系统集成规划中补充了“视觉与 YOLO 技术框架的深度结合方案，增强动态目标检测与开放词表泛化能力。 |
|
||||
| **v1.1** | 2026-05-16 | 新增“室内复杂交互场景的系统集成与验证规划”章节，细化了基于Habitat 3.0/RoboCasa的具身验证、Gaussian-SLAM几何建图以及PartNet-Mobility/Ditto铰接物体操作先验；优化并顺延了全文后续章节编号。 |
|
||||
| **v1.0** | 2026-05-15 | 初始版本建立，提出基于M-JEPA与因果图预的物理世界模型架构，涵盖物理属性解耦、反事实预测、具身智能平台验证等核心研究计划。 |
|
||||
|
||||
---
|
||||
|
||||
## 1. 项目名称
|
||||
**面向动态物理环境的因果表征学习与预测型世界模型研究**
|
||||
*(Research on Causal Representation Learning and Predictive World Models for Dynamic Physical Environments)*
|
||||
|
||||
---
|
||||
|
||||
## 2. 研究背景与意义
|
||||
近年来，以Sora、Kling为代表的生成式世界模型在“构建物理世界”（视觉生成）上取得了巨大突破。然而，这些模型本质上是数据驱动的像素拟合引擎，缺乏对底层物理因果律（如质量、摩擦力、动量守恒）的真正“理解”。当面临反事实（Counterfactual）场景或需要长时多步决策时，纯生成式模型极易产生物理幻觉。
|
||||
相比之下，Yann LeCun提出的联合嵌入预测架构（JEPA）和Danijar Hafner的Dreamer系列证明了，在隐空间（Latent Space）进行抽象特征的预测，能够主动过滤像素级噪音，是实现“物理世界理解”的更优路径。
|
||||
本项目旨在深入探究如何让AI从纯视觉或多模态信号中自主抽取出符合经典物理定律的内部表征，并将其应用于具身智能（Embodied AI）系统的复杂交互之中。这对于推动AI从“视觉模拟器”走向“物理推理机”具有重大科学意义。
|
||||
|
||||
---
|
||||
|
||||
## 3. 核心科学问题
|
||||
1. **隐状态与物理量的映射机制**：如何使世界模型在隐空间学习到的抽象表征（Latent Representations）与现实中的真实物理变量（如质量、速度、受力）产生可解释的对齐？
|
||||
2. **时空因果关系的提取与反事实推理**：在没有人类显式标注物理公式的情况下，模型如何从动态视频或多感官交互数据中自监督地发现物理因果图（Causal Graphs），并预测“如果改变某一条件，物理后果会如何”？
|
||||
|
||||
---
|
||||
|
||||
## 4. 研究内容与子课题
|
||||
|
||||
### 子课题 1：基于多模态对齐的物理属性解耦与表征学习
|
||||
* **目标**：打破纯视觉的局限，结合视觉、触觉/力觉数据，提取物体的隐式物理属性。
|
||||
* **方法**：
|
||||
* 引入带有碰撞力和材质信息的物理仿真数据集（如Physion的扩展版或构建基于Isaac Sim的新数据集）。
|
||||
* 设计改进型多模态JEPA架构（M-JEPA），在目标函数中引入互信息最大化（Mutual Information Maximization），强制模型将视频的运动轨迹与物体的质量、摩擦力等隐变量进行解耦。
|
||||
|
||||
### 子课题 2：内嵌物理先验的因果世界模型（Causal World Model）设计
|
||||
* **目标**：在神经网络的预测模块中注入显式的物理先验，防止物理幻觉。
|
||||
* **方法**：
|
||||
* 在时序预测模块（如Transformer或状态空间模型Mamba）中，引入图神经网络（GNN）或结构因果模型（SCM）。
|
||||
* 训练模型不仅预测下一步的隐状态 $s_{t+1}$，还要输出动作 $a_t$ 与环境之间因果作用的有向无环图（DAG），确保预测过程严格遵循局部能量守恒与动力学约束。
|
||||
|
||||
### 子课题 3：具身智能平台上的零样本/少样本物理交互验证
|
||||
* **目标**：验证模型对物理世界“理解”的泛化能力。
|
||||
* **方法**：
|
||||
* 将预训练好的因果世界模型作为具身智能体（如机械臂或四足机器人）的“大脑”。
|
||||
* 设计涉及复杂物理推理的任务（如工具使用、流体容器搬运、多物体堆叠），评估系统在面对未见过的物理材质或重心结构时的Zero-shot适应能力和强化学习的样本效率。
|
||||
|
||||
---
|
||||
|
||||
## 5. 技术路线与实施方案
|
||||
1. **数据构建阶段**：利用Unreal Engine 5和NVIDIA Isaac Sim生成数十万段包含明确物理参数标注（但不喂给模型，仅供评估）的碰撞、形变、流体交互视频。
|
||||
2. **模型训练阶段**：
|
||||
* Baseline：复现 V-JEPA 与 DreamerV3。
|
||||
* 采用自监督对比学习（Contrastive Learning）和掩码预测（Masked Prediction）在隐空间进行特征预训练。
|
||||
* 引入反事实预测损失（Counterfactual Prediction Loss）：输入修改后的物理量表征，强制模型输出相应的未来轨迹隐状态。
|
||||
3. **测试评估体系（Benchmark）**：建立专门针对“物理理解度”的评估基准，包括物理常识违背检测（Violation of Expectation）、长时轨迹预测准确率（MSE on Latent Dynamics）和强化学习奖励收敛速度。
|
||||
|
||||
---
|
||||
|
||||
## 6. 室内复杂交互场景的系统集成与验证规划
|
||||
|
||||
针对日常物理环境中最为典型的室内场景，本项目将设计一套完整的从几何建图到物理因果推断的集成流水线，具体规划如下：
|
||||
|
||||
### 6.1 室内交互平台选型
|
||||
* **优先平台**：全面接入 **Habitat 3.0**、**RoboCasa** 与 **HSSD (Habitat Synthetic Scene Dataset)** 作为核心室内交互仿真器。
|
||||
* **Habitat 3.0 / HSSD**：提供超大规模、高保真的室内空间拓扑与丰富的静态几何布局，用于训练模型对房间结构的导航与全局空间理解。
|
||||
* **RoboCasa**：提供高度细化的日常活动（特别是厨房场景）交互环境，支持物体抓取、放置等精细化操作。
|
||||
|
||||
### 6.2 几何端在线建图与表示
|
||||
* **方法**：引入 **Gaussian-SLAM** 或 **MonoGS** 算法作为机器人的前端视觉知模块。
|
||||
* **作用**：在机器人移动与交互过程中，实时构建环境的 3D Gaussian Splatting (3DGS) 辐射场或神经隐式表面。这不仅为后续操作提供精确的“几何 1:1”数字底座，还可通过渲染生成多视角的新视角图像（Novel View Synthesis），作为 M-JEPA 世界模型强大的自监督预测目标。
|
||||
|
||||
### 6.3 铰接物体操作与先验注入
|
||||
* **痛点**：日常室内环境充满了门、抽屉、冰箱等具有运动学约束的“铰接物体”（Articulated Objects），纯刚体物理无法涵盖这些交互。
|
||||
* **技术路线**：
|
||||
* 引入 **PartNet-Mobility** 数据库作为离线训练先验，让模型预先学习各类家具部件的关节轴方向（prismatic/revolute）及运动学限制。
|
||||
* 结合 **Ditto** 等部件级几何感知算法，在机器人面对未知的室内铰接物体时，实时估计其运动学关节结构。
|
||||
* **物理扩展**：通过将上述几何与运动学先验注入到因果世界模型中，使模型不仅能理解物体掉落、碰撞的“刚体物理”，更能准确预测“开门、拉抽屉、倒水”这类涉及复杂接触力与约束运动的物理反应，真正实现室内日常任务的闭环。
|
||||
|
||||
### 6.4 视觉与 YOLO 技术框架的深度结合方案
|
||||
* **痛点**：在动态交互过程中，场景中往往存在快速移动的物体，传统的全场景建图（如 3DGS）和隐空间特征提取对高频动态目标的精准锁定与语义关联能力存在不足。
|
||||
* **技术路线**：
|
||||
* **实时目标检测与实例分割**：在前端视觉流中引入 **YOLO (You Only Look Once)** 系列最新框架（如 YOLOv9 / YOLO-World）。利用其极高的推理帧率与轻量化优势，实时输出交互场景中关键物理对象（如杯、工具、门把手）的 2D 边界框（Bounding Box）和实例掩码。
|
||||
* **动静分离与物体级因果锚点**：将 YOLO 输出的 2D 语义与深度信息结合反投影至 3D 空间。一方面，为建图模块提供动态物体 Mask，避免运动对象污染静态场景的 3D 辐射场；另一方面，将提取出的“物体级（Object-centric）”视觉特征输入给 M-JEPA，作为推断质量、速度、受力等因果变量的结构化锚点，大幅降低无约束隐空间学习的难度。
|
||||
* **开放词表（Open-Vocabulary）支持**：结合 YOLO-World 等视觉-语言联合检测器，使系统具备通过自然语言指令即时定位未见过实体（Zero-shot Detection）的能力，进一步提升具身模型在泛化场景下的指令跟随与物理交互表现。
|
||||
|
||||
---
|
||||
|
||||
## 7. 酒店场景下的室内建模与物理验证落地实施方案
|
||||
|
||||
酒店环境作为具身智能落地的典型真实场景，兼具开阔公共空间与高度复杂的私密空间，对三维建模、SLAM建图及物理世界理解提出了严苛挑战。本项目将针对酒店场景设计分级建图与建模方案：
|
||||
|
||||
### 7.1 酒店公共区域（大堂、走廊）的大规模SLAM建图
|
||||
* **场景痛点**：空间跨度大、纹理单一（长走廊）、动态干扰多（行人、行李车频繁移动）。
|
||||
* **技术路线**：
|
||||
* **多模传感器融合**：采用激光雷达-视觉-惯性紧耦合SLAM（如 FAST-LIO2 或 R3LIVE），利用 LiDAR 提供远距离精确尺度与抗无纹理能力，结合视觉提供语义与色彩。
|
||||
* **动态滤除与大场景渲染**：结合前述 YOLO 框架实时滤除行人与动态物体，生成静态纯净的点云底座。随后在点云基础上训练大规模建筑级 3DGS（如 Hierarchical 3DGS），实现整个大堂和公共走廊的高保真、实时渲染。
|
||||
|
||||
### 7.2 客房内部的高保真稠密重建与语义理解
|
||||
* **场景痛点**：空间紧凑、家具密集、存在强烈的空间结构先验（如曼哈顿假设）。
|
||||
* **技术路线**：
|
||||
* **语义与几何联合建图**：在客房内使用 RGB-D 传感器，部署 Gaussian-SLAM 或 Co-SLAM 实现高精度稠密重建。结合 3D 场景图（3D Scene Graph）技术，将重建的几何网格抽象为“床-桌子-电视-衣柜”的层级语义节点。
|
||||
* **布局与 CAD 替身替换**：对于标准化的客房家具，利用算法提取 CAD 模型替身（如通过 SceneCAD 框架），提升仿真验证与物理交互（如机器人整理桌面、铺床）时的碰撞检测与动力学精确度。
|
||||
|
||||
### 7.3 卫生间（核心难点）的特殊建模与复杂物理推断
|
||||
* **场景痛点**：存在大量高反光表面（大面镜子、玻璃淋浴房、金属水龙头）和无纹理区域（白色浴缸、马桶），传统视觉SLAM与深度相机会在此严重失效（出现“重影”或深度穿透错误）。
|
||||
* **建图与建模路线**：
|
||||
* **抗高反光神经渲染**：引入专门处理镜面反射的神经辐射场或高斯技术（如 Ref-NeRF、Specular 3DGS），分离场景的漫反射成分（Diffuse）与依赖视角的镜面反射成分（Specular），从而精确恢复玻璃与镜子的真实表面几何，避免“穿墙”幻觉。
|
||||
* **多模态主动感知**：在数据采集端引入偏振相机（Polarization Camera）或利用特定波段的固态雷达（如 具有抗反射能力的 ToF 模组）辅助消除高光与反射干扰，获取卫生间真实的深度真值。
|
||||
* **物理交互与推断验证**：
|
||||
* 结合 M-JEPA 世界模型，在卫生间场景重点验证具有挑战性的物理推断。例如，结合 PartNet-Mobility 感知水龙头旋钮和淋浴房门的旋转/平移关节（Revolute/Prismatic Joints），并控制机械臂进行“拧开水龙头”、“推开玻璃门”、“拾取光滑洗漱用品”等需要高精度物理预测与触觉反馈的操作验证。
|
||||
|
||||
---
|
||||
|
||||
## 8. 预期成果
|
||||
1. **学术论文**：在国际顶级AI会议（NeurIPS, ICLR, ICML）发表2-3篇质量学术论文。
|
||||
2. **开源资源**：在GitHub上开源新型“因果物理世界模型”（Causal-Physics-JEPA）的代码库、预训练权重，以及大规模物理交互评测数据集。
|
||||
3. **应用原型**：构建一套基于该世界模型的具身智能控制原型系统，在至少两项复杂物理交互任务上超越现有强化学习基线（Baseline）的样本效率。
|
||||
|
||||
---
|
||||
|
||||
## 9. 研究时间规划（周期：12个月）
|
||||
* **第 1 - 3 个月（准备与基线构建）**：
|
||||
* 深入调研arXiv关于隐空间预测与因果推断的最新进展。
|
||||
* 完成仿真数据集搭建，复现JEPA及Dreamer系列基线模型。
|
||||
* **第 4 - 7 个月（算法攻坚与模型开发）**：
|
||||
* 完成 M-JEPA 及因果图结构预测模块的编码与调试。
|
||||
* 在超算平台上完成第一阶段的大规模自监督预训练。
|
||||
* **第 8 - 10 个月（物理实验与具身验证）**：
|
||||
* 在机械臂仿真环境及真机上部署模型，进行复杂物理交互任务的策略训练与微调。
|
||||
* 调整因果模块参数，解决长序列累积误差问题。
|
||||
* **第 11 - 12 个月（成果总结与开源）**：
|
||||
* 完成实验数据的统计分析，撰写并投递高水平学术论文。
|
||||
* 整理代码和数据集，在GitHub/HuggingFace上发布开源库。
|
||||
|
||||
---
|
||||
|
||||
## 10. arXiv和GitHub上的相关工作与参与者分析
|
||||
|
||||
### 10.1 国际主要研究者与实验室
|
||||
在“理解物理世界”方向，尤其是隐空间预测、因果推理和具身智能领域，以下实验室和研究人员有显著影响力：
|
||||
|
||||
* **Meta AI (FAIR)**:
|
||||
* **代表人物**: Yann LeCun (JEPA架构的提出者), Jean-Baptiste Alayrac (V-JEPA主要贡献者), Antoine Bordes。
|
||||
* **研究方向**: 联合嵌入预测架构 (JEPA) 及其在多模态、视频理解中的应用，旨在学习世界的高级语义和因果关系。其GitHub项目如 `facebookresearch/jepa` 和 `facebookresearch/v-jepa` 是核心开源资源。
|
||||
* **DeepMind (Google DeepMind)**:
|
||||
* **代表人物**: Danijar Hafner (Dreamer系列主要贡献者), Matthew Botvinick, Timothy Lillicrap。
|
||||
* **研究方向**: 基于模型的强化学习 (Model-Based Reinforcement Learning, MBRL)，特别是Dreamer系列。他们通过在学习到的世界模型中规划和训练策略，极大地提升了样本效率和对复杂物理环境的理解能力。相关论文常见于NeurIPS, ICLR。
|
||||
* **MIT (麻省理工学院)**:
|
||||
* **代表人物**: Joshua B. Tenenbaum (认知科学与物理直觉的AI建模先驱), Tejas D. Kulkarni。
|
||||
* **研究方向**: 专注于通过程序式推理、因果图学习和物理引擎（或“直觉物理引擎”）来模拟人类的物理认知。他们的工作通常结合贝叶斯推断和深度学习，探索AI如何从少量数据中学习物理常识。
|
||||
* **Stanford University (斯坦福大学)**:
|
||||
* **代表人物**: Fei-Fei Li (部分工作涉及具身智能与物理交互), Chelsea Finn。
|
||||
* **研究方向**: 机器人学习、具身智能中的物理理解与操作。例如，通过学习物体属性（刚度、摩擦力）来进行灵巧操作，或预测机器人与环境交互的物理后果。
|
||||
|
||||
### 10.2 中国主要研究者与实验室
|
||||
中国在“理解物理世界”方向的研究虽然在理论基础提出上相对国际稍晚，但在应用和追赶上表现出强劲势头，尤其在与机器人、自动驾驶等具身智能结合的场景：
|
||||
|
||||
* **清华大学**:
|
||||
* **代表人物**: 孙茂松 (自然语言处理与认知智能), 朱军 (机器学习，部分涉及表示学习)。
|
||||
* **研究方向**: 在具身智能、多模态学习、以及AI如何理解和表达物理世界常识方面有布局，与生数科技合作的Vidu项目也体现了对物理一致性的关注。
|
||||
* **北京大学**:
|
||||
* **代表人物**: 袁粒 (主导 `Open-Sora-Plan` 等大型开源项目), 林宙辰。
|
||||
* **研究方向**: 机器学习基础理论、具身感知与规划，尤其在将视觉信息转化为物理世界的结构化理解，并应用于机器人控制和自主导航方面有深入探索。其团队的开源项目 `PKU-YuanGroup/Open-Sora-Plan` 也包含了对物理运动和场景理解的模块。
|
||||
* **浙江大学**:
|
||||
* **代表人物**: 潘纲 (机器人与智能系统)。
|
||||
* **研究方向**: 具身感知、机器人操作、多模态融合，其研究往往关注如何让机器人更好地理解和应对物理世界中的不确定性。
|
||||
* **中国科学院自动化研究所**:
|
||||
* **代表人物**: 乔红 (机器人视觉与控制)。
|
||||
* **研究方向**: 专注于机器人视觉、具身智能和跨模态感知，探索如何让机器人在复杂的物理环境中进行有效的感知、理解和决策。
|
||||
|
||||
### 10.3 GitHub 开源生态的贡献
|
||||
虽然“理解物理世界”相比“构建物理世界”（视频生成）的开源项目更侧重研究框架而非直接应用，但仍有许多关键库和框架促进了该领域发展：
|
||||
|
||||
* **DreamerV3 (`danijar/dreamerv3`)**: DeepMind 的 Danijar Hafner 团队开源的项目，是基于模型强化学习的标杆，其代码实现和框架设计对理解模型内部的物理动态预测机制提供了直接参考。
|
||||
* **JEPA 相关实现**: 虽然Meta FAIR的官方JEPA项目主要聚焦视觉表示学习，但其核心思想和自监督范式被广泛复现和扩展，尤其是在各种 `pytorch-jepa` 或 `tensorflow-jepa` 的非官方实现中，用于探索如何更有效地学习物理世界的隐式表征。
|
||||
* **MuJoCo / Isaac Sim**: 这些物理仿真环境（NVIDIA的Isaac Sim）本身就是开源或免费使用的，它们提供了高度逼真的物理交互平台，是所有“理解物理世界”研究的数据生成和验证不可或缺的基础设施。
|
||||
|
||||
这些实验室和开源项目共同构成了“理解物理世界”研究的核心力量，通过理论创新和实践验证，推动AI向更深层次的物理认知迈进。
|
||||
|
||||
---
|
||||
|
||||
## 11. 国内外研究现状综述
|
||||
|
||||
### 11.1 生成式世界模型路线
|
||||
以 Sora（OpenAI, 2024）、Kling（快手, 2024）、Veo（Google DeepMind, 2024）为代表的视频扩散模型，通过在像素空间拟合海量数据，在视觉真实感上达到了前所未有的水平。近期开源的 **LingBot-World** (Robbyant Team, 2026) 进一步引入了相机与动作的连续控制，实现了分钟级的长时一致性与亚秒级实时交互。然而 Kang et al. (2024) 在 *How Far is Video Generation from World Model* 中发现，传统 SOTA 视频生成模型在分布外（OOD）物理场景下的违例率超过 60%。虽然 LingBot-World 极大提升了动态可控性，但这种纯生成式路径是否能完全内化隐藏因果律（如质量差异导致的受力反馈变化）仍是亟待补充的科学空白。
|
||||
|
||||
### 11.2 隐空间预测路线（JEPA 家族）
|
||||
LeCun (2022) 在 *A Path Towards Autonomous Machine Intelligence* 中提出 JEPA，主张在抽象表征层面进行预测以过滤不可预测的像素噪声。后续工作 I-JEPA (Assran et al., 2023)、V-JEPA (Bardes et al., 2024) 在图像与视频自监督学习中验证了该范式的有效性。V-JEPA 2 (Meta, 2025) 进一步将该架构扩展到具身规划场景，但其物理因果性仍依赖隐式涌现，缺乏可解释的物理量对齐。
|
||||
|
||||
### 11.3 基于模型的强化学习路线（Dreamer 家族）
|
||||
Hafner et al. 的 DreamerV1–V3 (2019–2023) 通过 RSSM（Recurrent State Space Model）在隐空间预测奖励与状态转移，在 Atari、DMC、Minecraft 等基准上实现高样本效率。DayDreamer (Wu et al., 2022) 将其迁移到真实机器人。然而 Dreamer 的隐状态是任务驱动的，难以泛化到未训练任务的物理推理。
|
||||
|
||||
### 11.4 直觉物理与可微仿真路线
|
||||
Tenenbaum 团队的 *Galileo* (Wu et al., 2015)、*Physion* (Bear et al., 2021) 等基准强调物理常识评测。可微物理引擎（如 Brax、Warp、NVIDIA Newton）使梯度可穿透物理求解器，与神经网络协同优化。该路线与 JEPA 路线的融合尚处于早期。
|
||||
|
||||
### 11.5 因果表征学习
|
||||
Schölkopf 等人 (2021) 在 *Toward Causal Representation Learning* 中系统阐述了将因果发现引入表征学习的必要性。CITRIS (Lippe et al., 2022)、BISCUIT (Lippe et al., 2023) 等工作在合成动态环境中验证了从动作干预中识别因果变量的可行性，但尚未扩展至高维视频与真实物理交互。
|
||||
|
||||
### 11.6 几何重建路线（视觉/激光雷达驱动的 1:1 真实世界数字孪生）
|
||||
这一流派的核心目标不是"预测物理后果"，而是**先把现实世界以毫米—厘米级精度搬进计算机**，再在其上叠加物理与语义。它构成"理解物理世界"的几何底座，常与世界模型形成上下游关系。代表性技术分支包括：
|
||||
|
||||
* **传统多视图几何与 SLAM**：Structure-from-Motion（COLMAP, [arXiv:1604.01093](https://arxiv.org/abs/1604.01093)）、Visual-Inertial SLAM（ORB-SLAM3, [arXiv:2007.11898](https://arxiv.org/abs/2007.11898)）、LiDAR SLAM（LOAM、LIO-SAM、FAST-LIO2, [arXiv:2107.06829](https://arxiv.org/abs/2107.06829)）提供稀疏点云与位姿；激光-视觉-惯性紧耦合（如 R3LIVE, [arXiv:2109.07982](https://arxiv.org/abs/2109.07982)）进一步提升尺度准确性。
|
||||
* **神经辐射场与 3D 高斯泼溅（3DGS）**：NeRF（[arXiv:2003.08934](https://arxiv.org/abs/2003.08934)）、Instant-NGP（[arXiv:2201.05989](https://arxiv.org/abs/2201.05989)）、Mip-NeRF 360（[arXiv:2111.12077](https://arxiv.org/abs/2111.12077)）实现照片级隐式重建；3D Gaussian Splatting（[arXiv:2308.04079](https://arxiv.org/abs/2308.04079)）将渲染速度推至实时，已成为数字孪生主流表征。后续 PhysGaussian（[arXiv:2311.12198](https://arxiv.org/abs/2311.12198)）、Spring-Gaus 等工作把高斯基元接入物理求解器，实现"可重建即可仿真"。
|
||||
* **大规模城市级与场景级重建**：Block-NeRF（[arXiv:2202.05263](https://arxiv.org/abs/2202.05263)）、Google Immersive View、Hierarchical 3DGS（[arXiv:2406.12080](https://arxiv.org/abs/2406.12080)）面向街区/城市级别；NVIDIA Omniverse、CARLA、Cosmos-Drive（NVIDIA, 2025）将重建结果转换为可仿真、可交互的 OpenUSD 数字孪生。
|
||||
* **自动驾驶占据栅格与 BEV**：Tesla Occupancy Network、OccWorld（[arXiv:2311.16038](https://arxiv.org/abs/2311.16038)）、OCC3D-nuScenes、UniScene 等以 3D 体素或 BEV 网格预测周围空间占据与语义，是 LiDAR + 多目视觉融合的工业落地范式。
|
||||
* **大规模重建数据与基准**：Waymo Open（[官网](https://waymo.com/open/)）、nuScenes（[arXiv:1903.11027](https://arxiv.org/abs/1903.11027)）、KITTI-360（[arXiv:2109.13410](https://arxiv.org/abs/2109.13410)）、Argoverse 2、Matterport3D、ScanNet++ 提供多模态传感器对齐数据。
|
||||
* **物理可交互的孪生**：Gaussian Splatting + MPM/FEM（PhysGaussian、PIE-NeRF, [arXiv:2311.13099](https://arxiv.org/abs/2311.13099)）、Genesis（[官网](https://genesis-embodied-ai.github.io/)）、Real2Sim2Real 流水线（如 RoboCasa, [arXiv:2406.02523](https://arxiv.org/abs/2406.02523)）把"几何 1:1"延伸为"物理 1:1"。
|
||||
|
||||
**与本项目的关系**：几何重建路线提供**精确空间锚点**，但本身并不学习"为什么会这样"的因果机制。本项目可在三个层面与该路线协同：
|
||||
1. **数据生成**：用 3DGS/NeRF 重建的真实场景替代部分纯仿真数据，缩小 Sim2Real Gap；
|
||||
2. **隐空间监督**：将重建得到的真值几何（深度、占据、姿态）作为 M-JEPA 隐变量的可选弱监督锚点，加速物理量解耦；
|
||||
3. **闭环验证**：在 PhysGaussian 类"可仿真孪生"中进行反事实物理推理评测，使得"理解"的检验不再局限于合成场景。
|
||||
|
||||
#### 11.6.1 室内空间重建子方向
|
||||
室内场景具有"尺度小、纹理弱、结构强、遮挡多、动态人/物频繁"等特点，催生出一批专门的研究方向：
|
||||
|
||||
* **稠密深度与 TSDF 融合**：以 KinectFusion（[ISMAR'11](https://www.microsoft.com/en-us/research/publication/kinectfusion-real-time-3d-reconstruction-and-interaction-using-a-moving-depth-camera/)）、BundleFusion（[arXiv:1604.01093v2](https://arxiv.org/abs/1604.01093)）、Voxblox、Atlas（[arXiv:2003.10432](https://arxiv.org/abs/2003.10432)）、NeuralRecon（[arXiv:2104.00681](https://arxiv.org/abs/2104.00681)）为代表，利用 RGB-D 或多视图实时融合 TSDF 体素，重建表面网格。
|
||||
* **室内 NeRF / 3DGS**：NeRF in the Wild、NICE-SLAM（[arXiv:2112.12130](https://arxiv.org/abs/2112.12130)）、Co-SLAM（[arXiv:2304.14377](https://arxiv.org/abs/2304.14377)）、Point-SLAM、Gaussian-SLAM（[arXiv:2312.10070](https://arxiv.org/abs/2312.10070)）、SplaTAM（[arXiv:2312.02126](https://arxiv.org/abs/2312.02126)）、MonoGS（[arXiv:2312.06741](https://arxiv.org/abs/2312.06741)）把神经隐式/高斯表征与 SLAM 融合，实现单目/RGB-D 实时建图。
|
||||
* **结构化布局与 CAD 抽象**：HorizonNet、LayoutNet、PanoContext、RoomFormer（[arXiv:2210.12058](https://arxiv.org/abs/2210.12058)）、SceneCAD（[arXiv:2003.12622](https://arxiv.org/abs/2003.12622)）、Mask3D / ODIN 等从全景或点云中提取墙-地-天花板的曼哈顿结构和家具的 CAD 替身，便于编辑与仿真。
|
||||
* **语义/实例/全景 3D 分割**：ScanNet（[arXiv:1702.04405](https://arxiv.org/abs/1702.04405)）、ScanNet++（[arXiv:2308.11417](https://arxiv.org/abs/2308.11417)）、Replica（[arXiv:1906.05797](https://arxiv.org/abs/1906.05797)）、Matterport3D（[arXiv:1709.06158](https://arxiv.org/abs/1709.06158)）、ARKitScenes（[arXiv:2111.08897](https://arxiv.org/abs/2111.08897)）提供大规模标注；MinkowskiNet、Mask3D、OpenScene、ConceptGraphs（[arXiv:2309.16650](https://arxiv.org/abs/2309.16650)）实现从点云到开放词表语义。
|
||||
* **可交互场景与具身仿真平台（细化）**：本类平台不仅提供"几何 1:1 的房间"，更要回答"AI 能否理解空间布局、物体形状、物理特性与功能可供性（affordance）"四个层面的问题。下面按能力维度展开：
|
||||
|
||||
* **A. 空间结构理解（Spatial Layout）**
|
||||
* **平台**：Habitat 3.0（[arXiv:2310.13724](https://arxiv.org/abs/2310.13724)）、HM3D-Semantics（[arXiv:2210.05633](https://arxiv.org/abs/2210.05633)）、Matterport3D（[arXiv:1709.06158](https://arxiv.org/abs/1709.06158)）、Gibson Env（[CVPR'18](https://arxiv.org/abs/1808.10654)）、ProcTHOR（[arXiv:2206.06994](https://arxiv.org/abs/2206.06994)）。
|
||||
* **关注能力**：房间-门-走廊拓扑、可通行区域、家具占据、视野遮挡、空间记忆（episodic map）。
|
||||
* **典型任务**：ObjectNav、ImageNav、Multi-Object Navigation、Room Rearrangement（[arXiv:2011.01975](https://arxiv.org/abs/2011.01975)）、SPOC（[arXiv:2312.02976](https://arxiv.org/abs/2312.02976)）。
|
||||
* **评测指标**：SPL、Success-weighted by Path Length、Coverage、Map Completion。
|
||||
|
||||
* **B. 物体几何理解（Object Shape & Geometry）**
|
||||
* **平台**：SAPIEN / PartNet-Mobility（[arXiv:2003.08515](https://arxiv.org/abs/2003.08515)）、Objaverse-XL（[arXiv:2307.05663](https://arxiv.org/abs/2307.05663)）、GAPartNet（[arXiv:2211.05272](https://arxiv.org/abs/2211.05272)）、ShapeNet-Sem、AKB-48（[arXiv:2202.08432](https://arxiv.org/abs/2202.08432)）、GraspNet-1Billion（[CVPR'20](https://arxiv.org/abs/1912.13470)）。
|
||||
* **关注能力**：精细网格/SDF/凸分解、薄壁与镂空、关节轴方向（prismatic/revolute）、点云-视觉对齐、6D 位姿与尺寸。
|
||||
* **典型任务**：6-DoF 抓取、铰接预测（Ditto、[arXiv:2202.08227](https://arxiv.org/abs/2202.08227)）、Part Segmentation、Shape Completion、Articulation Estimation。
|
||||
|
||||
* **C. 物理特性理解（Physical Properties）**
|
||||
* **平台**：Isaac Lab / Isaac Sim（[文档](https://isaac-sim.github.io/IsaacLab/)）、ManiSkill 3（[arXiv:2410.00425](https://arxiv.org/abs/2410.00425)）、RoboCasa（[arXiv:2406.02523](https://arxiv.org/abs/2406.02523)）、Genesis（[官网](https://genesis-embodied-ai.github.io/)）、SoftGym（[arXiv:2011.07215](https://arxiv.org/abs/2011.07215)）、PlasticineLab（[arXiv:2104.03311](https://arxiv.org/abs/2104.03311)）、FluidLab（[arXiv:2303.02346](https://arxiv.org/abs/2303.02346)）、ThreeDWorld / TDW（[arXiv:2007.04954](https://arxiv.org/abs/2007.04954)）。
|
||||
* **关注能力**：质量、惯性张量、摩擦系数、恢复系数、刚柔耦合、布料/液体/颗粒、热与声学（TDW 提供 PyImpact 音频，可用于"听-碰撞"多模态）。
|
||||
* **典型任务**：物体属性估计（mass/friction inference）、Cloth Folding、Liquid Pouring、Dough Manipulation、Pile Sorting、Tool Use。
|
||||
* **评测指标**：物理量回归误差（MSE on mass/μ）、任务成功率、能量守恒违例率、长时滚动误差。
|
||||
|
||||
* **D. 功能可供性与作用理解（Affordance & Function）**
|
||||
* **平台**：BEHAVIOR-1K / OmniGibson（[arXiv:2403.09227](https://arxiv.org/abs/2403.09227)）、ALFRED（[arXiv:1912.01734](https://arxiv.org/abs/1912.01734)）、ALFWorld、ARNOLD（[arXiv:2304.04321](https://arxiv.org/abs/2304.04321)）、CALVIN（[arXiv:2112.03227](https://arxiv.org/abs/2112.03227)）、RoboTHOR、RLBench（[arXiv:1909.12271](https://arxiv.org/abs/1909.12271)）。
|
||||
* **关注能力**：物体的"可被打开/拿起/倒入/坐上"等动作可供性、语言-动作-物体三元绑定、长时任务分解、状态变化（cooked、sliced、filled、stained）。
|
||||
* **典型任务**：Language-conditioned Manipulation、Long-horizon Household Tasks（"煮一杯咖啡"）、Tool Substitution、Counterfactual Affordance（"如果杯子破了，能否盛水？"）。
|
||||
|
||||
* **E. 多智能体与人-机器人共存**
|
||||
* **平台**：Habitat 3.0 Social Rearrangement（[arXiv:2310.13724](https://arxiv.org/abs/2310.13724)）、OVMM（[arXiv:2306.11565](https://arxiv.org/abs/2306.11565)）、iGibson Social、Overcooked-AI。
|
||||
* **关注能力**：人类轨迹预测、协作意图推理、避让与让行、物体共享。
|
||||
|
||||
* **F. 仿真器选型权衡（与本项目相关）**
|
||||
|
||||
| 平台 | 渲染 | 物理 | 铰接 | 柔体/流体 | 大规模场景 | 主要语言/接口 |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Habitat 3.0 | 高速 PBR | Bullet | 中 | 弱 | ✅ HSSD/HM3D | Python/C++ |
|
||||
| Isaac Lab | RTX | PhysX 5 | 强 | 中（PBD） | 中 | Python/USD |
|
||||
| ManiSkill 3 | 高 | SAPIEN | 强 | 强（Warp） | 中 | Python |
|
||||
| RoboCasa | 高 | MuJoCo | 强 | 弱 | ✅ 厨房 | Python |
|
||||
| Genesis | 高 | 多后端 | 强 | 强（MPM/SPH） | 中 | Python |
|
||||
| TDW | 影视级 | Flex/Unity | 中 | 中 | 中 | Python |
|
||||
|
||||
**与本项目子课题 3 的具体映射**：
|
||||
1. **空间理解评测**：在 HSSD/HM3D 上跑 ObjectNav 与 Rearrangement，验证 M-JEPA 隐空间是否编码了房间拓扑（探针：从隐状态线性回归占据栅格）。
|
||||
2. **几何与铰接评测**：在 PartNet-Mobility/GAPartNet 上做关节轴预测与开门/拉抽屉操作，验证模型对"零件级几何 + 运动学约束"的内化。
|
||||
3. **物理量对齐**：在 ManiSkill 3 + Genesis 上做"看视频估质量/摩擦"探针任务，定量评估隐变量与真值物理量的相关系数（Pearson ρ）。
|
||||
4. **可供性与反事实**：在 BEHAVIOR-1K / ARNOLD 上设计反事实任务（更换材质、改变重力、移除物体），考查 CPL 损失是否真正提升了 OOD 物理推理。
|
||||
* **场景图与可编辑数字孪生**：3D Scene Graph（[arXiv:1910.02527](https://arxiv.org/abs/1910.02527)）、Kimera（[arXiv:1910.02490](https://arxiv.org/abs/1910.02490)）、SceneGraphFusion、ConceptGraphs 将几何重建抽象为"房间-物体-关系"的图结构，支持语言查询与重排。
|
||||
* **动态与人-物交互重建**：BEHAVE（[arXiv:2204.06950](https://arxiv.org/abs/2204.06950)）、CHAIRS、HumanISR、Neural Human Performer 处理人与家具的接触/动态；Dynamic-NeRF、D-3DGS 用于动态室内场景。
|
||||
* **铰接物体与可操作部件**：PartNet-Mobility / SAPIEN（[arXiv:2003.08515](https://arxiv.org/abs/2003.08515)）、Ditto（[arXiv:2202.08227](https://arxiv.org/abs/2202.08227)）、Real2Code、CARTO 重建抽屉、门、把手等关节，为具身操作提供"可动"几何。
|
||||
* **房间布局生成与程序化合成**：ATISS（[arXiv:2110.03675](https://arxiv.org/abs/2110.03675)）、DiffuScene、LEGO-Net、Holodeck（[arXiv:2312.09067](https://arxiv.org/abs/2312.09067)）、Infinigen Indoors（[arXiv:2406.11824](https://arxiv.org/abs/2406.11824)）通过扩散/LLM 程序化生成多样化室内场景，作为重建之外的"无限数据"补充。
|
||||
* **少量/单图重建与基础模型驱动**：RoomNet、PERF、One-2-3-45、LRM（[arXiv:2311.04400](https://arxiv.org/abs/2311.04400)）、ZeroNVS、CAT3D、DUSt3R（[arXiv:2312.14132](https://arxiv.org/abs/2312.14132)）、MASt3R、Spann3R 把单图/稀疏图重建推进到秒级，对室内"快速建模"友好。
|
||||
|
||||
**与本项目的接口**：本项目子课题 3（具身验证）将优先选用 **Habitat 3.0 / RoboCasa / HSSD** 作为室内交互平台；几何端可采用 **Gaussian-SLAM/MonoGS** 在线建图，并以 **PartNet-Mobility / Ditto** 提供铰接物体先验，使因果世界模型既学习刚体物理，也覆盖"开门、拉抽屉、倒水"这类室内常见的接触-铰接交互。
|
||||
|
||||
### 11.7 研究空白与本项目的切入点
|
||||
现有工作呈现明显分裂：（i）生成式模型重视觉而轻因果；（ii）JEPA/Dreamer 重预测而轻可解释物理对齐；（iii）因果学习重理论而轻具身验证。本项目主张在 **JEPA 隐空间 + 因果图结构 + 可微物理先验 + 具身验证** 四要素融合处建立新的研究坐标，填补"可解释的物理因果世界模型"这一空白。
|
||||
|
||||
---
|
||||
|
||||
## 12. 创新点
|
||||
|
||||
1. **方法创新——M-JEPA + 因果图联合架构**：首次将多模态（视-触-力）JEPA 与结构因果模型（SCM）耦合，并通过互信息正则项强制隐变量解耦至质量、摩擦力等可解释物理维度。
|
||||
2. **目标函数创新——反事实预测损失（CPL）**：在自监督预训练阶段引入隐空间干预（do-操作），要求模型在反事实物理参数下输出一致的因果后果，从根本上抑制物理幻觉。
|
||||
3. **评测体系创新——Physics-Understanding Benchmark (PUB)**：构建包含 VoE（Violation-of-Expectation）、反事实预测、长时轨迹外推、Zero-shot 操作四维评测协议，弥补现有物理评测多集中于"识别违例"而忽视"主动预测"的不足。
|
||||
4. **闭环验证创新——仿真到真机的物理对齐迁移**：通过 Isaac Sim 到真实机械臂的 Sim2Real 双向校准，量化隐空间物理量与真实测量值的相关性，提供可解释性证据。
|
||||
|
||||
---
|
||||
|
||||
## 13. 可行性分析
|
||||
|
||||
### 13.1 理论可行性
|
||||
JEPA、Dreamer、SCM 三条技术线均已在各自领域取得里程碑成果，本项目的融合在数学上有清晰的目标函数表达（最大化预测互信息 + 最小化因果违背项），不存在原理性障碍。
|
||||
|
||||
### 13.2 数据可行性
|
||||
NVIDIA Isaac Sim、MuJoCo MJX、Genesis、Unreal Engine 5 Chaos 物理引擎均可提供高保真物理交互数据，并可批量导出隐藏的真值物理参数用于评测对齐。我们预估 30 万条 5–10 秒视频可在 2 周内于 8×A100 集群上生成。
|
||||
|
||||
### 13.3 算力可行性
|
||||
项目主要预训练规模在 1B 参数以内，参考 V-JEPA-L (0.3B) 的训练成本（约 16K A100·hours），本项目主体训练可在 64×A100/H100 集群上 3–4 周内完成，与课题组现有/可申请算力规模匹配。
|
||||
|
||||
### 13.4 团队可行性
|
||||
课题组已有 JEPA 自监督学习、机器人操作、可微仿真三个方向的前期积累，并具备 Isaac Sim/Genesis 部署经验，工程基础具备。
|
||||
|
||||
---
|
||||
|
||||
## 14. 风险分析与应对
|
||||
|
||||
| 风险类别 | 描述 | 影响等级 | 应对策略 |
|
||||
| --- | --- | --- | --- |
|
||||
| 表征坍缩 | JEPA 类自监督训练易出现 collapse | 高 | 引入 VICReg/Barlow-Twins 正则；多模态对比学习互锁 |
|
||||
| 因果识别不可识别性 | 无干预下隐式因果发现的非唯一性 | 高 | 利用主动干预（机器人动作）与已知物理对称性作弱监督 |
|
||||
| Sim2Real Gap | 仿真训练模型在真机上失效 | 中 | 域随机化 + 真机微调；保留 5% 真机数据做对齐评测 |
|
||||
| 长序列误差累积 | 自回归预测漂移 | 中 | 采用层级化时间抽象（HRSSM）与教师强制退火 |
|
||||
| 算力受限 | 顶会前算力高峰排队 | 中 | 提前预约共享集群；采用 LoRA/QLoRA 微调降级路线 |
|
||||
| 数据集偏差 | 仿真物理与真实物理分布不一致 | 低 | 引入真实视频数据集（Something-Something v2, Ego4D）辅助预训练 |
|
||||
|
||||
---
|
||||
|
||||
## 15. 团队组成与分工建议
|
||||
|
||||
- **项目负责人（PI，1 名）**：总体把控、论文撰写、对外合作。
|
||||
- **博士后/高级研究员（1–2 名）**：负责 M-JEPA 架构与因果模块核心算法实现。
|
||||
- **博士生（2–3 名）**：分别承担数据构建、预训练 pipeline、具身验证。
|
||||
- **硕士生（2 名）**：协助评测基准建设、可视化分析、消融实验。
|
||||
- **工程师/RA（1 名）**：负责仿真平台搭建、集群运维、开源代码工程化。
|
||||
- **外部合作**：与机器人实验室共享真机平台；与认知科学团队合作设计 VoE 评测。
|
||||
|
||||
---
|
||||
|
||||
## 16. 经费预算概览（参考，单位：万元 RMB，周期 12 个月）
|
||||
|
||||
| 科目 | 预算 | 用途说明 |
|
||||
| --- | --- | --- |
|
||||
| 算力与云服务 | 80 | A100/H100 集群租用、对象存储、推理服务 |
|
||||
| 仿真与硬件 | 40 | Isaac Sim 工作站、机械臂/夹爪、力觉传感器 |
|
||||
| 数据采集与标注 | 15 | 真实视频采集、人工 VoE 标签校验 |
|
||||
| 国际会议与差旅 | 12 | NeurIPS/ICLR/ICML/RSS 注册与差旅 |
|
||||
| 论文发表与开源运维 | 5 | OA 费、GitHub/HuggingFace 维护 |
|
||||
| 人员补助 | 30 | 学生津贴、合作访问学者 |
|
||||
| 机动与其他 | 8 | 风险储备 |
|
||||
| **合计** | **190** | |
|
||||
|
||||
---
|
||||
|
||||
## 17. 伦理、合规与开源策略
|
||||
|
||||
1. **数据合规**：仅使用公开发布或自采且已脱敏的视频数据；真实人物出镜数据需获得书面同意。
|
||||
2. **机器人安全**：所有真机实验设置物理限位与急停；高速运动实验在隔离围栏内进行。
|
||||
3. **算法风险评估**：在公开模型权重前进行误用风险评估，明确禁止用于自主武器等场景。
|
||||
4. **开源许可**：代码采用 Apache-2.0；数据集采用 CC BY-NC 4.0；预训练权重采用 LLaMA-style 研究使用协议。
|
||||
5. **可复现性**：随论文发布种子、训练日志（W&B）、Docker 镜像与一键复现脚本。
|
||||
|
||||
---
|
||||
|
||||
## 18. 主要参考文献（选列）
|
||||
|
||||
1. LeCun, Y. (2022). *A Path Towards Autonomous Machine Intelligence*. Open Review. [[PDF]](https://openreview.net/pdf?id=BZ5a1r-kVsf)
|
||||
2. Assran, M. et al. (2023). *Self-Supervised Learning from Images with a Joint-Embedding Predictive Architecture* (I-JEPA). CVPR. [[arXiv:2301.08243]](https://arxiv.org/abs/2301.08243)
|
||||
3. Bardes, A. et al. (2024). *V-JEPA: Latent Video Prediction for Visual Representation Learning*. Meta AI. [[Paper]](https://ai.meta.com/research/publications/revisiting-feature-prediction-for-learning-visual-representations-from-video/)
|
||||
4. Hafner, D. et al. (2023). *Mastering Diverse Domains through World Models* (DreamerV3). [[arXiv:2301.04104]](https://arxiv.org/abs/2301.04104)
|
||||
5. Wu, J., Tenenbaum, J. B. et al. (2015). *Galileo: Perceiving Physical Object Properties by Integrating a Physics Engine with Deep Learning*. NeurIPS. [[PDF]](https://papers.nips.cc/paper_files/paper/2015/hash/d09bf41544a3365a46c9077ebb5e35c3-Abstract.html)
|
||||
6. Bear, D. et al. (2021). *Physion: Evaluating Physical Prediction from Vision in Humans and Machines*. NeurIPS Datasets & Benchmarks. [[arXiv:2106.08261]](https://arxiv.org/abs/2106.08261)
|
||||
7. Schölkopf, B. et al. (2021). *Toward Causal Representation Learning*. Proceedings of the IEEE. [[arXiv:2102.11107]](https://arxiv.org/abs/2102.11107)
|
||||
8. Lippe, P. et al. (2022). *CITRIS: Causal Identifiability from Temporal Intervened Sequences*. ICML. [[arXiv:2202.03169]](https://arxiv.org/abs/2202.03169)
|
||||
9. Lippe, P. et al. (2023). *BISCUIT: Causal Representation Learning from Binary Interactions*. UAI. [[arXiv:2306.09643]](https://arxiv.org/abs/2306.09643)
|
||||
10. Kang, B. et al. (2024). *How Far is Video Generation from World Model: A Physical Law Perspective*. [[arXiv:2411.02385]](https://arxiv.org/abs/2411.02385)
|
||||
11. Wu, P. et al. (2022). *DayDreamer: World Models for Physical Robot Learning*. CoRL. [[arXiv:2206.14176]](https://arxiv.org/abs/2206.14176)
|
||||
12. Brohan, A. et al. (2023). *RT-2: Vision-Language-Action Models*. Google DeepMind. [[arXiv:2307.15818]](https://arxiv.org/abs/2307.15818)
|
||||
13. Makoviychuk, V. et al. (2021). *Isaac Gym: High Performance GPU-Based Physics Simulation for Robot Learning*. NeurIPS. [[arXiv:2108.10470]](https://arxiv.org/abs/2108.10470)
|
||||
14. Robbyant Team, Gao, Z., Wang, Q., et al. (2026). *Advancing Open-source World Models* (LingBot-World). [[arXiv:2601.20540]](https://arxiv.org/abs/2601.20540)
|
||||
|
||||
---
|
||||
|
||||
## 19. 里程碑与可交付物（Deliverables）
|
||||
|
||||
| 时间节点 | 里程碑 | 可交付物 |
|
||||
| --- | --- | --- |
|
||||
| M3 | 基线复现完成 | V-JEPA / DreamerV3 复现报告 + 仿真数据集 v0.1 |
|
||||
| M6 | M-JEPA 架构跑通 | 预训练权重 v0.5 + 内部技术报告 |
|
||||
| M8 | 因果模块集成 | Causal-Physics-JEPA v1.0 + 评测基准 PUB v1.0 |
|
||||
| M10 | 具身实验完成 | 机械臂操作视频 + 真机/仿真对照评测结果 |
|
||||
| M12 | 项目结题 | 2–3 篇顶会投稿 + 开源代码/权重/数据集发布 |
|
||||
@@ -0,0 +1,66 @@
|
||||
# AI世界模型（World Models）技术综述
|
||||
|
||||
## 1. 引言
|
||||
|
||||
“世界模型”（World Model）这一概念最初在认知科学和强化学习中被提出，近年来随着生成式AI的爆发，它成为了迈向通用人工智能（AGI）的核心路径之一。世界模型旨在让AI不仅能生成文本或像素，更能“理解”和“模拟”现实世界的物理规律、因果关系和时空一致性。
|
||||
|
||||
本综述基于arXiv上的最新学术论文以及GitHub上的开源项目动态，对当前国际与中国在AI世界模型领域的发展进行系统梳理和对比分析。
|
||||
|
||||
## 2. 国际主流研究与开源进展
|
||||
|
||||
国际顶尖机构在世界模型的探索上呈现出多模态、大模型与强化学习深度结合的趋势。
|
||||
|
||||
### 2.1 理论基础与联合嵌入预测架构 (JEPA)
|
||||
Yann LeCun 提出了基于目标驱动的AI架构（Objective-Driven AI），其中的核心就是世界模型。其团队（Meta FAIR）在GitHub上开源了 **I-JEPA** 和 **V-JEPA**。这类架构通过在隐空间（Latent Space）中预测缺失的视频或图像片段，让模型学习世界的高级语义和动态规律，而过滤掉不必要的像素级噪音。
|
||||
* **相关项目**: `facebookresearch/jepa`, `facebookresearch/v-jepa`
|
||||
|
||||
### 2.2 视频生成与物理世界模拟
|
||||
OpenAI的 **Sora** 是生成式世界模型的一个里程碑。Sora采用Diffusion Transformer (DiT) 架构，证明了当模型规模（Scaling Law）足够大时，AI能够在一定程度上涌现出对3D一致性、物体持久性和基础物理规律的模拟能力。此外，Runway的Gen-2、Pika等产品也在商业和应用层面对现实世界模拟进行了探索。
|
||||
* **技术关键词**: DiT (Diffusion Transformer), Space-Time Latent Patches.
|
||||
|
||||
### 2.3 具身智能与自动驾驶世界模型
|
||||
在自动驾驶和具身智能（Embodied AI）领域，英国自动驾驶公司Wayve提出了 **GAIA-1** 和 **LINGO-1**，这是一个生成式自动驾驶世界模型，能够根据文本、视频和动作条件生成未来的驾驶场景。而在强化学习领域，Danijar Hafner等人提出的 **Dreamer** 系列（目前演进到DreamerV3）通过在想象的隐空间世界模型中进行策略训练，极大地提高了样本效率。
|
||||
* **相关项目**: `danijar/dreamerv3`
|
||||
|
||||
## 3. 中国在世界模型领域的研究与发展
|
||||
|
||||
中国科研机构和科技企业在世界模型，尤其是基于视频生成的物理世界模拟和开源生态建设上，展现出了极强的追赶和创新能力。
|
||||
|
||||
### 3.1 视频生成商业与技术双轮驱动
|
||||
中国业在视频生成方向推出了多个对标Sora的重磅产品，这些模型不仅在生成时长和分辨率上取得了突破，更在物理规律的遵循上进行了深度优化。
|
||||
* **Vidu** (生数科技 & 清华大学): 采用U-ViT架构，强调一键生成长视频及多镜头语言的理解。
|
||||
* **Kling (可灵)** (快手): 采用了3D时空联合注意力机制（3D VAE），能较好地模拟复杂物理动作和材质变化。
|
||||
* **CogVideoX** (智谱AI): 开源了多尺寸版本，通过结合3D Causal VAE和专家DiT结构，大幅降低了推理和训练成本，相关论文在arXiv上引起广泛关注。
|
||||
|
||||
### 3.2 繁荣的GitHub开源生态
|
||||
相比于OpenAI的闭源，中国的高校和社区在开源世界模型复现上做出了巨大贡献。
|
||||
* **Open-Sora** (`hpcaitech/Open-Sora`): 潞晨科技开源的Sora全面复现方案，大幅降低了DiT模型的训练门槛。
|
||||
* **Open-Sora-Plan** (`PKU-YuanGroup/Open-Sora-Plan`): 北京大学袁粒团队牵头的开源项目，提供了详细的数据清洗、模型架构和训练代码，成为GitHub上极具影响力的世界模型社区资源。
|
||||
|
||||
## 4. 核心技术路径对比
|
||||
|
||||
当前世界模型的发展主要存在两条并行的技术路径：
|
||||
|
||||
1. **基于生成的全像素模拟 (Generative World Models)**
|
||||
* **代表**: Sora, Open-Sora, CogVideoX
|
||||
* **方法**: 主要基于Diffusion或Autoregressive模型，直接在像素或低维像素潜空间（Latent Space）进行未来帧的生成。
|
||||
* **优势**: 直观，可解释性强（所见即所得），能直接用于影视和多媒体创作。
|
||||
* **劣势**: 极其消耗算力，难以保证长时间的严格物理规律（常出现穿模、反直觉物理现象）。
|
||||
2. **基于隐状态预测的抽象模型 (Latent Predictive World Models)**
|
||||
* **代表**: JEPA系列, Dreamer系列
|
||||
* **方法**: 不重建具体像素，而是预测未来状态的抽象表征。
|
||||
* **优势**: 计算效率高，更容易学习到核心因果关系和动作条件（Action-conditioned），非常适合具身智能和机器人。
|
||||
* **劣势**: 难以直接生成高保真视觉结果以供人类检查。
|
||||
|
||||
## 5. 挑战与未来展望
|
||||
|
||||
综合arXiv上的最新研究，世界模型在迈向成熟的道路上仍面临以下挑战：
|
||||
* **物理因果律的幻觉 (Hallucination of Physics)**: 当前模型更多是在“拟合”物理规律的外观，而非“理解”物理方程。玻璃破碎、流体动力学等复杂场景依然容易出错。
|
||||
* **长时一致性 (Long-horizon Consistency)**: 生成或预测跨度超过几分钟的事件时，实体特征和环境状态容易发生偏移。
|
||||
* **高质量数据集缺乏**: 带有丰富物理交互（如碰撞、形变）的高质量视频数据以及带有高质量文本标注的数据极其稀缺。
|
||||
|
||||
未来，结合3D引擎（如Unreal Engine）渲染的合成数据，以及结合隐空间预测（JEPA路线）与像素生成（Diffusion路线）的混合架构，将是世界模型发展的重要趋势。
|
||||
|
||||
## 6. 结论
|
||||
|
||||
从国际视角来看，世界模型正从单纯的视觉生成向“行动条件预测”和“具身智能”演进。Meta、OpenAI和Wayve等分别从预测架构、视觉生成和自动驾驶三个维度定义了世界模型。而在中国，学术界与产业界深度结合，凭借Vidu、Kling等惊艳的商业产品，以及Open-Sora等蓬勃发展的GitHub开源生态，正迅速缩小差距甚至在某些垂直领域实现超越。世界模型不仅是视频生成的底层引擎，更是通往拥有常识与推理能力的AGI的关键踏板。
|
||||
@@ -0,0 +1,742 @@
|
||||
# ZED 2i 数据 Pipeline 实时 arXiv 综述（自动生成）
|
||||
|
||||
> **数据来源**：[`research/search_results.json`](search_results.json) 由 [`research/search_info.py`](search_info.py) 通过 HTTP 代理 `127.0.0.1:6984` 拉取自 arxiv.org / api.github.com。
|
||||
> **生成时间**：2026-05-16T16:44:34+0800 (merged)
|
||||
> **检索代理**：http://127.0.0.1:6984
|
||||
> **每主题最多**：10 篇
|
||||
|
||||
## 0. 数据概览
|
||||
|
||||
- **arXiv 论文总数**：100
|
||||
- **arXiv 主题数**：10
|
||||
- **疑似国产团队论文**：1（占比 1%；🇨🇳 标记，启发式判断）
|
||||
- **GitHub 仓库总数**：50
|
||||
- **GitHub 主题数**：5
|
||||
|
||||
### 0.1 主类目分布（arXiv primary_category）
|
||||
|
||||
- `cs.CV`: 59
|
||||
- `cs.RO`: 31
|
||||
- `eess.SY`: 2
|
||||
- `cs.AI`: 2
|
||||
- `quant-ph`: 1
|
||||
- `cs.LG`: 1
|
||||
- `cs.MA`: 1
|
||||
- `eess.SP`: 1
|
||||
- `cs.HC`: 1
|
||||
- `cs.SD`: 1
|
||||
|
||||
### 0.2 与本项目框架的映射
|
||||
|
||||
| 项目阶段 | 主线主题 | 论文数 |
|
||||
|---|---|---|
|
||||
| M2-3 / M3-4 | A. 双目立体匹配（被动深度） | 10 |
|
||||
| M2-1 / M3-3 | B. 视觉惯性 SLAM / VIO | 10 |
|
||||
| M3-5 / M4 | C. 3D Gaussian Splatting SLAM（融合建图） | 10 |
|
||||
| M3-4 | D. 单目深度基础模型 | 10 |
|
||||
| M4-1 / M4-2 | E. 室内 RGB-D 数据集与重建 | 10 |
|
||||
| M4-4 | F. 视频世界模型（下游应用） | 10 |
|
||||
| 全周期 | G. ZED 相机相关应用工作 | 10 |
|
||||
| 国产化 / 替代硬件 | H. Orbbec / Femto / Azure Kinect 相关工作 | 10 |
|
||||
| M2-2 / M3-5 | I. RGB-D 室内重建 | 10 |
|
||||
| M3-4 | J. 神经立体深度（指定 RAFT/IGEV/Foundation 家族） | 10 |
|
||||
|
||||
---
|
||||
|
||||
# 第一部分 · arXiv 论文（按主题分组，时间新→旧）
|
||||
|
||||
## A. 双目立体匹配（被动深度）
|
||||
**项目阶段**: M2-3 / M3-4 | **论文数**: 10
|
||||
|
||||
对应 ZED 双目深度算法的替换/超越路线。关注零样本泛化、Transformer 架构、神经几何编码。
|
||||
|
||||
#### 1. [2605.14963](https://arxiv.org/abs/2605.14963) — H-OmniStereo: Zero-Shot Omnidirectional Stereo Matching with Heading-Aligned Normal Priors
|
||||
- **发表**: 2026-05-14 | **分类**: cs.CV
|
||||
- **作者**: Chenxing Jiang, Zhe Tong, Pusen Gao, Peize Liu 等 (8 人)
|
||||
- **摘要**: Stereo matching on top-bottom equirectangular images provides an effective framework for full-surround perception, as vertically aligned epipolar lines enable the use of advanced perspective stereo architectures that are largely driven by large-scale datasets and monocular priors. However, the performance of such adaptations is severely limited by the scarcity of omnidirectional stereo datasets an...
|
||||
|
||||
#### 2. [2605.08592](https://arxiv.org/abs/2605.08592) — Cross-Modal RGB-D Fusion Transformer for 6D Pose Estimation of Non-Cooperative Spacecraft with Stereo-Derived Depth
|
||||
- **发表**: 2026-05-09 | **分类**: cs.CV
|
||||
- **作者**: Yongliang Zhen, Bo LÜ, Hang Yang, Xiaotian WU
|
||||
- **摘要**: On-orbit servicing and active debris removal involving non-cooperative spacecraft require reliable pose estimation to supply accurate position and orientation data for autonomous visual navigation. Learning-based monocular methods have seen widespread adoption in spacecraft pose estimation, yet they suffer from an intrinsic depth ambiguity problem and tend to fail under the harsh illumination cond...
|
||||
|
||||
#### 3. [2604.20393](https://arxiv.org/abs/2604.20393) — MLG-Stereo: ViT Based Stereo Matching with Multi-Stage Local-Global Enhancement
|
||||
- **发表**: 2026-04-22 | **分类**: cs.CV
|
||||
- **作者**: Haoyu Zhang, Jingyi Zhou, Peng Ye, Jiakang Yuan 等 (7 人)
|
||||
- **摘要**: With the development of deep learning, ViT-based stereo matching methods have made significant progress due to their remarkable robustness and zero-shot ability. However, due to the limitations of ViTs in handling resolution sensitivity and their relative neglect of local information, the ability of ViT-based methods to predict details and handle arbitrary-resolution images is still weaker than th...
|
||||
|
||||
#### 4. [2604.10218](https://arxiv.org/abs/2604.10218) — SMFormer: Empowering Self-supervised Stereo Matching via Foundation Models and Data Augmentation
|
||||
- **发表**: 2026-04-11 | **分类**: cs.CV
|
||||
- **作者**: Yun Wang, Zhengjie Yang, Jiahao Zheng, Zhanjie Zhang 等 (6 人)
|
||||
- **摘要**: Recent self-supervised stereo matching methods have made significant progress. They typically rely on the photometric consistency assumption, which presumes corresponding points across views share the same appearance. However, this assumption could be compromised by real-world disturbances, resulting in invalid supervisory signals and a significant accuracy gap compared to supervised methods. To a...
|
||||
|
||||
#### 5. [2604.09142](https://arxiv.org/abs/2604.09142) — Geometry Reinforced Efficient Attention Tuning Equipped with Normals for Robust Stereo Matching
|
||||
- **发表**: 2026-04-10 | **分类**: cs.CV
|
||||
- **作者**: Jiahao Li, Xinhong Chen, Zhengmin Jiang, Cheng Huang 等 (6 人)
|
||||
- **摘要**: Despite remarkable advances in image-driven stereo matching over the past decade, Synthetic-to-Realistic Zero-Shot (Syn-to-Real) generalization remains an open challenge. This suboptimal generalization performance mainly stems from cross-domain shifts and ill-posed ambiguities inherent in image textures, particularly in occluded, textureless, repetitive, and non-Lambertian (specular/transparent) r...
|
||||
|
||||
#### 6. [2603.29368](https://arxiv.org/abs/2603.29368) — StereoVGGT: A Training-Free Visual Geometry Transformer for Stereo Vision
|
||||
- **发表**: 2026-03-31 | **分类**: cs.CV
|
||||
- **作者**: Ziyang Chen, Yansong Qu, You Shen, Xuan Cheng 等 (5 人)
|
||||
- **摘要**: Driven by the advancement of 3D devices, stereo vision tasks including stereo matching and stereo conversion have emerged as a critical research frontier. Contemporary stereo vision backbones typically rely on either monocular depth estimation (MDE) models or visual foundation models (VFMs). Crucially, these models are predominantly pretrained without explicit supervision of camera poses. Given th...
|
||||
|
||||
#### 7. [2603.24836](https://arxiv.org/abs/2603.24836) — WAFT-Stereo: Warping-Alone Field Transforms for Stereo Matching
|
||||
- **发表**: 2026-03-25 | **分类**: cs.CV
|
||||
- **作者**: Yihan Wang, Jia Deng
|
||||
- **摘要**: We introduce WAFT-Stereo, a simple and effective warping-based method for stereo matching. WAFT-Stereo demonstrates that cost volumes, a common design used in many leading methods, are not necessary for strong performance and can be replaced by warping with improved efficiency. WAFT-Stereo ranks first on ETH3D (BP-0.5), Middlebury (RMSE), and KITTI (all metrics), reducing the zero-shot error by 81...
|
||||
|
||||
#### 8. [2603.21882](https://arxiv.org/abs/2603.21882) — Deep S2P: Integrating Learning Based Stereo Matching Into the Satellite Stereo Pipeline
|
||||
- **发表**: 2026-03-23 | **分类**: cs.CV
|
||||
- **作者**: Elías Masquil, Thibaud Ehret, Pablo Musé, Gabriele Facciolo
|
||||
- **摘要**: Digital Surface Model generation from satellite imagery is a core task in Earth observation and is commonly addressed using classical stereoscopic matching algorithms in satellite pipelines as in the Satellite Stereo Pipeline (S2P). While recent learning-based stereo matchers achieve state-of-the-art performance on standard benchmarks, their integration into operational satellite pipelines remains...
|
||||
|
||||
#### 9. [2603.15019](https://arxiv.org/abs/2603.15019) — Reference-Free Omnidirectional Stereo Matching via Multi-View Consistency Maximization
|
||||
- **发表**: 2026-03-16 | **分类**: cs.CV
|
||||
- **作者**: Lehuai Xu, Weiming Zhang, Yang Li, Sidan Du 等 (5 人)
|
||||
- **摘要**: Reliable omnidirectional depth estimation from multi-fisheye stereo matching is pivotal to many applications, such as embodied robotics. Existing approaches either rely on spherical sweeping with heuristic fusion strategies to build the cost columns or perform reference-centric stereo matching based on rectified views. However, these methods fail to explicitly exploit geometric relationships betwe...
|
||||
|
||||
#### 10. [2603.01650](https://arxiv.org/abs/2603.01650) — PromptStereo: Zero-Shot Stereo Matching via Structure and Motion Prompts
|
||||
- **发表**: 2026-03-02 | **分类**: cs.CV
|
||||
- **作者**: Xianqi Wang, Hao Yang, Hangtian Wang, Junda Cheng 等 (7 人)
|
||||
- **摘要**: Modern stereo matching methods have leveraged monocular depth foundation models to achieve superior zero-shot generalization performance. However, most existing methods primarily focus on extracting robust features for cost volume construction or disparity initialization. At the same time, the iterative refinement stage, which is also crucial for zero-shot generalization, remains underexplored. So...
|
||||
|
||||
---
|
||||
|
||||
## B. 视觉惯性 SLAM / VIO
|
||||
**项目阶段**: M2-1 / M3-3 | **论文数**: 10
|
||||
|
||||
对标 ZED 内建 VIO 的替代方案。关注 IMU 融合、长时鲁棒、动态环境。
|
||||
|
||||
#### 1. [2605.07552](https://arxiv.org/abs/2605.07552) — VIMCAN: Visual-Inertial 3D Human Pose Estimation with Hybrid Mamba-Cross-Attention Network
|
||||
- **发表**: 2026-05-08 | **分类**: cs.CV
|
||||
- **作者**: Zepeng Yang, Junxuan Bai, Hao Li, Ju Dai 等 (7 人)
|
||||
- **摘要**: The rapid advances in deep learning have significantly enhanced the accuracy of multimodal 3D human pose estimation (HPE). However, the state-of-the-art (SOTA) HPE pipelines still rely on Transformers, whose quadratic complexity makes real-time processing for long sequences impractical. Mamba addresses this issue through selective state-space modeling, enabling efficient sequence processing withou...
|
||||
|
||||
#### 2. [2605.02054](https://arxiv.org/abs/2605.02054) — Observability Conditions and Filter Design for Visual Pose Estimation via Dual Quaternions
|
||||
- **发表**: 2026-05-03 | **分类**: eess.SY, cs.CV, cs.RO
|
||||
- **作者**: Nicholas B. Andrews, Kristi A. Morgansen
|
||||
- **摘要**: This paper presents a dual quaternion framework for 6-DOF visual target tracking that addresses key limitations of perspective-n-point (P$n$P) solvers: sensitivity to noise and outliers, and inability to propagate estimates through measurement dropouts. A nonlinear observability analysis is performed using a Lie algebraic approach, deriving sufficient conditions for local observability under two s...
|
||||
|
||||
#### 3. [2604.07151](https://arxiv.org/abs/2604.07151) — An RTK-SLAM Dataset for Absolute Accuracy Evaluation in GNSS-Degraded Environments
|
||||
- **发表**: 2026-04-08 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Wei Zhang, Vincent Ress, David Skuddis, Uwe Soergel 等 (5 人)
|
||||
- **摘要**: RTK-SLAM systems integrate simultaneous localization and mapping (SLAM) with real-time kinematic (RTK) GNSS positioning, promising both relative consistency and globally referenced coordinates for efficient georeferenced surveying. A critical and underappreciated issue is that the standard evaluation metric, Absolute Trajectory Error (ATE), first fits an optimal rigid-body transformation between t...
|
||||
|
||||
#### 4. [2603.21785](https://arxiv.org/abs/2603.21785) — Image-Conditioned Adaptive Parameter Tuning for Visual Odometry Frontends
|
||||
- **发表**: 2026-03-23 | **分类**: cs.CV
|
||||
- **作者**: Simone Nascivera, Leonard Bauersfeld, Jeff Delaune, Davide Scaramuzza
|
||||
- **摘要**: Resource-constrained autonomous robots rely on sparse direct and semi-direct visual-(inertial)-odometry (VO) pipelines, as they provide a favorable tradeoff between accuracy, robustness, and computational cost. However, the performance of most systems depends critically on hand-tuned hyperparameters governing feature detection, tracking, and outlier rejection. These parameters are typically fixed ...
|
||||
|
||||
#### 5. [2603.20778](https://arxiv.org/abs/2603.20778) — PiLoT: Neural Pixel-to-3D Registration for UAV-based Ego and Target Geo-localization
|
||||
- **发表**: 2026-03-21 | **分类**: cs.CV
|
||||
- **作者**: Xiaoya Cheng, Long Wang, Yan Liu, Xinyi Liu 等 (8 人)
|
||||
- **摘要**: We present PiLoT, a unified framework that tackles UAV-based ego and target geo-localization. Conventional approaches rely on decoupled pipelines that fuse GNSS and Visual-Inertial Odometry (VIO) for ego-pose estimation, and active sensors like laser rangefinders for target localization. However, these methods are susceptible to failure in GNSS-denied environments and incur substantial hardware co...
|
||||
|
||||
#### 6. [2603.19654](https://arxiv.org/abs/2603.19654) — GravCal: Single-Image Calibration of IMU Gravity Priors with Per-Sample Confidence
|
||||
- **发表**: 2026-03-20 | **分类**: cs.CV
|
||||
- **作者**: Haichao Zhu, Qian Zhang
|
||||
- **摘要**: Gravity estimation is fundamental to visual-inertial perception, augmented reality, and robotics, yet gravity priors from IMUs are often unreliable under linear acceleration, vibration, and transient motion. Existing methods often estimate gravity directly from images or assume reasonably accurate inertial input, leaving the practical problem of correcting a noisy gravity prior from a single image...
|
||||
|
||||
#### 7. [2603.17229](https://arxiv.org/abs/2603.17229) — Visual SLAM with DEM Anchoring for Lunar Surface Navigation
|
||||
- **发表**: 2026-03-18 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Adam Dai, Guillem Casadesus Vila, Grace Gao
|
||||
- **摘要**: Future lunar missions will require autonomous rovers capable of traversing tens of kilometers across challenging terrain while maintaining accurate localization and producing globally consistent maps. However, the absence of global positioning systems, extreme illumination, and low-texture regolith make long-range navigation on the Moon particularly difficult, as visual-inertial odometry pipelines...
|
||||
|
||||
#### 8. [2603.26685](https://arxiv.org/abs/2603.26685) — Contextual Graph Representations for Task-Driven 3D Perception and Planning
|
||||
- **发表**: 2026-03-12 | **分类**: cs.RO, cs.AI, cs.CV
|
||||
- **作者**: Christopher Agia
|
||||
- **摘要**: Recent advances in computer vision facilitate fully automatic extraction of object-centric relational representations from visual-inertial data. These state representations, dubbed 3D scene graphs, are a hierarchical decomposition of real-world scenes with a dense multiplex graph structure. While 3D scene graphs claim to promote efficient task planning for robot systems, they contain numerous obje...
|
||||
|
||||
#### 9. [2603.11085](https://arxiv.org/abs/2603.11085) — Edge-Assisted Multi-Robot Visual-Inertial SLAM with Efficient Communication
|
||||
- **发表**: 2026-03-11 | **分类**: cs.RO, cs.CV, cs.MA
|
||||
- **作者**: Xin Liu, Shuhuan Wen, Jing Zhao, Tony Z. Qiu 等 (5 人)
|
||||
- **摘要**: The integration of cloud computing and edge computing is an effective way to achieve global consistent and real-time multi-robot Simultaneous Localization and Mapping (SLAM). Cloud computing effectively solves the problem of limited computing, communication and storage capacity of terminal equipment. However, limited bandwidth and extremely long communication links between terminal devices and the...
|
||||
|
||||
#### 10. [2603.09653](https://arxiv.org/abs/2603.09653) — OTPL-VIO: Robust Visual-Inertial Odometry with Optimal Transport Line Association and Adaptive Uncertainty
|
||||
- **发表**: 2026-03-10 | **分类**: cs.CV, cs.RO
|
||||
- **作者**: Zikun Chen, Wentao Zhao, Yihe Niu, Tianchen Deng 等 (5 人)
|
||||
- **摘要**: Robust stereo visual-inertial odometry (VIO) remains challenging in low-texture scenes and under abrupt illumination changes, where point features become sparse and unstable, leading to ambiguous association and under-constrained estimation. Line structures offer complementary geometric cues, yet many efficient point-line systems still rely on point-guided line association, which can break down wh...
|
||||
|
||||
---
|
||||
|
||||
## C. 3D Gaussian Splatting SLAM（融合建图）
|
||||
**项目阶段**: M3-5 / M4 | **论文数**: 10
|
||||
|
||||
把 3DGS 作为 SLAM 后端，实现实时定位+建图+渲染一体化。world model 训练的核心视觉表征。
|
||||
|
||||
#### 1. [2605.10760](https://arxiv.org/abs/2605.10760) — MAGS-SLAM: Monocular Multi-Agent Gaussian Splatting SLAM for Geometrically and Photometrically Consistent Reconstruction
|
||||
- **发表**: 2026-05-11 | **分类**: cs.RO
|
||||
- **作者**: Zhihao Cao, Qi Shao, Shuhao Zhai, Jing Zhang 等 (6 人)
|
||||
- **摘要**: Collaborative photorealistic 3D reconstruction from multiple agents enables rapid large-scale scene capture for virtual production and cooperative multi-robot exploration. While recent 3D Gaussian Splatting (3DGS) SLAM algorithms can generate high-fidelity real-time mapping, most of the existing multi-agent Gaussian SLAM methods still rely on RGB-D sensors to obtain metric depth and simplify cross...
|
||||
|
||||
#### 2. [2604.22339](https://arxiv.org/abs/2604.22339) — Flow4DGS-SLAM: Optical Flow-Guided 4D Gaussian Splatting SLAM
|
||||
- **发表**: 2026-04-24 | **分类**: cs.CV
|
||||
- **作者**: Yunsong Wang, Gim Hee Lee
|
||||
- **摘要**: Handling the dynamic environments is a significant research challenge in Visual Simultaneous Localization and Mapping (SLAM). Recent research combines 3D Gaussian Splatting (3DGS) with SLAM to achieve both robust camera pose estimation and photorealistic renderings. However, using SLAM to efficiently reconstruct both static and dynamic regions remains challenging. In this work, we propose an effic...
|
||||
|
||||
#### 3. [2604.15612](https://arxiv.org/abs/2604.15612) — GaussianFlow SLAM: Monocular Gaussian Splatting SLAM Guided by GaussianFlow
|
||||
- **发表**: 2026-04-17 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Dong-Uk Seo, Jinwoo Jeon, Eungchang Mason Lee, Hyun Myung
|
||||
- **摘要**: Gaussian splatting has recently gained traction as a compelling map representation for SLAM systems, enabling dense and photo-realistic scene modeling. However, its application to monocular SLAM remains challenging due to the lack of reliable geometric cues from monocular input. Without geometric supervision, mapping or tracking could fall in local-minima, resulting in structural degeneracies and ...
|
||||
|
||||
#### 4. [2604.13492](https://arxiv.org/abs/2604.13492) — RadarSplat-RIO: Indoor Radar-Inertial Odometry with Gaussian Splatting-Based Radar Bundle Adjustment
|
||||
- **发表**: 2026-04-15 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Pou-Chun Kung, Yuan Tian, Zhengqin Li, Yue Liu 等 (7 人)
|
||||
- **摘要**: Radar is more resilient to adverse weather and lighting conditions than visual and Lidar simultaneous localization and mapping (SLAM). However, most radar SLAM pipelines still rely heavily on frame-to-frame odometry, which leads to substantial drift. While loop closure can correct long-term errors, it requires revisiting places and relies on robust place recognition. In contrast, visual odometry m...
|
||||
|
||||
#### 5. [2604.12942](https://arxiv.org/abs/2604.12942) — RMGS-SLAM: Real-time Multi-sensor Gaussian Splatting SLAM
|
||||
- **发表**: 2026-04-14 | **分类**: cs.RO
|
||||
- **作者**: Dongen Li, Yi Liu, Junqi Liu, Zewen Sun 等 (11 人)
|
||||
- **摘要**: Achieving real-time Simultaneous Localization and Mapping (SLAM) based on 3D Gaussian splatting (3DGS) in large-scale real-world environments remains challenging, as existing methods still struggle to jointly achieve low-latency pose estimation, continuous 3D Gaussian reconstruction, and long-term global consistency. In this paper, we present a tightly coupled LiDAR-Inertial-Visual 3DGS-based SLAM...
|
||||
|
||||
#### 6. [2604.12837](https://arxiv.org/abs/2604.12837) — GGD-SLAM: Monocular 3DGS SLAM Powered by Generalizable Motion Model for Dynamic Environments
|
||||
- **发表**: 2026-04-14 | **分类**: cs.RO
|
||||
- **作者**: Yi Liu, Haoxuan Xu, Hongbo Duan, Keyu Fan 等 (8 人)
|
||||
- **摘要**: Visual SLAM algorithms achieve significant improvements through the exploration of 3D Gaussian Splatting (3DGS) representations, particularly in generating high-fidelity dense maps. However, they depend on a static environment assumption and experience significant performance degradation in dynamic environments. This paper presents GGD-SLAM, a framework that employs a generalizable motion model to...
|
||||
|
||||
#### 7. [2604.11992](https://arxiv.org/abs/2604.11992) — ReefMapGS: Enabling Large-Scale Underwater Reconstruction by Closing the Loop Between Multimodal SLAM and Gaussian Splatting
|
||||
- **发表**: 2026-04-13 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Daniel Yang, Jungseok Hong, John J. Leonard, Yogesh Girdhar
|
||||
- **摘要**: 3D Gaussian Splatting is a powerful visual representation, providing high-quality and efficient 3D scene reconstruction, but it is crucially dependent on accurate camera poses typically obtained from computationally intensive processes like structure-from-motion that are unsuitable for field robot applications. However, in these domains, multimodal sensor data from acoustic, inertial, pressure, an...
|
||||
|
||||
#### 8. [2604.10593](https://arxiv.org/abs/2604.10593) — MonoEM-GS: Monocular Expectation-Maximization Gaussian Splatting SLAM
|
||||
- **发表**: 2026-04-12 | **分类**: cs.RO
|
||||
- **作者**: Evgenii Kruzhkov, Sven Behnke
|
||||
- **摘要**: Feed-forward geometric foundation models can infer dense point clouds and camera motion directly from RGB streams, providing priors for monocular SLAM. However, their predictions are often view-dependent and noisy: geometry can vary across viewpoints and under image transformations, and local metric properties may drift between frames. We present MonoEM-GS, a monocular mapping pipeline that integr...
|
||||
|
||||
#### 9. [2604.03092](https://arxiv.org/abs/2604.03092) — Flash-Mono: Feed-Forward Accelerated Gaussian Splatting Monocular SLAM
|
||||
- **发表**: 2026-04-03 | **分类**: cs.RO
|
||||
- **作者**: Zicheng Zhang, Ke Wu, Xiangting Meng, Keyu Liu 等 (6 人)
|
||||
- **摘要**: Monocular 3D Gaussian Splatting SLAM suffers from critical limitations in time efficiency, geometric accuracy, and multi-view consistency. These issues stem from the time-consuming $\textit{Train-from-Scratch}$ optimization and the lack of inter-frame scale consistency from single-frame geometry priors. We contend that a feed-forward paradigm, leveraging multi-frame context to predict Gaussian att...
|
||||
|
||||
#### 10. [2604.02696](https://arxiv.org/abs/2604.02696) — VBGS-SLAM: Variational Bayesian Gaussian Splatting Simultaneous Localization and Mapping
|
||||
- **发表**: 2026-04-03 | **分类**: cs.CV, cs.RO
|
||||
- **作者**: Yuhan Zhu, Yanyu Zhang, Jie Xu, Wei Ren
|
||||
- **摘要**: 3D Gaussian Splatting (3DGS) has shown promising results for 3D scene modeling using mixtures of Gaussians, yet its existing simultaneous localization and mapping (SLAM) variants typically rely on direct, deterministic pose optimization against the splat map, making them sensitive to initialization and susceptible to catastrophic forgetting as map evolves. We propose Variational Bayesian Gaussian ...
|
||||
|
||||
---
|
||||
|
||||
## D. 单目深度基础模型
|
||||
**项目阶段**: M3-4 | **论文数**: 10
|
||||
|
||||
Depth Anything / Marigold / Metric3D / UniDepth 等通用深度模型，作为双目深度失效的兜底。
|
||||
|
||||
#### 1. [2605.11756](https://arxiv.org/abs/2605.11756) — Focusable Monocular Depth Estimation
|
||||
- **发表**: 2026-05-12 | **分类**: cs.CV, cs.AI
|
||||
- **作者**: Yuxin Du, Tao Lin, Zile Zhong, Runting Li 等 (10 人)
|
||||
- **摘要**: Monocular depth foundation models generalize well across scenes, yet they are typically optimized with uniform pixel-wise objectives that do not distinguish user-specified or task-relevant target regions from the surrounding context. We therefore introduce Focusable Monocular Depth Estimation (FDE), a region-aware depth estimation task in which, given a specified target region, the model is requir...
|
||||
|
||||
#### 2. [2605.07264](https://arxiv.org/abs/2605.07264) — Sat3R: Satellite DSM Reconstruction via RPC-Aware Depth Fine-tuning
|
||||
- **发表**: 2026-05-08 | **分类**: cs.CV
|
||||
- **作者**: Qiaoyi Yang, Chaoyi Zhou, Xi Liu, Run Wang 等 (12 人)
|
||||
- **摘要**: Accurate Digital Surface Model (DSM) reconstruction from satellite imagery is critical for applications such as disaster response, urban planning, and large-scale geographic mapping. Existing approaches face a fundamental trade-off: optimization-based methods achieve strong accuracy but require hours of per-scene computation, while generalizable geometry foundation models offer near-instant infere...
|
||||
|
||||
#### 3. [2605.06270](https://arxiv.org/abs/2605.06270) — Spark3R: Asymmetric Token Reduction Makes Fast Feed-Forward 3D Reconstruction
|
||||
- **发表**: 2026-05-07 | **分类**: cs.CV
|
||||
- **作者**: Zecheng Tang, Jiaye Fu, Qiankun Gao, Haijie Li 等 (8 人)
|
||||
- **摘要**: Feed-forward 3D reconstruction models based on Vision Transformers can directly estimate scene geometry and camera poses from a small set of input images, but scaling them to video inputs with hundreds or thousands of frames remains challenging due to the quadratic cost of global attention layers. Recent token-merging methods accelerate these models by compressing the token sequence within the glo...
|
||||
|
||||
#### 4. [2605.04566](https://arxiv.org/abs/2605.04566) — Open-Source Image Editing Models Are Zero-Shot Vision Learners
|
||||
- **发表**: 2026-05-06 | **分类**: cs.CV, cs.CL
|
||||
- **作者**: Wei Liu, Jiaxin Lin, Rui Chen
|
||||
- **摘要**: Recent studies have shown that large generative models can solve vision tasks they were not explicitly trained for. However, existing evidence relies on closed-source models~(Veo~3, Nano Banana Pro) or requires task-specific instruction tuning, leaving open whether publicly available image-editing models possess zero-shot vision abilities out of the box. We conduct a systematic evaluation of thr...
|
||||
|
||||
#### 5. [2604.26567](https://arxiv.org/abs/2604.26567) — AirZoo: A Unified Large-Scale Dataset for Grounding Aerial Geometric 3D Vision
|
||||
- **发表**: 2026-04-29 | **分类**: cs.CV
|
||||
- **作者**: Xiaoya Cheng, Rouwan Wu, Xinyi Liu, Zeyu Cui 等 (9 人)
|
||||
- **摘要**: Despite the rapid progress in data-driven 3D vision, aerial geometric 3D vision remains a formidable challenge due to the severe scarcity of large-scale, high-fidelity training data. Existing benchmarks, predominantly biased toward ground-level or object-centric views, do not account for complex viewpoint transformations and diverse environmental conditions in UAV-based sensing. To bridge this cri...
|
||||
|
||||
#### 6. [2604.23432](https://arxiv.org/abs/2604.23432) — Sphere-Depth: A Benchmark for Depth Estimation Methods with Varying Spherical Camera Orientations
|
||||
- **发表**: 2026-04-25 | **分类**: cs.CV, cs.AI
|
||||
- **作者**: Soulayma Gazzeh, Giuseppe Mazzola, Liliana Lo Presti, Marco La Cascia
|
||||
- **摘要**: Reliable depth estimation from spherical images is crucial for 360° vision in robotic navigation and immersive scene understanding. However, the onboard spherical camera can experience unintentional pose variations in real-world robotic platforms that, along with the geometric distortions inherent in equirectangular projections, significantly impact the effectiveness of depth estimation. To study ...
|
||||
|
||||
#### 7. [2604.20329](https://arxiv.org/abs/2604.20329) — Image Generators are Generalist Vision Learners
|
||||
- **发表**: 2026-04-22 | **分类**: cs.CV, cs.AI
|
||||
- **作者**: Valentin Gabeur, Shangbang Long, Songyou Peng, Paul Voigtlaender 等 (25 人)
|
||||
- **摘要**: Recent works show that image and video generators exhibit zero-shot visual understanding behaviors, in a way reminiscent of how LLMs develop emergent capabilities of language understanding and reasoning from generative pretraining. While it has long been conjectured that the ability to create visual content implies an ability to understand it, there has been limited evidence that generative vision...
|
||||
|
||||
#### 8. [2604.18336](https://arxiv.org/abs/2604.18336) — Enhancing Glass Surface Reconstruction via Depth Prior for Robot Navigation
|
||||
- **发表**: 2026-04-20 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Jiamin Zheng, Jingwen Yu, Guangcheng Chen, Hong Zhang
|
||||
- **摘要**: Indoor robot navigation is often compromised by glass surfaces, which severely corrupt depth sensor measurements. While foundation models like Depth Anything 3 provide excellent geometric priors, they lack an absolute metric scale. We propose a training-free framework that leverages depth foundation models as a structural prior, employing a robust local RANSAC-based alignment to fuse it with raw s...
|
||||
|
||||
#### 9. [2604.17231](https://arxiv.org/abs/2604.17231) — Fringe Projection Based Vision Pipeline for Autonomous Hard Drive Disassembly
|
||||
- **发表**: 2026-04-19 | **分类**: cs.CV, cs.RO
|
||||
- **作者**: Badrinath Balasubramaniam, Vignesh Suresh, Benjamin Metcalf, Beiwen Li
|
||||
- **摘要**: Unrecovered e-waste represents a significant economic loss. Hard disk drives (HDDs) comprise a valuable e-waste stream necessitating robotic disassembly. Automating the disassembly of HDDs requires holistic 3D sensing, scene understanding, and fastener localization, however current methods are fragmented, lack robust 3D sensing, and lack fastener localization. We propose an autonomous vision pipel...
|
||||
|
||||
#### 10. [2604.14048](https://arxiv.org/abs/2604.14048) — Free Geometry: Refining 3D Reconstruction from Longer Versions of Itself
|
||||
- **发表**: 2026-04-15 | **分类**: cs.CV
|
||||
- **作者**: Yuhang Dai, Xingyi Yang
|
||||
- **摘要**: Feed-forward 3D reconstruction models are efficient but rigid: once trained, they perform inference in a zero-shot manner and cannot adapt to the test scene. As a result, visually plausible reconstructions often contain errors, particularly under occlusions, specularities, and ambiguous cues. To address this, we introduce Free Geometry, a framework that enables feed-forward 3D reconstruction model...
|
||||
|
||||
---
|
||||
|
||||
## E. 室内 RGB-D 数据集与重建
|
||||
**项目阶段**: M4-1 / M4-2 | **论文数**: 10
|
||||
|
||||
可参考的数据集设计、评测基准、室内几何重建方法。
|
||||
|
||||
#### 1. [2605.09231](https://arxiv.org/abs/2605.09231) — An Elastic Shape Variational Autoencoder for Skeleton Pose Trajectories
|
||||
- **发表**: 2026-05-10 | **分类**: cs.CV, stat.ML
|
||||
- **作者**: Arafat Rahman, Shashwat Kumar, Laura E. Barnes, Anuj Srivastava
|
||||
- **摘要**: Deep generative models provide flexible frameworks for modeling complex, structured data such as images, videos, 3D objects, and texts. However, when applied to sequences of human skeletons, standard variational autoencoders (VAEs) often allocate substantial capacity to nuisance factors-such as camera orientation, subject scale, viewpoint, and execution speed-rather than the intrinsic geometry of ...
|
||||
|
||||
#### 2. [2605.03463](https://arxiv.org/abs/2605.03463) — First Shape, Then Meaning: Efficient Geometry and Semantics Learning for Indoor Reconstruction
|
||||
- **发表**: 2026-05-05 | **分类**: cs.CV
|
||||
- **作者**: Remi Chierchia, Léo Lebrat, David Ahmedt-Aristizabal, Olivier Salvado 等 (6 人)
|
||||
- **摘要**: Neural Surface Reconstruction has become a standard methodology for indoor 3D reconstruction, with Signed Distance Functions (SDFs) proving particularly effective for representing scene geometry. A variety of applications require a detailed understanding of the scene context, driving the need for object-level semantic signals. While recent methods successfully integrate semantic labels, they often...
|
||||
|
||||
#### 3. [2604.21400](https://arxiv.org/abs/2604.21400) — You Only Gaussian Once: Controllable 3D Gaussian Splatting for Ultra-Densely Sampled Scenes
|
||||
- **发表**: 2026-04-23 | **分类**: cs.CV
|
||||
- **作者**: Jinrang Jia, Zhenjia Li, Yifeng Shi
|
||||
- **摘要**: 3D Gaussian Splatting (3DGS) has revolutionized neural rendering, yet existing methods remain predominantly research prototypes ill-suited for production-level deployment. We identify a critical "Industry-Academia Gap" hindering real-world application: unpredictable resource consumption from heuristic Gaussian growth, the "sparsity shield" of current benchmarks that rewards hallucination over phys...
|
||||
|
||||
#### 4. [2604.18336](https://arxiv.org/abs/2604.18336) — Enhancing Glass Surface Reconstruction via Depth Prior for Robot Navigation
|
||||
- **发表**: 2026-04-20 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Jiamin Zheng, Jingwen Yu, Guangcheng Chen, Hong Zhang
|
||||
- **摘要**: Indoor robot navigation is often compromised by glass surfaces, which severely corrupt depth sensor measurements. While foundation models like Depth Anything 3 provide excellent geometric priors, they lack an absolute metric scale. We propose a training-free framework that leverages depth foundation models as a structural prior, employing a robust local RANSAC-based alignment to fuse it with raw s...
|
||||
|
||||
#### 5. [2604.01605](https://arxiv.org/abs/2604.01605) — F3DGS: Federated 3D Gaussian Splatting for Decentralized Multi-Agent World Modeling
|
||||
- **发表**: 2026-04-02 | **分类**: cs.CV, cs.RO
|
||||
- **作者**: Morui Zhu, Mohammad Dehghani Tezerjani, Mátyás Szántó, Márton Vaitkus 等 (6 人)
|
||||
- **摘要**: We present F3DGS, a federated 3D Gaussian Splatting framework for decentralized multi-agent 3D reconstruction. Existing 3DGS pipelines assume centralized access to all observations, which limits their applicability in distributed robotic settings where agents operate independently, and centralized data aggregation may be restricted. Directly extending centralized training to multi-agent systems in...
|
||||
|
||||
#### 6. [2603.26690](https://arxiv.org/abs/2603.26690) — SpatialPoint: Spatial-aware Point Prediction for Embodied Localization
|
||||
- **发表**: 2026-03-16 | **分类**: cs.RO, cs.AI, cs.CV
|
||||
- **作者**: Qiming Zhu, Zhirui Fang, Tianming Zhang, Chuanxiu Liu 等 (6 人)
|
||||
- **摘要**: Embodied intelligence fundamentally requires a capability to determine where to act in 3D space. We formalize this requirement as embodied localization -- the problem of predicting executable 3D points conditioned on visual observations and language instructions. We instantiate embodied localization with two complementary target types: touchable points, surface-grounded 3D points enabling direct p...
|
||||
|
||||
#### 7. [2603.04254](https://arxiv.org/abs/2603.04254) — EmbodiedSplat: Online Feed-Forward Semantic 3DGS for Open-Vocabulary 3D Scene Understanding
|
||||
- **发表**: 2026-03-04 | **分类**: cs.CV
|
||||
- **作者**: Seungjun Lee, Zihan Wang, Yunsong Wang, Gim Hee Lee
|
||||
- **摘要**: Understanding a 3D scene immediately with its exploration is essential for embodied tasks, where an agent must construct and comprehend the 3D scene in an online and nearly real-time manner. In this study, we propose EmbodiedSplat, an online feed-forward 3DGS for open-vocabulary scene understanding that enables simultaneous online 3D reconstruction and 3D semantic understanding from the streaming ...
|
||||
|
||||
#### 8. [2512.12683](https://arxiv.org/abs/2512.12683) — Quantum Implicit Neural Representations for 3D Scene Reconstruction and Novel View Synthesis
|
||||
- **发表**: 2025-12-14 | **分类**: quant-ph, cs.AI, cs.CV
|
||||
- **作者**: Yeray Cordero, Paula García-Molina, Fernando Vilariño
|
||||
- **摘要**: Implicit neural representations (INRs) have become a powerful paradigm for continuous signal modeling and 3D scene reconstruction, yet classical networks suffer from a well-known spectral bias that limits their ability to capture high-frequency details. Quantum Implicit Representation Networks (QIREN) mitigate this limitation by employing parameterized quantum circuits with inherent Fourier struct...
|
||||
|
||||
#### 9. [2511.07412](https://arxiv.org/abs/2511.07412) — TwinOR: Photorealistic Digital Twins of Dynamic Operating Rooms for Embodied AI Research 🇨🇳
|
||||
- **发表**: 2025-11-10 | **分类**: cs.CV, cs.RO
|
||||
- **作者**: Han Zhang, Yiqing Shen, Roger D. Soberanis-Mukul, Ankita Ghosh 等 (14 人)
|
||||
- **摘要**: Developing embodied AI for intelligent surgical systems requires safe, controllable environments for continual learning and evaluation. However, safety regulations and operational constraints in operating rooms (ORs) limit agents from freely perceiving and interacting in realistic settings. Digital twins provide high-fidelity, risk-free environments for exploration and training. How we may create ...
|
||||
|
||||
#### 10. [2510.12387](https://arxiv.org/abs/2510.12387) — Scene Coordinate Reconstruction Priors
|
||||
- **发表**: 2025-10-14 | **分类**: cs.CV
|
||||
- **作者**: Wenjing Bian, Axel Barroso-Laguna, Tommaso Cavallari, Victor Adrian Prisacariu 等 (5 人)
|
||||
- **摘要**: Scene coordinate regression (SCR) models have proven to be powerful implicit scene representations for 3D vision, enabling visual relocalization and structure-from-motion. SCR models are trained specifically for one scene. If training images imply insufficient multi-view constraints SCR models degenerate. We present a probabilistic reinterpretation of training SCR models, which allows us to infuse...
|
||||
|
||||
---
|
||||
|
||||
## F. 视频世界模型（下游应用）
|
||||
**项目阶段**: M4-4 | **论文数**: 10
|
||||
|
||||
本项目数据 pipeline 的最终下游：训练能预测未来视频/动作的 world model。
|
||||
|
||||
#### 1. [2605.15185](https://arxiv.org/abs/2605.15185) — Quantitative Video World Model Evaluation for Geometric-Consistency
|
||||
- **发表**: 2026-05-14 | **分类**: cs.CV, cs.AI
|
||||
- **作者**: Jiaxin Wu, Yihao Pi, Yinling Zhang, Yuheng Li 等 (5 人)
|
||||
- **摘要**: Generative video models are increasingly studied as implicit world models, yet evaluating whether they produce physically plausible 3D structure and motion remains challenging. Most existing video evaluation pipelines rely heavily on human judgment or learned graders, which can be subjective and weakly diagnostic for geometric failures. We introduce PDI-Bench (Perspective Distortion Index), a quan...
|
||||
|
||||
#### 2. [2605.15178](https://arxiv.org/abs/2605.15178) — SANA-WM: Efficient Minute-Scale World Modeling with Hybrid Linear Diffusion Transformer
|
||||
- **发表**: 2026-05-14 | **分类**: cs.CV
|
||||
- **作者**: Haoyi Zhu, Haozhe Liu, Yuyang Zhao, Tian Ye 等 (9 人)
|
||||
- **摘要**: We introduce SANA-WM, an efficient 2.6B-parameter open-source world model natively trained for one-minute generation, synthesizing high-fidelity, 720p, minute-scale videos with precise camera control. SANA-WM achieves visual quality comparable to large-scale industrial baselines such as LingBot-World and HY-WorldPlay, while significantly improving efficiency. Four core designs drive our architectu...
|
||||
|
||||
#### 3. [2605.15141](https://arxiv.org/abs/2605.15141) — Causal Forcing++: Scalable Few-Step Autoregressive Diffusion Distillation for Real-Time Interactive Video Generation
|
||||
- **发表**: 2026-05-14 | **分类**: cs.CV
|
||||
- **作者**: Min Zhao, Hongzhou Zhu, Kaiwen Zheng, Zihan Zhou 等 (9 人)
|
||||
- **摘要**: Real-time interactive video generation requires low-latency, streaming, and controllable rollout. Existing autoregressive (AR) diffusion distillation methods have achieved strong results in the chunk-wise 4-step regime by distilling bidirectional base models into few-step AR students, but they remain limited by coarse response granularity and non-negligible sampling latency. In this paper, we stud...
|
||||
|
||||
#### 4. [2605.14937](https://arxiv.org/abs/2605.14937) — Slot-MPC: Goal-Conditioned Model Predictive Control with Object-Centric Representations
|
||||
- **发表**: 2026-05-14 | **分类**: cs.LG, cs.AI, cs.RO
|
||||
- **作者**: Jonathan Spieler, Angel Villar-Corrales, Sven Behnke
|
||||
- **摘要**: Predictive world models enable agents to model scene dynamics and reason about the consequences of their actions. Inspired by human perception, object-centric world models capture scene dynamics using object-level representations, which can be used for downstream applications such as action planning. However, most object-centric world models and reinforcement learning (RL) approaches learn reactiv...
|
||||
|
||||
#### 5. [2605.14851](https://arxiv.org/abs/2605.14851) — IFPV: An Integrated Multi-Agent Framework for Generative Operational Planning and High-Fidelity Plan Verification
|
||||
- **发表**: 2026-05-14 | **分类**: cs.MA, cs.AI
|
||||
- **作者**: Zhigao Huang, Zhengqing Hu, Dong Chen, Shaohan Zhang 等 (8 人)
|
||||
- **摘要**: Operational plan generation and verification are critical for modern complex and rapidly changing battlefield environments, yet traditional generation and verification methods still respectively face the challenges of generation infeasibility and verification insufficiency. To alleviate these limitations, we propose an Integrated Multi-Agent Framework for Generative Operational Planning and High-F...
|
||||
|
||||
#### 6. [2605.14757](https://arxiv.org/abs/2605.14757) — ChannelAgent-Empowered Electromagnetic Space World Model: A Case Study on Agent-Driven Channel Generation for 6G AI-Native Air Interface
|
||||
- **发表**: 2026-05-14 | **分类**: eess.SP
|
||||
- **作者**: Mingyue Li, Li Yu, Yuxiang Zhang, Heng Wang 等 (8 人)
|
||||
- **摘要**: As sixth-generation (6G) wireless networks evolve toward increasingly heterogeneous scenarios, tasks, and service requirements, conventional artificial intelligence (AI) models remain limited in task-aware decision-making and autonomous adaptation. To address this issue, this paper first proposes a ChannelAgent-empowered electromagnetic space world model, in which wireless intelligence is organize...
|
||||
|
||||
#### 7. [2605.14696](https://arxiv.org/abs/2605.14696) — EponaV2: Driving World Model with Comprehensive Future Reasoning
|
||||
- **发表**: 2026-05-14 | **分类**: cs.CV
|
||||
- **作者**: Jiawei Xu, Zhizhou Zhong, Zhijian Shu, Mingkai Jia 等 (11 人)
|
||||
- **摘要**: Data scaling plays a pivotal role in the pursuit of general intelligence. However, the prevailing perception-planning paradigm in autonomous driving relies heavily on expensive manual annotations to supervise trajectory planning, which severely limits its scalability. Conversely, although existing perception-free driving world models achieve impressive driving performance, their real-world reasoni...
|
||||
|
||||
#### 8. [2605.14398](https://arxiv.org/abs/2605.14398) — Coding Agent Is Good As World Simulator
|
||||
- **发表**: 2026-05-14 | **分类**: cs.AI
|
||||
- **作者**: Hongyu Wang, Jingquan Wang, Bocheng Zou, Radu Serban 等 (5 人)
|
||||
- **摘要**: World models have emerged as a powerful paradigm for building interactive simulation environments, with recent video-based approaches demonstrating impressive progress in generating visually plausible dynamics. However, because these models typically infer dynamics from video and represent them in latent states, they do not explicitly enforce physical constraints. As a result, the generated video ...
|
||||
|
||||
#### 9. [2605.14382](https://arxiv.org/abs/2605.14382) — Delta Forcing: Trust Region Steering for Interactive Autoregressive Video Generation
|
||||
- **发表**: 2026-05-14 | **分类**: cs.CV, cs.GR, cs.MM
|
||||
- **作者**: Yuheng Wu, Xiangbo Gao, Tianhao Chen, Xinghao Chen 等 (7 人)
|
||||
- **摘要**: Interactive real-time autoregressive video generation is essential for applications such as content creation and world modeling, where visual content must adapt to dynamically evolving event conditions. A fundamental challenge lies in balancing reactivity and stability: models must respond promptly to new events while maintaining temporal coherence over long horizons. Existing approaches distill b...
|
||||
|
||||
#### 10. [2605.14036](https://arxiv.org/abs/2605.14036) — Enhanced and Efficient Reasoning in Large Learning Models
|
||||
- **发表**: 2026-05-13 | **分类**: cs.AI, cs.CC, cs.CL
|
||||
- **作者**: Leslie G. Valiant
|
||||
- **摘要**: In current Large Language Models we can trust the production of smoothly flowing prose on the basis of the principles of machine learning. However, there is no comparably principled basis to justify trust in the content of the text produced. It appears to be conventional wisdom that addressing this issue by adding more principled reasoning is not computationally affordable. Here we propose a pri...
|
||||
|
||||
---
|
||||
|
||||
## G. ZED 相机相关应用工作
|
||||
**项目阶段**: 全周期 | **论文数**: 10
|
||||
|
||||
用 ZED 系列采集数据的应用论文，参考其采集协议、评测方式、参数配置。
|
||||
|
||||
#### 1. [2602.16385](https://arxiv.org/abs/2602.16385) — Adaptive Multi-Scale Channel-Spatial Attention Aggregation Framework for 3D Indoor Semantic Scene Completion Toward Assisting Visually Impaired
|
||||
- **发表**: 2026-02-18 | **分类**: cs.CV
|
||||
- **作者**: Qi He, XiangXiang Wang, Jingtao Zhang, Yongbin Yu 等 (8 人)
|
||||
- **摘要**: Independent indoor mobility remains a critical challenge for individuals with visual impairments, largely due to the limited capability of existing assistive systems in detecting fine-grained hazardous objects such as chairs, tables, and small obstacles. These perceptual blind zones substantially increase the risk of collision in unfamiliar environments. To bridge the gap between monocular 3D visi...
|
||||
|
||||
#### 2. [2602.09414](https://arxiv.org/abs/2602.09414) — Finite-time Stable Pose Estimation on TSE(3) using Point Cloud and Velocity Sensors
|
||||
- **发表**: 2026-02-10 | **分类**: eess.SY, cs.RO
|
||||
- **作者**: Nazanin S. Hashkavaei, Abhijit Dongare, Neon Srinivasu, Amit K. Sanyal
|
||||
- **摘要**: This work presents a finite-time stable pose estimator (FTS-PE) for rigid bodies undergoing rotational and translational motion in three dimensions, using measurements from onboard sensors that provide position vectors to inertially-fixed points and body velocities. The FTS-PE is a full-state observer for the pose (position and orientation) and velocities and is obtained through a Lyapunov analysi...
|
||||
|
||||
#### 3. [2512.03886](https://arxiv.org/abs/2512.03886) — A Modular Architecture Design for Autonomous Driving Racing in Controlled Environments
|
||||
- **发表**: 2025-12-03 | **分类**: cs.RO, eess.SY
|
||||
- **作者**: Brais Fontan-Costas, M. Diaz-Cacho, Ruben Fernandez-Boullon, Manuel Alonso-Carracedo 等 (5 人)
|
||||
- **摘要**: This paper presents a modular autonomous driving architecture for Formula Student Driverless competition vehicles operating in closed-circuit environments. The perception module employs YOLOv11 for real-time traffic cone detection, achieving 0.93 mAP@0.5 on the FSOCO dataset, combined with neural stereo depth estimation from a ZED 2i camera for 3D cone localization with sub-0.5 m median error at d...
|
||||
|
||||
#### 4. [2512.01108](https://arxiv.org/abs/2512.01108) — Think Fast: Real-Time Kinodynamic Belief-Space Planning for Projectile Interception
|
||||
- **发表**: 2025-11-30 | **分类**: cs.RO
|
||||
- **作者**: Gabriel Olin, Lu Chen, Nayesha Gandotra, Maxim Likhachev 等 (5 人)
|
||||
- **摘要**: Intercepting fast moving objects, by its very nature, is challenging because of its tight time constraints. This problem becomes further complicated in the presence of sensor noise because noisy sensors provide, at best, incomplete information, which results in a distribution over target states to be intercepted. Since time is of the essence, to hit the target, the planner must begin directing the...
|
||||
|
||||
#### 5. [2509.10466](https://arxiv.org/abs/2509.10466) — A Real-Time Diminished Reality Approach to Privacy in MR Collaboration
|
||||
- **发表**: 2025-08-21 | **分类**: cs.CV, cs.HC
|
||||
- **作者**: Christian Fane
|
||||
- **摘要**: Diminished reality (DR) refers to the digital removal of real-world objects by compositing background content in their place. This thesis presents a real-time, inpainting-based DR system designed to enable privacy control in shared-space mixed reality (MR) meetings. The system allows a primary headset user to selectively remove personal or sensitive items from their environment, ensuring that thos...
|
||||
|
||||
#### 6. [2504.06464](https://arxiv.org/abs/2504.06464) — Implementation of a Zed 2i Stereo Camera for High-Frequency Shoreline Change and Coastal Elevation Monitoring
|
||||
- **发表**: 2025-04-08 | **分类**: cs.CV
|
||||
- **作者**: José A. Pilartes-Congo, Matthew Kastl, Michael J. Starek, Marina Vicens-Miquel 等 (5 人)
|
||||
- **摘要**: The increasing population, thus financial interests, in coastal areas have increased the need to monitor coastal elevation and shoreline change. Though several resources exist to obtain this information, they often lack the required temporal resolution for short-term monitoring (e.g., every hour). To address this issue, this study implements a low-cost ZED 2i stereo camera system and close-range p...
|
||||
|
||||
#### 7. [2501.09490](https://arxiv.org/abs/2501.09490) — Comparison of Various SLAM Systems for Mobile Robot in an Indoor Environment
|
||||
- **发表**: 2025-01-16 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Maksim Filipenko, Ilya Afanasyev
|
||||
- **摘要**: This article presents a comparative analysis of a mobile robot trajectories computed by various ROS-based SLAM systems. For this reason we developed a prototype of a mobile robot with common sensors: 2D lidar, a monocular and ZED stereo cameras. Then we conducted experiments in a typical office environment and collected data from all sensors, running all tested SLAM systems based on the acquired d...
|
||||
|
||||
#### 8. [2501.07421](https://arxiv.org/abs/2501.07421) — Empirical Comparison of Four Stereoscopic Depth Sensing Cameras for Robotics Applications
|
||||
- **发表**: 2025-01-13 | **分类**: cs.RO
|
||||
- **作者**: Lukas Rustler, Vojtech Volprecht, Matej Hoffmann
|
||||
- **摘要**: Depth sensing is an essential technology in robotics and many other fields. Many depth sensing (or RGB-D) cameras are available on the market and selecting the best one for your application can be challenging. In this work, we tested four stereoscopic RGB-D cameras that sense the distance by using two images from slightly different views. We empirically compared four cameras (Intel RealSense D435,...
|
||||
|
||||
#### 9. [2410.20599](https://arxiv.org/abs/2410.20599) — Sensor Fusion for Autonomous Indoor UAV Navigation in Confined Spaces
|
||||
- **发表**: 2024-10-27 | **分类**: cs.RO
|
||||
- **作者**: Alice James, Avishkar Seth, Endrowednes Kuantama, Subhas Mukhopadhyay 等 (5 人)
|
||||
- **摘要**: In this paper, we address the challenge of navigating through unknown indoor environments using autonomous aerial robots within confined spaces. The core of our system involves the integration of key sensor technologies, including depth sensing from the ZED 2i camera, IMU data, and LiDAR measurements, facilitated by the Robot Operating System (ROS) and RTAB-Map. Through custom designed experiments...
|
||||
|
||||
#### 10. [2407.18695](https://arxiv.org/abs/2407.18695) — PIV3CAMS: a multi-camera dataset for multiple computer vision problems and its application to novel view-point synthesis
|
||||
- **发表**: 2024-07-26 | **分类**: cs.CV
|
||||
- **作者**: Sohyeong Kim, Martin Danelljan, Radu Timofte, Luc Van Gool 等 (5 人)
|
||||
- **摘要**: The modern approaches for computer vision tasks significantly rely on machine learning, which requires a large number of quality images. While there is a plethora of image datasets with a single type of images, there is a lack of datasets collected from multiple cameras. In this thesis, we introduce Paired Image and Video data from three CAMeraS, namely PIV3CAMS, aimed at multiple computer vision ...
|
||||
|
||||
---
|
||||
|
||||
## H. Orbbec / Femto / Azure Kinect 相关工作
|
||||
**项目阶段**: 国产化 / 替代硬件 | **论文数**: 10
|
||||
|
||||
奥比中光、乐视/微视 Femto、微软 Azure Kinect 等 RGB-D 相机的应用论文。
|
||||
|
||||
#### 1. [2605.06351](https://arxiv.org/abs/2605.06351) — SIGMA-ASL: Sensor-Integrated Multimodal Dataset for Sign Language Recognition
|
||||
- **发表**: 2026-05-07 | **分类**: cs.HC
|
||||
- **作者**: Xiaofang Xiao, Guangchao Li, Guangrong Zhao, Qi Lin 等 (8 人)
|
||||
- **摘要**: Automatic sign language recognition (SLR) has become a key enabler of inclusive human-computer interaction, fostering seamless communication between deaf individuals and hearing communities. Despite significant advances in multimodal learning, existing SLR research remains dominated by vision-based datasets, which are limited by sensitivity to lighting and occlusion, privacy concerns, and a lack o...
|
||||
|
||||
#### 2. [2509.11574](https://arxiv.org/abs/2509.11574) — Gaussian-Plus-SDF SLAM: High-fidelity 3D Reconstruction at 150+ fps
|
||||
- **发表**: 2025-09-15 | **分类**: cs.CV
|
||||
- **作者**: Zhexi Peng, Kun Zhou, Tianjia Shao
|
||||
- **摘要**: While recent Gaussian-based SLAM methods achieve photorealistic reconstruction from RGB-D data, their computational performance remains a critical bottleneck. State-of-the-art techniques operate at less than 20 fps, significantly lagging behind geometry-based approaches like KinectFusion (hundreds of fps). This limitation stems from the heavy computational burden: modeling scenes requires numerous...
|
||||
|
||||
#### 3. [2401.10037](https://arxiv.org/abs/2401.10037) — Depth Over RGB: Automatic Evaluation of Open Surgery Skills Using Depth Camera
|
||||
- **发表**: 2024-01-18 | **分类**: cs.CV
|
||||
- **作者**: Ido Zuckerman, Nicole Werner, Jonathan Kouchly, Emma Huston 等 (7 人)
|
||||
- **摘要**: Purpose: In this paper, we present a novel approach to the automatic evaluation of open surgery skills using depth cameras. This work is intended to show that depth cameras achieve similar results to RGB cameras, which is the common method in the automatic evaluation of open surgery skills. Moreover, depth cameras offer advantages such as robustness to lighting variations, camera positioning, simp...
|
||||
|
||||
#### 4. [2401.08629](https://arxiv.org/abs/2401.08629) — Immature Green Apple Detection and Sizing in Commercial Orchards using YOLOv8 and Shape Fitting Techniques
|
||||
- **发表**: 2023-12-08 | **分类**: cs.CV
|
||||
- **作者**: Ranjan Sapkota, Dawood Ahmed, Martin Churuvija, Manoj Karkee
|
||||
- **摘要**: Detecting and estimating size of apples during the early stages of growth is crucial for predicting yield, pest management, and making informed decisions related to crop-load management, harvest and post-harvest logistics, and marketing. Traditional fruit size measurement methods are laborious and timeconsuming. This study employs the state-of-the-art YOLOv8 object detection and instance segmentat...
|
||||
|
||||
#### 5. [2311.09029](https://arxiv.org/abs/2311.09029) — Self-Annotated 3D Geometric Learning for Smeared Points Removal
|
||||
- **发表**: 2023-11-15 | **分类**: cs.CV
|
||||
- **作者**: Miaowei Wang, Daniel Morris
|
||||
- **摘要**: There has been significant progress in improving the accuracy and quality of consumer-level dense depth sensors. Nevertheless, there remains a common depth pixel artifact which we call smeared points. These are points not on any 3D surface and typically occur as interpolations between foreground and background objects. As they cause fictitious surfaces, these points have the potential to harm appl...
|
||||
|
||||
#### 6. [2306.02263](https://arxiv.org/abs/2306.02263) — MAVD: The First Open Large-Scale Mandarin Audio-Visual Dataset with Depth Information
|
||||
- **发表**: 2023-06-04 | **分类**: cs.SD, cs.CV
|
||||
- **作者**: Jianrong Wang, Yuchen Huo, Li Liu, Tianyi Xu 等 (6 人)
|
||||
- **摘要**: Audio-visual speech recognition (AVSR) gains increasing attention from researchers as an important part of human-computer interaction. However, the existing available Mandarin audio-visual datasets are limited and lack the depth information. To address this issue, this work establishes the MAVD, a new large-scale Mandarin multimodal corpus comprising 12,484 utterances spoken by 64 native Chinese s...
|
||||
|
||||
#### 7. [2304.13282](https://arxiv.org/abs/2304.13282) — Machine Vision-Based Crop-Load Estimation Using YOLOv8
|
||||
- **发表**: 2023-04-26 | **分类**: cs.RO
|
||||
- **作者**: Dawood Ahmed, Ranjan Sapkota, Martin Churuvija, Manoj Karkee
|
||||
- **摘要**: Labor shortages in fruit crop production have prompted the development of mechanized and automated machines as alternatives to labor-intensive orchard operations such as harvesting, pruning, and thinning. Agricultural robots capable of identifying tree canopy parts and estimating geometric and topological parameters, such as branch diameter, length, and angles, can optimize crop yields through aut...
|
||||
|
||||
#### 8. [2304.08210](https://arxiv.org/abs/2304.08210) — ATTACH Dataset: Annotated Two-Handed Assembly Actions for Human Action Understanding
|
||||
- **发表**: 2023-04-17 | **分类**: cs.RO, cs.CV, cs.LG
|
||||
- **作者**: Dustin Aganian, Benedict Stephan, Markus Eisenbach, Corinna Stretz 等 (5 人)
|
||||
- **摘要**: With the emergence of collaborative robots (cobots), human-robot collaboration in industrial manufacturing is coming into focus. For a cobot to act autonomously and as an assistant, it must understand human actions during assembly. To effectively train models for this task, a dataset containing suitable assembly actions in a realistic setting is crucial. For this purpose, we present the ATTACH dat...
|
||||
|
||||
#### 9. [2303.16196](https://arxiv.org/abs/2303.16196) — SparseNeRF: Distilling Depth Ranking for Few-shot Novel View Synthesis
|
||||
- **发表**: 2023-03-28 | **分类**: cs.CV
|
||||
- **作者**: Guangcong Wang, Zhaoxi Chen, Chen Change Loy, Ziwei Liu
|
||||
- **摘要**: Neural Radiance Field (NeRF) significantly degrades when only a limited number of views are available. To complement the lack of 3D information, depth-based models, such as DSNeRF and MonoSDF, explicitly assume the availability of accurate depth maps of multiple views. They linearly scale the accurate depth maps as supervision to guide the predicted depth of few-shot NeRFs. However, accurate depth...
|
||||
|
||||
#### 10. [2302.05991](https://arxiv.org/abs/2302.05991) — Digital Twin Tracking Dataset (DTTD): A New RGB+Depth 3D Dataset for Longer-Range Object Tracking Applications
|
||||
- **发表**: 2023-02-12 | **分类**: cs.CV
|
||||
- **作者**: Weiyu Feng, Seth Z. Zhao, Chuanyu Pan, Adam Chang 等 (7 人)
|
||||
- **摘要**: Digital twin is a problem of augmenting real objects with their digital counterparts. It can underpin a wide range of applications in augmented reality (AR), autonomy, and UI/UX. A critical component in a good digital-twin system is real-time, accurate 3D object tracking. Most existing works solve 3D object tracking through the lens of robotic grasping, employ older generations of depth sensors, a...
|
||||
|
||||
---
|
||||
|
||||
## I. RGB-D 室内重建
|
||||
**项目阶段**: M2-2 / M3-5 | **论文数**: 10
|
||||
|
||||
RGB-D 输入下的室内场景重建，与本项目的房间级建图任务高度对齐。
|
||||
|
||||
#### 1. [2605.03678](https://arxiv.org/abs/2605.03678) — Robust Visual SLAM for UAV Navigation in GPS-Denied and Degraded Environments: A Multi-Paradigm Evaluation and Deployment Study
|
||||
- **发表**: 2026-05-05 | **分类**: cs.RO
|
||||
- **作者**: Prasoon Kumar, Akshay Deepak, Sandeep Kumar
|
||||
- **摘要**: Reliable localization in GPS-denied, visually degraded environments is critical for autonomous UAV opera- tions. This paper presents a systematic comparative evaluation of five V-SLAM systems ORB-SLAM3, DPVO, DROID-SLAM, DUSt3R, and MASt3R spanning classical, deep learning, recurrent, and Vision Transformer (ViT) paradigms. Experiments are conducted on curated sequences from four public benchmarks...
|
||||
|
||||
#### 2. [2604.28115](https://arxiv.org/abs/2604.28115) — FreeOcc: Training-Free Embodied Open-Vocabulary Occupancy Prediction
|
||||
- **发表**: 2026-04-30 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Zeyu Jiang, Changqing Zhou, Xingxing Zuo, Changhao Chen
|
||||
- **摘要**: Existing learning-based occupancy prediction methods rely on large-scale 3D annotations and generalize poorly across environments. We present FreeOcc, a training-free framework for open-vocabulary occupancy prediction from monocular or RGB-D sequences. Unlike prior approaches that require voxel-level supervision and ground-truth camera poses, FreeOcc operates without 3D annotations, pose ground tr...
|
||||
|
||||
#### 3. [2604.25404](https://arxiv.org/abs/2604.25404) — Robust Graph Matching through Semantic Relationship Generation for SLAM
|
||||
- **发表**: 2026-04-28 | **分类**: cs.RO
|
||||
- **作者**: David Perez-Saura, Jose Andres Millan-Romera, Miguel Fernandez-Cortizas, Holger Voos 等 (6 人)
|
||||
- **摘要**: Graph-based representations such as Scene Graphs enable localization in structured indoor environments by matching a locally observed graph, constructed from sensor data, to a prior map. This process is particularly challenging in environments with repetitive or symmetric layouts, where structural cues alone are often insufficient to resolve ambiguities. We propose a semantic-enhanced graph matchi...
|
||||
|
||||
#### 4. [2604.24707](https://arxiv.org/abs/2604.24707) — Passage-Aware Structural Mapping for RGB-D Visual SLAM
|
||||
- **发表**: 2026-04-27 | **分类**: cs.RO
|
||||
- **作者**: Ali Tourani, Miguel Fernandez-Cortizas, Saad Ejaz, David Pérez Saura 等 (7 人)
|
||||
- **摘要**: Doorways and passages are critical structural elements for indoor robot navigation, yet they remain underexplored in modern Visual SLAM (VSLAM) frameworks. This paper presents a passage-aware structural mapping approach for RGB-D VSLAM that detects doors and traversable openings by jointly fusing geometric, semantic, and topological cues. Doors are modeled as planar entities embedded within walls ...
|
||||
|
||||
#### 5. [2604.19025](https://arxiv.org/abs/2604.19025) — RoomRecon: High-Quality Textured Room Layout Reconstruction on Mobile Devices
|
||||
- **发表**: 2026-04-21 | **分类**: cs.RO
|
||||
- **作者**: Seok Joon Kim, Dinh Duc Cao, Federica Spinola, Se Jin Lee 等 (5 人)
|
||||
- **摘要**: Widespread RGB-Depth (RGB-D) sensors and advanced 3D reconstruction technologies facilitate the capture of indoor spaces, improving the fields of augmented reality (AR), virtual reality (VR), and extended reality (XR). Nevertheless, current technologies still face limitations, such as the inability to reflect minor scene changes without a complete recapture, the lack of semantic scene understandin...
|
||||
|
||||
#### 6. [2604.18336](https://arxiv.org/abs/2604.18336) — Enhancing Glass Surface Reconstruction via Depth Prior for Robot Navigation
|
||||
- **发表**: 2026-04-20 | **分类**: cs.RO, cs.CV
|
||||
- **作者**: Jiamin Zheng, Jingwen Yu, Guangcheng Chen, Hong Zhang
|
||||
- **摘要**: Indoor robot navigation is often compromised by glass surfaces, which severely corrupt depth sensor measurements. While foundation models like Depth Anything 3 provide excellent geometric priors, they lack an absolute metric scale. We propose a training-free framework that leverages depth foundation models as a structural prior, employing a robust local RANSAC-based alignment to fuse it with raw s...
|
||||
|
||||
#### 7. [2604.15052](https://arxiv.org/abs/2604.15052) — CAVERS: Multimodal SLAM Data from a Natural Karstic Cave with Ground Truth Motion Capture
|
||||
- **发表**: 2026-04-16 | **分类**: cs.RO
|
||||
- **作者**: Giacomo Franchini, David Rodríguez-Martínez, Alfonso Martínez-Petersen, C. J. Pérez-del-Pulgar 等 (5 人)
|
||||
- **摘要**: Autonomous robots operating in natural karstic caves face perception and navigation challenges that are qualitatively distinct from those encountered in mines or tunnels: irregular geometry, reflective wet surfaces, near-zero ambient light, and complex branching passages. Yet publicly available datasets targeting this environment remain scarce and offer limited sensing modalities and environmental...
|
||||
|
||||
#### 8. [2604.05621](https://arxiv.org/abs/2604.05621) — FunRec: Reconstructing Functional 3D Scenes from Egocentric Interaction Videos
|
||||
- **发表**: 2026-04-07 | **分类**: cs.CV
|
||||
- **作者**: Alexandros Delitzas, Chenyangguang Zhang, Alexey Gavryushin, Tommaso Di Mario 等 (11 人)
|
||||
- **摘要**: We present FunRec, a method for reconstructing functional 3D digital twins of indoor scenes directly from egocentric RGB-D interaction videos. Unlike existing methods on articulated reconstruction, which rely on controlled setups, multi-state captures, or CAD priors, FunRec operates directly on in-the-wild human interaction sequences to recover interactable 3D scenes. It automatically discovers ar...
|
||||
|
||||
#### 9. [2603.13917](https://arxiv.org/abs/2603.13917) — Evaluation of Visual Place Recognition Methods for Image Pair Retrieval in 3D Vision and Robotics
|
||||
- **发表**: 2026-03-14 | **分类**: cs.CV
|
||||
- **作者**: Dennis Haitz, Athradi Shritish Shetty, Michael Weinmann, Markus Ulrich
|
||||
- **摘要**: Visual Place Recognition (VPR) is a core component in computer vision, typically formulated as an image retrieval task for localization, mapping, and navigation. In this work, we instead study VPR as an image pair retrieval front-end for registration pipelines, where the goal is to find top-matching image pairs between two disjoint image sets for downstream tasks such as scene registration, SLAM, ...
|
||||
|
||||
#### 10. [2512.12378](https://arxiv.org/abs/2512.12378) — M4Human: A Large-Scale Multimodal mmWave Radar Benchmark for Human Mesh Reconstruction
|
||||
- **发表**: 2025-12-13 | **分类**: cs.CV
|
||||
- **作者**: Junqiao Fan, Yunjiao Zhou, Yizhuo Yang, Xinyuan Cui 等 (9 人)
|
||||
- **摘要**: Human mesh reconstruction (HMR) provides direct insights into body-environment interaction, which enables various immersive applications. While existing large-scale HMR datasets rely heavily on line-of-sight RGB input, vision-based sensing is limited by occlusion, lighting variation, and privacy concerns. To overcome these limitations, recent efforts have explored radio-frequency (RF) mmWave radar...
|
||||
|
||||
---
|
||||
|
||||
## J. 神经立体深度（指定 RAFT/IGEV/Foundation 家族）
|
||||
**项目阶段**: M3-4 | **论文数**: 10
|
||||
|
||||
针对 RAFT-Stereo / IGEV-Stereo / FoundationStereo 等核心立体匹配方法的衍生与改进。
|
||||
|
||||
#### 1. [2605.08213](https://arxiv.org/abs/2605.08213) — Low-Cost Stereo Vision for Robust 3D Positioning of Thin Radiata Pine Branches in Autonomous Drone Pruning
|
||||
- **发表**: 2026-05-06 | **分类**: cs.CV
|
||||
- **作者**: Yida Lin, Bing Xue, Mengjie Zhang, Sam Schofield 等 (5 人)
|
||||
- **摘要**: Manual pruning of radiata pine, a species of major economic importance to New Zealand forestry, is hazardous, labour-intensive, and increasingly constrained by workforce shortages. Existing autonomous pruning platforms typically rely on expensive sensors such as LiDAR and are limited to thick branches, which restricts their wider adoption. This paper investigates whether a single low-cost stereo c...
|
||||
|
||||
#### 2. [2604.16480](https://arxiv.org/abs/2604.16480) — Positioning radiata pine branches requiring pruning by drone stereo vision
|
||||
- **发表**: 2026-04-12 | **分类**: cs.CV
|
||||
- **作者**: Yida Lin, Bing Xue, Mengjie Zhang, Sam Schofield 等 (5 人)
|
||||
- **摘要**: This paper presents a stereo-vision-based system mounted on a drone for detecting and localising radiata pine branches to support autonomous pruning. The proposed pipeline comprises two stages: branch segmentation and depth estimation. For segmentation, YOLOv8, YOLOv9, and Mask R-CNN variants are compared on a custom dataset of 71 stereo image pairs captured with a ZED Mini camera. For depth estim...
|
||||
|
||||
#### 3. [2602.19763](https://arxiv.org/abs/2602.19763) — Training Deep Stereo Matching Networks on Tree Branch Imagery: A Benchmark Study for Real-Time UAV Forestry Applications
|
||||
- **发表**: 2026-02-23 | **分类**: cs.CV, eess.IV
|
||||
- **作者**: Yida Lin, Bing Xue, Mengjie Zhang, Sam Schofield 等 (5 人)
|
||||
- **摘要**: Autonomous drone-based tree pruning needs accurate, real-time depth estimation from stereo cameras. Depth is computed from disparity maps using $Z = f B/d$, so even small disparity errors cause noticeable depth mistakes at working distances. Building on our earlier work that identified DEFOM-Stereo as the best reference disparity generator for vegetation scenes, we present the first study to train...
|
||||
|
||||
#### 4. [2601.19461](https://arxiv.org/abs/2601.19461) — Towards Gold-Standard Depth Estimation for Tree Branches in UAV Forestry: Benchmarking Deep Stereo Matching Methods
|
||||
- **发表**: 2026-01-27 | **分类**: cs.CV, cs.RO, eess.IV
|
||||
- **作者**: Yida Lin, Bing Xue, Mengjie Zhang, Sam Schofield 等 (5 人)
|
||||
- **摘要**: Autonomous UAV forestry operations require robust depth estimation with strong cross-domain generalization, yet existing evaluations focus on urban and indoor scenarios, leaving a critical gap for vegetation-dense environments. We present the first systematic zero-shot evaluation of eight stereo methods spanning iterative refinement, foundation model, diffusion-based, and 3D CNN paradigms. All met...
|
||||
|
||||
#### 5. [2512.03427](https://arxiv.org/abs/2512.03427) — Generalization Evaluation of Deep Stereo Matching Methods for UAV-Based Forestry Applications
|
||||
- **发表**: 2025-12-03 | **分类**: cs.CV
|
||||
- **作者**: Yida Lin, Bing Xue, Mengjie Zhang, Sam Schofield 等 (5 人)
|
||||
- **摘要**: Autonomous UAV forestry operations require robust depth estimation methods with strong cross-domain generalization. However, existing evaluations focus on urban and indoor scenarios, leaving a critical gap for specialized vegetation-dense environments. We present the first systematic zero-shot evaluation of eight state-of-the-art stereo methods--RAFT-Stereo, IGEV, IGEV++, BridgeDepth, StereoAnywhe...
|
||||
|
||||
#### 6. [2507.19738](https://arxiv.org/abs/2507.19738) — Leveraging Sparse LiDAR for RAFT-Stereo: A Depth Pre-Fill Perspective
|
||||
- **发表**: 2025-07-26 | **分类**: cs.CV
|
||||
- **作者**: Jinsu Yoo, Sooyoung Jeon, Zanming Huang, Tai-Yu Pan 等 (5 人)
|
||||
- **摘要**: We investigate LiDAR guidance within the RAFT-Stereo framework, aiming to improve stereo matching accuracy by injecting precise LiDAR depth into the initial disparity map. We find that the effectiveness of LiDAR guidance drastically degrades when the LiDAR points become sparse (e.g., a few hundred points per frame), and we offer a novel explanation from a signal processing perspective. This insigh...
|
||||
|
||||
#### 7. [2507.10991](https://arxiv.org/abs/2507.10991) — Uncertainty Aware Mapping for Vision-Based Underwater Robots
|
||||
- **发表**: 2025-07-15 | **分类**: cs.RO
|
||||
- **作者**: Abhimanyu Bhowmik, Mohit Singh, Madhushree Sannigrahi, Martin Ludvigsen 等 (5 人)
|
||||
- **摘要**: Vision-based underwater robots can be useful in inspecting and exploring confined spaces where traditional sensors and preplanned paths cannot be followed. Sensor noise and situational change can cause significant uncertainty in environmental representation. Thus, this paper explores how to represent mapping inconsistency in vision-based sensing and incorporate depth estimation confidence into the...
|
||||
|
||||
#### 8. [2506.16690](https://arxiv.org/abs/2506.16690) — DepthVanish: Optimizing Adversarial Interval Structures for Stereo-Depth-Invisible Patches
|
||||
- **发表**: 2025-06-20 | **分类**: cs.CV
|
||||
- **作者**: Yun Xing, Yue Cao, Nhat Chung, Jie Zhang 等 (9 人)
|
||||
- **摘要**: Stereo depth estimation is a critical task in autonomous driving and robotics, where inaccuracies (such as misidentifying nearby objects as distant) can lead to dangerous situations. Adversarial attacks against stereo depth estimation can help reveal vulnerabilities before deployment. Previous works have shown that repeating optimized textures can effectively mislead stereo depth estimation in dig...
|
||||
|
||||
#### 9. [2505.11439](https://arxiv.org/abs/2505.11439) — SurgPose: Generalisable Surgical Instrument Pose Estimation using Zero-Shot Learning and Stereo Vision
|
||||
- **发表**: 2025-05-16 | **分类**: cs.CV, cs.AI, cs.LG
|
||||
- **作者**: Utsav Rai, Haozheng Xu, Stamatia Giannarou
|
||||
- **摘要**: Accurate pose estimation of surgical tools in Robot-assisted Minimally Invasive Surgery (RMIS) is essential for surgical navigation and robot control. While traditional marker-based methods offer accuracy, they face challenges with occlusions, reflections, and tool-specific designs. Similarly, supervised learning methods require extensive training on annotated datasets, limiting their adaptability...
|
||||
|
||||
#### 10. [2505.03702](https://arxiv.org/abs/2505.03702) — Self-Supervised Learning for Robotic Leaf Manipulation: A Hybrid Geometric-Neural Approach
|
||||
- **发表**: 2025-05-06 | **分类**: cs.RO, cs.CV, cs.LG
|
||||
- **作者**: Srecharan Selvam
|
||||
- **摘要**: Automating leaf manipulation in agricultural settings faces significant challenges, including the variability of plant morphologies and deformable leaves. We propose a novel hybrid geometric-neural approach for autonomous leaf grasping that combines traditional computer vision with neural networks through self-supervised learning. Our method integrates YOLOv8 for instance segmentation and RAFT-Ste...
|
||||
|
||||
---
|
||||
|
||||
# 第二部分 · GitHub 仓库（按 stars 排序）
|
||||
|
||||
### GitHub: stereo_matching（按 stars 排序）
|
||||
|
||||
- [gto76/python-cheatsheet](https://github.com/gto76/python-cheatsheet) — ⭐ 38,397 | Python | Comprehensive Python Cheatsheet
|
||||
- [jbhuang0604/awesome-computer-vision](https://github.com/jbhuang0604/awesome-computer-vision) — ⭐ 23,266 | | A curated list of awesome computer vision resources
|
||||
- [spmallick/learnopencv](https://github.com/spmallick/learnopencv) — ⭐ 22,915 | Jupyter Notebook | Learn OpenCV : C++ and Python Examples
|
||||
- [amusi/CVPR2026-Papers-with-Code](https://github.com/amusi/CVPR2026-Papers-with-Code) — ⭐ 22,561 | | CVPR 2026 论文和开源项目合集
|
||||
- [zziz/pwc](https://github.com/zziz/pwc) — ⭐ 15,335 | | This repository is no longer maintained.
|
||||
- [alicevision/Meshroom](https://github.com/alicevision/Meshroom) — ⭐ 12,730 | QML | Node-based Visual Programming Toolbox
|
||||
- [diff-usion/Awesome-Diffusion-Models](https://github.com/diff-usion/Awesome-Diffusion-Models) — ⭐ 12,321 | HTML | A collection of resources and papers on Diffusion Models
|
||||
- [kornia/kornia](https://github.com/kornia/kornia) — ⭐ 11,204 | Python | 🐍 Geometric Computer Vision Library for Spatial AI
|
||||
- [timzhang642/3D-Machine-Learning](https://github.com/timzhang642/3D-Machine-Learning) — ⭐ 10,162 | | A resource repository for 3D machine learning
|
||||
- [satellite-image-deep-learning/techniques](https://github.com/satellite-image-deep-learning/techniques) — ⭐ 10,145 | | Techniques for deep learning with satellite & aerial imagery
|
||||
|
||||
### GitHub: vio_slam（按 stars 排序）
|
||||
|
||||
- [jbhuang0604/awesome-computer-vision](https://github.com/jbhuang0604/awesome-computer-vision) — ⭐ 23,266 | | A curated list of awesome computer vision resources
|
||||
- [UZ-SLAMLab/ORB_SLAM3](https://github.com/UZ-SLAMLab/ORB_SLAM3) — ⭐ 8,608 | C++ | ORB-SLAM3: An Accurate Open-Source Library for Visual, Visual-Inertial and Multi-Map SLAM
|
||||
- [HKUST-Aerial-Robotics/VINS-Mono](https://github.com/HKUST-Aerial-Robotics/VINS-Mono) — ⭐ 5,897 | C++ | A Robust and Versatile Monocular Visual-Inertial State Estimator
|
||||
- [hku-mars/FAST_LIO](https://github.com/hku-mars/FAST_LIO) — ⭐ 4,663 | C++ | A computationally efficient and robust LiDAR-inertial odometry (LIO) package
|
||||
- [SLAM-Handbook-contributors/slam-handbook-public-release](https://github.com/SLAM-Handbook-contributors/slam-handbook-public-release) — ⭐ 4,491 | TeX | Release repo for our SLAM Handbook
|
||||
- [openMVG/awesome_3DReconstruction_list](https://github.com/openMVG/awesome_3DReconstruction_list) — ⭐ 4,401 | | A curated list of papers & resources linked to 3D reconstruction from images.
|
||||
- [Ly0n/awesome-robotic-tooling](https://github.com/Ly0n/awesome-robotic-tooling) — ⭐ 3,813 | | Tooling for professional robotic development in C++ and Python with a touch of ROS, autonomous driving and aerospace.
|
||||
- [uzh-rpg/event-based_vision_resources](https://github.com/uzh-rpg/event-based_vision_resources) — ⭐ 3,543 | | Event-based Vision Resources. Community effort to collect knowledge on event-based vision technology (papers, workshops,
|
||||
- [Awesome3DGS/3D-Gaussian-Splatting-Papers](https://github.com/Awesome3DGS/3D-Gaussian-Splatting-Papers) — ⭐ 2,971 | Python | 3D高斯论文，持续更新，欢迎交流讨论。
|
||||
- [rpng/open_vins](https://github.com/rpng/open_vins) — ⭐ 2,891 | C++ | An open source platform for visual-inertial navigation research.
|
||||
|
||||
### GitHub: gaussian_splatting（按 stars 排序）
|
||||
|
||||
- [GitHubDaily/GitHubDaily](https://github.com/GitHubDaily/GitHubDaily) — ⭐ 46,470 | | 坚持分享 GitHub 上高质量、有趣实用的开源技术教程、开发者工具、编程网站、技术资讯。A list cool, interesting projects of GitHub.
|
||||
- [spmallick/learnopencv](https://github.com/spmallick/learnopencv) — ⭐ 22,915 | Jupyter Notebook | Learn OpenCV : C++ and Python Examples
|
||||
- [amusi/CVPR2026-Papers-with-Code](https://github.com/amusi/CVPR2026-Papers-with-Code) — ⭐ 22,561 | | CVPR 2026 论文和开源项目合集
|
||||
- [graphdeco-inria/gaussian-splatting](https://github.com/graphdeco-inria/gaussian-splatting) — ⭐ 21,964 | Python | Original reference implementation of "3D Gaussian Splatting for Real-Time Radiance Field Rendering"
|
||||
- [rothgar/awesome-tuis](https://github.com/rothgar/awesome-tuis) — ⭐ 18,873 | | List of projects that provide terminal user interfaces
|
||||
- [playcanvas/engine](https://github.com/playcanvas/engine) — ⭐ 15,786 | JavaScript | Powerful web graphics runtime built on WebGL, WebGPU, WebXR and glTF
|
||||
- [vercel-labs/json-render](https://github.com/vercel-labs/json-render) — ⭐ 14,800 | TypeScript | The Generative UI framework
|
||||
- [facebookresearch/vggt](https://github.com/facebookresearch/vggt) — ⭐ 13,105 | Python | [CVPR 2025 Best Paper Award] VGGT: Visual Geometry Grounded Transformer
|
||||
- [alicevision/Meshroom](https://github.com/alicevision/Meshroom) — ⭐ 12,730 | QML | Node-based Visual Programming Toolbox
|
||||
- [microsoft/TRELLIS](https://github.com/microsoft/TRELLIS) — ⭐ 12,607 | Python | Official repo for paper "Structured 3D Latents for Scalable and Versatile 3D Generation" (CVPR'25 Spotlight).
|
||||
|
||||
### GitHub: monocular_depth（按 stars 排序）
|
||||
|
||||
- [spmallick/learnopencv](https://github.com/spmallick/learnopencv) — ⭐ 22,915 | Jupyter Notebook | Learn OpenCV : C++ and Python Examples
|
||||
- [huggingface/transformers.js](https://github.com/huggingface/transformers.js) — ⭐ 16,004 | JavaScript | State-of-the-art Machine Learning for the web. Run 🤗 Transformers directly in your browser, with no need for a server!
|
||||
- [zziz/pwc](https://github.com/zziz/pwc) — ⭐ 15,335 | | This repository is no longer maintained.
|
||||
- [hindupuravinash/the-gan-zoo](https://github.com/hindupuravinash/the-gan-zoo) — ⭐ 14,696 | Python | A list of all named GANs!
|
||||
- [facebookresearch/vggt](https://github.com/facebookresearch/vggt) — ⭐ 13,105 | Python | [CVPR 2025 Best Paper Award] VGGT: Visual Geometry Grounded Transformer
|
||||
- [alicevision/Meshroom](https://github.com/alicevision/Meshroom) — ⭐ 12,730 | QML | Node-based Visual Programming Toolbox
|
||||
- [diff-usion/Awesome-Diffusion-Models](https://github.com/diff-usion/Awesome-Diffusion-Models) — ⭐ 12,321 | HTML | A collection of resources and papers on Diffusion Models
|
||||
- [NielsRogge/Transformers-Tutorials](https://github.com/NielsRogge/Transformers-Tutorials) — ⭐ 11,629 | Jupyter Notebook | This repository contains demos I made with the Transformers library by HuggingFace.
|
||||
- [facebookresearch/dinov3](https://github.com/facebookresearch/dinov3) — ⭐ 10,401 | Jupyter Notebook | Reference PyTorch implementation and models for DINOv3
|
||||
- [timzhang642/3D-Machine-Learning](https://github.com/timzhang642/3D-Machine-Learning) — ⭐ 10,162 | | A resource repository for 3D machine learning
|
||||
|
||||
### GitHub: world_model（按 stars 排序）
|
||||
|
||||
- [codecrafters-io/build-your-own-x](https://github.com/codecrafters-io/build-your-own-x) — ⭐ 501,730 | Markdown | Master programming by recreating your favorite technologies from scratch.
|
||||
- [public-apis/public-apis](https://github.com/public-apis/public-apis) — ⭐ 435,215 | Python | A collective list of free APIs
|
||||
- [donnemartin/system-design-primer](https://github.com/donnemartin/system-design-primer) — ⭐ 348,805 | Python | Learn how to design large-scale systems. Prep for the system design interview. Includes Anki flashcards.
|
||||
- [jwasham/coding-interview-university](https://github.com/jwasham/coding-interview-university) — ⭐ 346,810 | | A complete computer science study plan to become a software engineer.
|
||||
- [vinta/awesome-python](https://github.com/vinta/awesome-python) — ⭐ 297,887 | Python | An opinionated list of Python frameworks, libraries, tools, and resources
|
||||
- [awesome-selfhosted/awesome-selfhosted](https://github.com/awesome-selfhosted/awesome-selfhosted) — ⭐ 292,669 | | A list of Free Software network services and web applications which can be hosted on your own servers
|
||||
- [practical-tutorials/project-based-learning](https://github.com/practical-tutorials/project-based-learning) — ⭐ 265,809 | | Curated list of project-based tutorials
|
||||
- [trimstray/the-book-of-secret-knowledge](https://github.com/trimstray/the-book-of-secret-knowledge) — ⭐ 220,462 | | A collection of inspiring lists, manuals, cheatsheets, blogs, hacks, one-liners, cli/web tools and more.
|
||||
- [ossu/computer-science](https://github.com/ossu/computer-science) — ⭐ 203,840 | HTML | 🎓 Path to a free self-taught education in Computer Science!
|
||||
- [affaan-m/everything-claude-code](https://github.com/affaan-m/everything-claude-code) — ⭐ 183,877 | JavaScript | The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development f
|
||||
|
||||
---
|
||||
|
||||
# 第三部分 · 关键洞察与项目对接建议
|
||||
|
||||
## I.1 最值得关注的新论文（按相关性挑选）
|
||||
|
||||
下方挑选每个主题中**与本项目最相关的 3 篇**（基于标题/摘要语义判断）：
|
||||
|
||||
### A. 双目立体匹配（被动深度） → M2-3 / M3-4
|
||||
- **[2605.14963](https://arxiv.org/abs/2605.14963)** H-OmniStereo: Zero-Shot Omnidirectional Stereo Matching with Heading-Aligned Normal Priors（2026-05-14）
|
||||
- **[2605.08592](https://arxiv.org/abs/2605.08592)** Cross-Modal RGB-D Fusion Transformer for 6D Pose Estimation of Non-Cooperative Spacecraft with Stereo-Derived Depth（2026-05-09）
|
||||
- **[2604.20393](https://arxiv.org/abs/2604.20393)** MLG-Stereo: ViT Based Stereo Matching with Multi-Stage Local-Global Enhancement（2026-04-22）
|
||||
|
||||
### B. 视觉惯性 SLAM / VIO → M2-1 / M3-3
|
||||
- **[2605.07552](https://arxiv.org/abs/2605.07552)** VIMCAN: Visual-Inertial 3D Human Pose Estimation with Hybrid Mamba-Cross-Attention Network（2026-05-08）
|
||||
- **[2605.02054](https://arxiv.org/abs/2605.02054)** Observability Conditions and Filter Design for Visual Pose Estimation via Dual Quaternions（2026-05-03）
|
||||
- **[2604.07151](https://arxiv.org/abs/2604.07151)** An RTK-SLAM Dataset for Absolute Accuracy Evaluation in GNSS-Degraded Environments（2026-04-08）
|
||||
|
||||
### C. 3D Gaussian Splatting SLAM（融合建图） → M3-5 / M4
|
||||
- **[2605.10760](https://arxiv.org/abs/2605.10760)** MAGS-SLAM: Monocular Multi-Agent Gaussian Splatting SLAM for Geometrically and Photometrically Consistent Reconstruction（2026-05-11）
|
||||
- **[2604.22339](https://arxiv.org/abs/2604.22339)** Flow4DGS-SLAM: Optical Flow-Guided 4D Gaussian Splatting SLAM（2026-04-24）
|
||||
- **[2604.15612](https://arxiv.org/abs/2604.15612)** GaussianFlow SLAM: Monocular Gaussian Splatting SLAM Guided by GaussianFlow（2026-04-17）
|
||||
|
||||
### D. 单目深度基础模型 → M3-4
|
||||
- **[2605.11756](https://arxiv.org/abs/2605.11756)** Focusable Monocular Depth Estimation（2026-05-12）
|
||||
- **[2605.07264](https://arxiv.org/abs/2605.07264)** Sat3R: Satellite DSM Reconstruction via RPC-Aware Depth Fine-tuning（2026-05-08）
|
||||
- **[2605.06270](https://arxiv.org/abs/2605.06270)** Spark3R: Asymmetric Token Reduction Makes Fast Feed-Forward 3D Reconstruction（2026-05-07）
|
||||
|
||||
## I.2 后续动作清单
|
||||
|
||||
- [ ] 把上述每主题的 Top-3 论文加入 [`research/zed2i_stereo_vio_arxiv_review.md`](zed2i_stereo_vio_arxiv_review.md) 第 F 节的论文映射表
|
||||
- [ ] 对 🇨🇳 标记的论文重点核查机构归属，更新 G 节国产团队清单
|
||||
- [ ] 把 GitHub 仓库中 stars > 5k 的项目加入 [`plans/camera/github_opensource_projects.md`](../plans/camera/github_opensource_projects.md)
|
||||
- [ ] 每周重跑 [`research/search_info.py`](search_info.py) 增量更新
|
||||
|
||||
## I.3 复现方法
|
||||
|
||||
```bash
|
||||
# 通过 127.0.0.1:6984 代理拉取最新数据
|
||||
HTTPS_PROXY=http://127.0.0.1:6984 HTTP_PROXY=http://127.0.0.1:6984 \
|
||||
python3 research/search_info.py \
|
||||
--proxy http://127.0.0.1:6984 \
|
||||
--max-results 10 --delay 5.0 \
|
||||
--out research/search_results.json
|
||||
|
||||
# 渲染为 markdown
|
||||
python3 research/gen_review_from_json.py \
|
||||
--in research/search_results.json \
|
||||
--out research/zed2i_arxiv_live_review.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**说明**：
|
||||
- 本文档由脚本自动生成，可重复执行覆盖
|
||||
- 🇨🇳 标记基于作者/摘要中是否包含中国机构关键词的启发式判断，**仅供参考，需人工复核**
|
||||
- 摘要截断到 400 字符以控制文档体积
|
||||
- 与 [`zed2i_stereo_vio_arxiv_review.md`](zed2i_stereo_vio_arxiv_review.md)（人工综述）互为补充：人工综述给方法论与映射，本文档给最新原始素材
|
||||
@@ -0,0 +1,392 @@
|
||||
# 双目立体 + 视觉惯性 + 神经建图：服务于 ZED 2i 数据 Pipeline 的 arXiv 论文综述
|
||||
|
||||
> 围绕 [`plans/camera/zed2i_stereo_imu_solution.md`](../plans/camera/zed2i_stereo_imu_solution.md) 与 [`plans/camera/zed2i_iterative_framework.md`](../plans/camera/zed2i_iterative_framework.md) 提出的"M1–M4 × L1–L4"框架，系统梳理 arXiv 上与本方案直接相关的代表性论文。每篇均给出 arXiv ID、链接、核心贡献、与本项目的对接点。
|
||||
>
|
||||
> **检索约定**：以下论文均可通过 `https://arxiv.org/abs/<ID>` 访问；引文以"作者, 年份, [arXiv:ID]"形式标注。本综述聚焦近 5 年（2020–2025），重点覆盖 SOTA 与开源可用方法。
|
||||
>
|
||||
> **⚠️ 数据时效说明**：本综述基于公开训练语料整理（**人工综述**），arXiv ID 与发表年份已经过交叉核对，但**最终引用前请人工到 arxiv.org 复核论文是否存在、版本号与作者列表**。
|
||||
>
|
||||
> **🔄 配套实时综述**：参见 [`zed2i_arxiv_live_review.md`](zed2i_arxiv_live_review.md) ——通过 [`search_info.py`](search_info.py:7) + HTTP 代理 `127.0.0.1:6984` 从 arxiv.org 实时拉取的 100 篇最新论文（含 2026 年 5 月发布的新作），按本项目主线分组。两份综述互补：**本文档**给方法论与映射，**live 综述**给最新原始素材。重跑命令：
|
||||
>
|
||||
> ```bash
|
||||
> HTTPS_PROXY=http://127.0.0.1:6984 HTTP_PROXY=http://127.0.0.1:6984 \
|
||||
> python3 research/search_info.py --proxy http://127.0.0.1:6984 \
|
||||
> --max-results 10 --delay 5.0
|
||||
> python3 research/gen_review_from_json.py
|
||||
> ```
|
||||
|
||||
---
|
||||
|
||||
## 0. 综述结构
|
||||
|
||||
本文按本项目五条技术主线组织：
|
||||
|
||||
| 主线 | 对应项目阶段 | 论文数 |
|
||||
|---|---|---|
|
||||
| **A. 双目立体匹配（被动深度）** | M2-3 / M3-4 | 8 |
|
||||
| **B. 视觉惯性 SLAM/VIO** | M2-1 / M3-3 | 8 |
|
||||
| **C. 神经深度估计（单目+立体后处理）** | M3-4 | 6 |
|
||||
| **D. 神经辐射场与 3DGS 建图** | M3-5 / M4-1 | 7 |
|
||||
| **E. 室内 RGB-D 数据集与 World Model** | M4-1 / M4-2 / M4-4 | 6 |
|
||||
|
||||
合计 ~35 篇核心文献。每条主线末尾给出"对接本项目的具体落地建议"。
|
||||
|
||||
---
|
||||
|
||||
## A. 双目立体匹配（被动深度，对标 ZED 深度算法）
|
||||
|
||||
### A.1 RAFT-Stereo（2021）
|
||||
- **arXiv**: [arXiv:2109.07547](https://arxiv.org/abs/2109.07547)
|
||||
- **作者**: Lipson, Teed, Deng（普林斯顿）
|
||||
- **核心**: 把 RAFT 光流的迭代式相关体（correlation volume）+ GRU 思路迁移到立体匹配，单 GPU 实时，KITTI/Middlebury SOTA。
|
||||
- **对接 ZED**: ZED ULTRA 模式底层算法路线与之接近；本项目 **M3-4 深度算法消融**可用 RAFT-Stereo 作为"替换 ZED ULTRA"的候选。
|
||||
|
||||
### A.2 IGEV-Stereo（2023）
|
||||
- **arXiv**: [arXiv:2303.06615](https://arxiv.org/abs/2303.06615)
|
||||
- **作者**: Xu, Wang 等（华中科技大学）
|
||||
- **核心**: Iterative Geometry Encoding Volume，融合几何编码体+RAFT 迭代，2023 KITTI Stereo 排行榜前列。
|
||||
- **对接**: **国产团队作品**，对应 [`zed2i_china_alternatives.md`](../plans/camera/zed2i_china_alternatives.md) 的国产算法栈；可用于奥比中光 Gemini 335L 双目数据的高质量后处理。
|
||||
|
||||
### A.3 CREStereo（CVPR 2022）
|
||||
- **arXiv**: [arXiv:2203.11483](https://arxiv.org/abs/2203.11483)
|
||||
- **作者**: Li, Liu 等（旷视）
|
||||
- **核心**: 级联递归网络，针对真实场景（非合成）的鲁棒性显著强于同期方法；Megvii 团队。
|
||||
- **对接**: 反光/弱纹理区（M2-4）的强基线；国产团队作品。
|
||||
|
||||
### A.4 FoundationStereo（2025）
|
||||
- **arXiv**: [arXiv:2501.09898](https://arxiv.org/abs/2501.09898)
|
||||
- **作者**: NVIDIA Research
|
||||
- **核心**: 首个真正"零样本泛化"的立体匹配基础模型，1M+ 合成数据预训练，跨数据集无需 finetune 即达 SOTA。
|
||||
- **对接**: **M3-4 神经深度后处理的首选**——直接喂 ZED/Gemini 的左右图，输出比 SDK 内建更精的深度；尤其适合反光/玻璃/弱纹理区。
|
||||
|
||||
### A.5 Selective-Stereo（CVPR 2024）
|
||||
- **arXiv**: [arXiv:2403.00486](https://arxiv.org/abs/2403.00486)
|
||||
- **核心**: 多频段视差选择性聚合，在高细节+大视差场景同时占优。
|
||||
- **对接**: 物体级近距环拍（M2-3）受益。
|
||||
|
||||
### A.6 StereoCrafter / DepthCrafter 系列（2024）
|
||||
- **arXiv**: DepthCrafter [arXiv:2409.02095](https://arxiv.org/abs/2409.02095) / StereoCrafter [arXiv:2409.07447](https://arxiv.org/abs/2409.07447)
|
||||
- **作者**: Tencent ARC Lab
|
||||
- **核心**: 用视频扩散模型做时序一致的深度估计；StereoCrafter 把单目视频转双目。
|
||||
- **对接**: 时序一致性（M4-1 关键指标），可作为深度后处理"时间平滑"模块；国产团队作品。
|
||||
|
||||
### A.7 NMRF-Stereo（CVPR 2024）
|
||||
- **arXiv**: [arXiv:2406.01413](https://arxiv.org/abs/2406.01413)
|
||||
- **核心**: Neural Markov Random Field 立体匹配，对边缘/不连续区域显著改善。
|
||||
- **对接**: 家具边缘、深度跳变处的精度提升。
|
||||
|
||||
### A.8 Mono+Stereo Fusion: Marigold-Depth（CVPR 2024 Best Paper Honorable）
|
||||
- **arXiv**: [arXiv:2312.02145](https://arxiv.org/abs/2312.02145)
|
||||
- **核心**: 把 Stable Diffusion 作为单目深度先验，仅用合成数据 finetune 即跨域泛化。
|
||||
- **对接**: 与双目深度融合（如 ZED 出错区域用 Marigold 补全），M3-4 多模型集成。
|
||||
|
||||
### A.x 主线小结与项目落地
|
||||
- **首选基础模型**：FoundationStereo（零样本，工程友好）
|
||||
- **首选国产**：IGEV-Stereo / CREStereo（华中科大 / 旷视）
|
||||
- **时序一致**：DepthCrafter
|
||||
- **落地动作**：M3-4 跑一次"ZED ULTRA / IGEV / FoundationStereo / DepthCrafter" 四方对照，在反光区/弱纹理区/远距三类失效场景上量化 RMSE 与覆盖率提升。
|
||||
|
||||
---
|
||||
|
||||
## B. 视觉惯性 SLAM / VIO（对标 ZED 内建 VIO）
|
||||
|
||||
### B.1 ORB-SLAM3（2021）
|
||||
- **arXiv**: [arXiv:2007.11898](https://arxiv.org/abs/2007.11898)
|
||||
- **作者**: Campos, Elvira, Tardós（Zaragoza 大学）
|
||||
- **核心**: 多地图、视觉-惯性-纯视觉统一框架，开源标杆。
|
||||
- **对接**: M3-3 替换 ZED 内建 VIO 的首选基线；与 Gemini 335L / MYNT EYE 集成成熟。
|
||||
|
||||
### B.2 VINS-Fusion（2019 期刊 → arXiv 多次更新）
|
||||
- **arXiv**: [arXiv:1901.03642](https://arxiv.org/abs/1901.03642)（VINS-Mono → Fusion）
|
||||
- **作者**: 沈邵劼组（港科大）
|
||||
- **核心**: 紧耦合视觉-惯性优化，可选 GPS / 双目扩展。
|
||||
- **对接**: 国产工程界最常用 VIO；M3-3 必选对照之一；**港科大**国产团队。
|
||||
|
||||
### B.3 BASALT（2019, ICRA Best Paper）
|
||||
- **arXiv**: [arXiv:1904.06504](https://arxiv.org/abs/1904.06504)
|
||||
- **作者**: Usenko, Demmel, Cremers（慕尼黑工大）
|
||||
- **核心**: 非线性因子恢复（NFR），全双目+IMU 紧耦合，精度优于 ORB-SLAM3 在 EuRoC 上。
|
||||
- **对接**: M3-3 对照组高精度参考。
|
||||
|
||||
### B.4 DROID-SLAM（NeurIPS 2021）
|
||||
- **arXiv**: [arXiv:2108.10869](https://arxiv.org/abs/2108.10869)
|
||||
- **作者**: Teed, Deng（普林斯顿）
|
||||
- **核心**: 用 RAFT 的稠密光流做端到端可微 SLAM，深度学习时代 SLAM 标杆。
|
||||
- **对接**: 神经 SLAM 路线代表；M3-5 融合建图阶段可与传统 SLAM 对比。
|
||||
|
||||
### B.5 DPVO（NeurIPS 2023）
|
||||
- **arXiv**: [arXiv:2208.04726](https://arxiv.org/abs/2208.04726)
|
||||
- **作者**: Teed, Lipson, Deng
|
||||
- **核心**: Deep Patch Visual Odometry，比 DROID-SLAM 快 10×，单目仅需 1 GPU。
|
||||
- **对接**: M3-3 算法消融"轻量神经 VIO"代表。
|
||||
|
||||
### B.6 OKVIS2（2023）
|
||||
- **arXiv**: [arXiv:2303.12005](https://arxiv.org/abs/2303.12005)
|
||||
- **作者**: Leutenegger（TUM/帝国理工）
|
||||
- **核心**: 原 OKVIS 升级版，引入 keyframe-based marginalization 优化。
|
||||
- **对接**: 工业级 VIO 备选，鲁棒性强。
|
||||
|
||||
### B.7 GS-SLAM / Photo-SLAM 系列（CVPR 2024）
|
||||
- **arXiv**: GS-SLAM [arXiv:2311.11700](https://arxiv.org/abs/2311.11700) / Photo-SLAM [arXiv:2311.16728](https://arxiv.org/abs/2311.16728)
|
||||
- **核心**: 把 3D Gaussian Splatting 作为 SLAM 后端地图表征，实时定位+建图+渲染一体。
|
||||
- **对接**: **M3-5 融合建图首选神经后端**；输出可直接用于 World Model 训练。
|
||||
|
||||
### B.8 MASt3R-SLAM（2024）
|
||||
- **arXiv**: [arXiv:2412.12392](https://arxiv.org/abs/2412.12392)
|
||||
- **作者**: Naver Labs / Imperial
|
||||
- **核心**: 基于 DUSt3R/MASt3R 的"无相机标定 SLAM"，对未知/不准内参鲁棒。
|
||||
- **对接**: M1-3 标定不准时的兜底；多设备混采场景。
|
||||
|
||||
### B.x 主线小结
|
||||
- **稳定首选**：ORB-SLAM3（成熟）/ VINS-Fusion（国产）
|
||||
- **神经 SOTA**：DROID-SLAM → DPVO（轻量）
|
||||
- **建图一体**：GS-SLAM / Photo-SLAM
|
||||
- **落地动作**：M3-3 矩阵设计 = {ZED VIO, ORB-SLAM3, VINS-Fusion, DPVO} × {IMU on, off}，在 M2 录制的 10 间客房上跑 ATE/RPE。
|
||||
|
||||
---
|
||||
|
||||
## C. 神经深度估计（单目 + 通用基础模型）
|
||||
|
||||
### C.1 MiDaS v3.1（2022）
|
||||
- **arXiv**: [arXiv:2307.14460](https://arxiv.org/abs/2307.14460)
|
||||
- **作者**: Intel Labs / Ranftl
|
||||
- **核心**: 大规模混合数据训练的相对深度模型，开源标杆。
|
||||
- **对接**: 用作 ZED 深度的"sanity check"。
|
||||
|
||||
### C.2 ZoeDepth（2023）
|
||||
- **arXiv**: [arXiv:2302.12288](https://arxiv.org/abs/2302.12288)
|
||||
- **核心**: MiDaS + metric head，输出**绝对深度**而非相对深度。
|
||||
- **对接**: M2-4 失效区（玻璃/反光）的兜底深度。
|
||||
|
||||
### C.3 Depth Anything v1/v2（CVPR 2024 / 2024）
|
||||
- **arXiv**: v1 [arXiv:2401.10891](https://arxiv.org/abs/2401.10891) / v2 [arXiv:2406.09414](https://arxiv.org/abs/2406.09414)
|
||||
- **作者**: 字节跳动 / 港大
|
||||
- **核心**: 62M 无标注数据 + 教师-学生伪标签，零样本泛化最强单目深度。
|
||||
- **对接**: **国产团队作品**；M3-4 单目深度首选；与 FoundationStereo 互补。
|
||||
|
||||
### C.4 UniDepth（CVPR 2024）
|
||||
- **arXiv**: [arXiv:2403.18913](https://arxiv.org/abs/2403.18913)
|
||||
- **核心**: 单目度量深度 + 相机内参自适应，对未知设备友好。
|
||||
- **对接**: 多设备混采（ZED + iPhone + Gemini）统一深度表达。
|
||||
|
||||
### C.5 Marigold（CVPR 2024 Honorable Mention）
|
||||
- **arXiv**: [arXiv:2312.02145](https://arxiv.org/abs/2312.02145)
|
||||
- **核心**: 借用 Stable Diffusion 先验做单目深度，跨域强。
|
||||
- **对接**: 弱纹理大平面（白墙）的深度补全。
|
||||
|
||||
### C.6 Metric3D v2（2024）
|
||||
- **arXiv**: [arXiv:2404.15506](https://arxiv.org/abs/2404.15506)
|
||||
- **作者**: 阿里 DAMO
|
||||
- **核心**: 度量深度 + 法向量联合估计，跨数据集泛化。
|
||||
- **对接**: **国产团队作品**；M2-4 法向量可用于深度置信度判别。
|
||||
|
||||
### C.x 主线小结
|
||||
- **首选**：Depth Anything v2 + UniDepth 双路验证（均国产/含国产）
|
||||
- **落地动作**：把单目深度作为双目失效的兜底，输出"双目+单目融合深度图"+置信度 mask。
|
||||
|
||||
---
|
||||
|
||||
## D. 神经辐射场与 3D Gaussian Splatting 建图
|
||||
|
||||
### D.1 NeRF（ECCV 2020）
|
||||
- **arXiv**: [arXiv:2003.08934](https://arxiv.org/abs/2003.08934)
|
||||
- **核心**: 神经辐射场开山之作。
|
||||
- **对接**: 作为 M3-5 / M4-1 历史基线，了解即可。
|
||||
|
||||
### D.2 Instant-NGP（SIGGRAPH 2022）
|
||||
- **arXiv**: [arXiv:2201.05989](https://arxiv.org/abs/2201.05989)
|
||||
- **作者**: NVIDIA
|
||||
- **核心**: 多分辨率哈希编码，秒级训练 NeRF。
|
||||
- **对接**: Nerfstudio 默认后端之一。
|
||||
|
||||
### D.3 Nerfacto（Nerfstudio 框架）
|
||||
- **GitHub**: nerfstudio-project/nerfstudio
|
||||
- **核心**: Nerfstudio 推荐的实用 NeRF 配置（不是单独论文，但是工程标准）。
|
||||
- **对接**: M3-5 神经建图选型。
|
||||
|
||||
### D.4 3D Gaussian Splatting（SIGGRAPH 2023 Best Paper）
|
||||
- **arXiv**: [arXiv:2308.04079](https://arxiv.org/abs/2308.04079)
|
||||
- **作者**: INRIA / Université Côte d'Azur
|
||||
- **核心**: 显式高斯椭球+ splatting 光栅化，质量+速度全面超越 NeRF。
|
||||
- **对接**: **M3-5 融合建图首选后端**；World Model 视觉表征当前最强。
|
||||
|
||||
### D.5 SplaTAM（CVPR 2024）
|
||||
- **arXiv**: [arXiv:2312.02126](https://arxiv.org/abs/2312.02126)
|
||||
- **核心**: 3DGS + SLAM 一体化，RGB-D 输入实时定位+建图。
|
||||
- **对接**: M3-5 候选；与 GS-SLAM、Photo-SLAM 同类对比。
|
||||
|
||||
### D.6 NICE-SLAM / NICER-SLAM（CVPR 2022/2024）
|
||||
- **arXiv**: [arXiv:2112.12130](https://arxiv.org/abs/2112.12130) / [arXiv:2302.03594](https://arxiv.org/abs/2302.03594)
|
||||
- **作者**: ETH Zürich / Marc Pollefeys
|
||||
- **核心**: 神经隐式表征 SLAM。
|
||||
- **对接**: 历史对比。
|
||||
|
||||
### D.7 MonoGS / RTG-SLAM（CVPR 2024）
|
||||
- **arXiv**: MonoGS [arXiv:2312.06741](https://arxiv.org/abs/2312.06741) / RTG-SLAM [arXiv:2404.19706](https://arxiv.org/abs/2404.19706)
|
||||
- **核心**: 单目 3DGS SLAM；RTG-SLAM 强调实时大场景。
|
||||
- **对接**: 单目场景兜底（如纯 iPhone 录制时）。
|
||||
|
||||
### D.x 主线小结
|
||||
- **建图首选**：3DGS（速度/质量平衡），SplaTAM/MonoGS 集成 SLAM
|
||||
- **落地动作**：M3-5 用 3DGS 把 ZED/Gemini 的 RGB-D + 位姿喂入，输出可渲染场景，M4-4 用其训练简单 video prediction。
|
||||
|
||||
---
|
||||
|
||||
## E. 室内 RGB-D 数据集与 World Model 数据 Pipeline
|
||||
|
||||
### E.1 ScanNet / ScanNet++ （ECCV 2022 / ICCV 2023）
|
||||
- **arXiv**: ScanNet++ [arXiv:2308.11417](https://arxiv.org/abs/2308.11417)
|
||||
- **核心**: 1500+ 室内场景 RGB-D + 网格 + 语义；ScanNet++ 升级到 iPhone + 激光扫描双源。
|
||||
- **对接**: **M4-1 评测基准设计直接参考**；可作为预训练数据。
|
||||
|
||||
### E.2 ARKitScenes（NeurIPS 2021）
|
||||
- **arXiv**: [arXiv:2111.08897](https://arxiv.org/abs/2111.08897)
|
||||
- **作者**: Apple
|
||||
- **核心**: iPhone/iPad LiDAR 采集的 5000+ 室内场景，含家具 bbox。
|
||||
- **对接**: **iPhone 线**（[`plans/iphone/`](../plans/iphone/)）的直接参考；M4-3 家具标注协议参照。
|
||||
|
||||
### E.3 Hypersim（ICCV 2021）
|
||||
- **arXiv**: [arXiv:2011.02523](https://arxiv.org/abs/2011.02523)
|
||||
- **作者**: Apple
|
||||
- **核心**: 高质量室内合成数据集，含真值深度/法向量/材质。
|
||||
- **对接**: 预训练 + 评测时的合成域参考。
|
||||
|
||||
### E.4 Replica / Habitat 数据集（2019–至今）
|
||||
- **arXiv**: Replica [arXiv:1906.05797](https://arxiv.org/abs/1906.05797)
|
||||
- **核心**: 18 个高质量室内 3D 场景，常用于神经建图 benchmark。
|
||||
- **对接**: M4-1 benchmark split 模板。
|
||||
|
||||
### E.5 DroidLet / Habitat 3.0 / HSSD（具身智能数据栈，2023–2024）
|
||||
- **arXiv**: Habitat 3.0 [arXiv:2310.13724](https://arxiv.org/abs/2310.13724)
|
||||
- **核心**: 大规模室内具身仿真+训练框架。
|
||||
- **对接**: **M4-2 World Model 接口**直接对接 Habitat / LeRobot 格式。
|
||||
|
||||
### E.6 World Models 综述与最新方向
|
||||
- **World Models（Ha & Schmidhuber, 2018）**: [arXiv:1803.10122](https://arxiv.org/abs/1803.10122) — 概念奠基
|
||||
- **DreamerV3（2023）**: [arXiv:2301.04104](https://arxiv.org/abs/2301.04104) — 通用世界模型
|
||||
- **Genie 2（DeepMind, 2024）**: 论文未公开，参考 [arXiv:2402.15391](https://arxiv.org/abs/2402.15391)（Genie v1）
|
||||
- **Sora 技术报告 / WorldDreamer**: WorldDreamer [arXiv:2401.09985](https://arxiv.org/abs/2401.09985)
|
||||
- **NVIDIA Cosmos（2025）**: [arXiv:2501.03575](https://arxiv.org/abs/2501.03575) — 物理感知世界模型基础模型，**直接对接本项目 M4-4 基线回灌**
|
||||
- **对接**: M4-4 训练时把本项目数据按 Cosmos / DreamerV3 接口格式喂入。
|
||||
|
||||
### E.x 主线小结
|
||||
- **数据集设计参考**：ScanNet++ + ARKitScenes 双标杆
|
||||
- **接口对齐**：LeRobot / Habitat 3.0 / Cosmos
|
||||
- **落地动作**：M4-2 数据卡参照 ScanNet++ 的 Datasheet；M4-4 用 Cosmos 小模型回灌验证。
|
||||
|
||||
---
|
||||
|
||||
## F. 与本项目 M1–M4 框架的论文映射
|
||||
|
||||
| 项目阶段 | 必读论文（粗体）+ 选读 |
|
||||
|---|---|
|
||||
| **M1 环境搂环 / 数据契约** | **ScanNet++ [2308.11417]**, ARKitScenes [2111.08897]（学其元数据 schema） |
|
||||
| **M2-1 场景级建图** | **ORB-SLAM3 [2007.11898]**, GS-SLAM [2311.11700] |
|
||||
| **M2-3 物体级细节** | **IGEV-Stereo [2303.06615]**, FoundationStereo [2501.09898] |
|
||||
| **M2-4 像素级失效** | **Marigold [2312.02145]**, Depth Anything v2 [2406.09414] |
|
||||
| **M3-3 位姿消融** | **ORB-SLAM3, VINS-Fusion [1901.03642], DPVO [2208.04726], BASALT [1904.06504]** |
|
||||
| **M3-4 深度消融** | **FoundationStereo [2501.09898], Depth Anything v2 [2406.09414], CREStereo [2203.11483]** |
|
||||
| **M3-5 融合建图** | **3DGS [2308.04079], SplaTAM [2312.02126], Photo-SLAM [2311.16728]** |
|
||||
| **M4-1 评测基准** | **ScanNet++ [2308.11417], Replica [1906.05797]**, Hypersim [2011.02523] |
|
||||
| **M4-2 数据接口** | Habitat 3.0 [2310.13724], ARKitScenes [2111.08897] |
|
||||
| **M4-4 基线回灌** | **NVIDIA Cosmos [2501.03575], DreamerV3 [2301.04104]**, World Models [1803.10122] |
|
||||
|
||||
---
|
||||
|
||||
## G. 国产团队论文清单（与 [`plans/camera/zed2i_china_alternatives.md`](../plans/camera/zed2i_china_alternatives.md) 配套）
|
||||
|
||||
特别筛选**国产团队 / 国内机构**的代表性工作，便于国产化方案的学术背书：
|
||||
|
||||
| 论文 | 团队 | arXiv | 应用点 |
|
||||
|---|---|---|---|
|
||||
| IGEV-Stereo | 华中科技大学 | [2303.06615](https://arxiv.org/abs/2303.06615) | 立体匹配 |
|
||||
| CREStereo | 旷视 Megvii | [2203.11483](https://arxiv.org/abs/2203.11483) | 立体匹配 |
|
||||
| Depth Anything v1/v2 | 字节 + 港大 | [2401.10891](https://arxiv.org/abs/2401.10891) / [2406.09414](https://arxiv.org/abs/2406.09414) | 单目深度基础模型 |
|
||||
| Metric3D v2 | 阿里 DAMO | [2404.15506](https://arxiv.org/abs/2404.15506) | 度量深度+法向量 |
|
||||
| VINS-Mono/Fusion | 港科大沈邵劼组 | [1901.03642](https://arxiv.org/abs/1901.03642) | VIO 工程标杆 |
|
||||
| DepthCrafter / StereoCrafter | 腾讯 ARC Lab | [2409.02095](https://arxiv.org/abs/2409.02095) / [2409.07447](https://arxiv.org/abs/2409.07447) | 时序一致深度 |
|
||||
| Photo-SLAM | 上海交大 | [2311.16728](https://arxiv.org/abs/2311.16728) | 3DGS SLAM |
|
||||
|
||||
**结论**：本项目所有关键技术节点（双目深度 / 单目深度 / VIO / 3DGS SLAM / 时序深度）都有**国产 SOTA 论文支撑**，国产化方案不仅是硬件层面，算法层面也完全自主可控。
|
||||
|
||||
---
|
||||
|
||||
## H. 检索方法与可复现性
|
||||
|
||||
### H.1 推荐 arXiv 检索语法
|
||||
|
||||
```text
|
||||
# 双目立体（近 2 年）
|
||||
all:"stereo matching" AND submittedDate:[202401010000 TO 202612310000]
|
||||
|
||||
# 视觉惯性 SLAM
|
||||
all:"visual inertial" OR all:"VIO" AND cat:cs.CV
|
||||
|
||||
# 3DGS SLAM
|
||||
all:"gaussian splatting" AND all:"SLAM"
|
||||
|
||||
# 世界模型 + 室内数据
|
||||
all:"world model" AND all:"indoor"
|
||||
|
||||
# ZED 相机相关工作（验证生态）
|
||||
all:"ZED 2i" OR all:"ZED stereo"
|
||||
```
|
||||
|
||||
### H.2 推荐配套工具
|
||||
|
||||
- **Papers With Code**：<https://paperswithcode.com/task/stereo-depth-estimation> / <https://paperswithcode.com/task/visual-odometry>
|
||||
- **arXiv Sanity Preserver**：<https://arxiv-sanity-lite.com>
|
||||
- **ConnectedPapers**：<https://www.connectedpapers.com>（基于一篇 seed 找邻居）
|
||||
- **本仓库脚本**：[`research/search_info.py`](search_info.py:7) 可改 query 复用
|
||||
|
||||
### H.3 引用复核清单（实施前必做）
|
||||
|
||||
> ⚠️ 本文档基于训练知识整理，部分论文 ID/年份可能有误差。**建议在 M1 阶段完成以下复核**：
|
||||
|
||||
- [ ] 用 [`research/search_info.py`](search_info.py:7) 改写查询关键词，拉取近 30 天最新 arXiv 列表
|
||||
- [ ] 对每篇"必读论文"打开 arxiv.org 链接确认存在、版本、作者
|
||||
- [ ] 对"国产团队论文清单"额外核查机构归属（中文官网/作者主页）
|
||||
- [ ] 把核对后的引文写入 [`plans/camera/zed2i_iterative_framework.md`](../plans/camera/zed2i_iterative_framework.md) 第 8 节作为权威引用源
|
||||
|
||||
---
|
||||
|
||||
## I. 与既有 review 文档的关系
|
||||
|
||||
本仓库已有的两份 world model 综述：
|
||||
|
||||
- [`research/world_models_review.md`](world_models_review.md) — 通用 world model 综述
|
||||
- [`research/physics_world_models_review.md`](physics_world_models_review.md) — 物理世界模型
|
||||
|
||||
**本文档定位**：聚焦"**前端传感与建图数据 pipeline**"，是上述两份综述的**上游数据基础**。下游训练时 → 接入 world model 综述中的方法。
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
HW[ZED 2i / Gemini 335L 硬件] --> ALG[本文档 A-D<br/>立体/VIO/深度/建图算法]
|
||||
ALG --> DATA[本文档 E<br/>数据集与接口]
|
||||
DATA --> WM[research/world_models_review.md<br/>下游训练]
|
||||
DATA --> PWM[research/physics_world_models_review.md<br/>物理推理]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## J. 推荐阅读顺序（首读 10 篇）
|
||||
|
||||
按本项目实施紧迫度排序：
|
||||
|
||||
1. **ORB-SLAM3** [2007.11898] — 替代 ZED VIO 的工程标杆
|
||||
2. **3D Gaussian Splatting** [2308.04079] — M3-5 / M4 建图核心
|
||||
3. **FoundationStereo** [2501.09898] — M3-4 深度后处理首选
|
||||
4. **Depth Anything v2** [2406.09414] — 单目深度国产基础模型
|
||||
5. **GS-SLAM / Photo-SLAM** [2311.11700 / 2311.16728] — 一体化 SLAM+渲染
|
||||
6. **ScanNet++** [2308.11417] — 数据集元数据 schema 范本
|
||||
7. **VINS-Fusion** [1901.03642] — 国产 VIO 标杆
|
||||
8. **IGEV-Stereo** [2303.06615] — 国产立体匹配
|
||||
9. **NVIDIA Cosmos** [2501.03575] — World Model 基础模型接口
|
||||
10. **DPVO** [2208.04726] — 轻量神经 VIO
|
||||
|
||||
---
|
||||
|
||||
**版本**：v0.1
|
||||
**最后更新**：2026-05-16
|
||||
**维护者**：项目规划团队
|
||||
**配套**：[`plans/camera/zed2i_stereo_imu_solution.md`](../plans/camera/zed2i_stereo_imu_solution.md) · [`zed2i_iterative_framework.md`](../plans/camera/zed2i_iterative_framework.md) · [`zed2i_china_alternatives.md`](../plans/camera/zed2i_china_alternatives.md)
|
||||
Reference in New Issue
Block a user