feat: add scripts for fetching arXiv papers and generating structured reviews
Sync to site1 / sync (push) Has been cancelled
Sync to site1 / sync (push) Has been cancelled
- Implemented `fetch_crowdroom_papers.py` to scrape arXiv papers related to CrowdRoom with customizable query topics and output options. - Created `gen_review_from_json.py` to render structured markdown reviews from search results JSON, including statistics and insights on papers and GitHub repositories. - Developed `search_info.py` for thematic searches on arXiv and GitHub, supporting HTTP/SOCKS5 proxies and structured output.
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
# Research Data
|
||||
|
||||
本目录存放论文调研的原始数据文件和缓存。
|
||||
|
||||
## 文件说明
|
||||
|
||||
| 文件 | 说明 |
|
||||
|------|------|
|
||||
| `search_results.json` | arXiv + GitHub 历史检索缓存(ZED2i 相关,含 2026-04/05 最新论文) |
|
||||
| `crowdroom_papers_raw.json` | CrowdRoom 专属 arXiv 抓取结果(当前因 429 限速为空) |
|
||||
| `lyra2_paper.txt` | Lyra 2.0 论文全文(arXiv:2604.13036,NVIDIA,2026-04) |
|
||||
| `lyra2_paper.pdf` | Lyra 2.0 论文 PDF 原文 |
|
||||
|
||||
## 注意事项
|
||||
|
||||
- `search_results.json` 是重要的历史缓存,请勿随意删除
|
||||
- 数据文件不纳入 git 版本控制(见 `.gitignore`)
|
||||
- 重新抓取请使用 `../tools/` 目录下的脚本
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"meta": {
|
||||
"generated_at": "2026-05-20T16:30:00+0800",
|
||||
"proxy": "direct + http://127.0.0.1:6984 fallback",
|
||||
"max_results_per_topic": 20,
|
||||
"max_days": 365,
|
||||
"sort": "submittedDate",
|
||||
"topic_count": 10,
|
||||
"total_raw": 0,
|
||||
"total_after_date_filter": 0,
|
||||
"total_unique": 0,
|
||||
"fetch_status": "FAILED_RATE_LIMIT",
|
||||
"fetch_error": "All requests blocked by arXiv API with HTTP 429 'Rate exceeded'. Host IP and proxy IP both flagged by arXiv's rate limiter (window > 2 hours). Tried direct connection, http://127.0.0.1:6984 proxy, IPv4, IPv6 — all returned 429 immediately or after exponential backoff (30/60/90/120/150/180s). Two topics ran 6 retries each (~15 min) and still failed.",
|
||||
"fetch_attempts": [
|
||||
{"time": "2026-05-20T13:18 CST", "topic": "roomplan_arkit", "result": "0 papers; 4x 429 after 30/60/90/120s backoff"},
|
||||
{"time": "2026-05-20T16:32 CST (background)", "topic": "roomplan_arkit", "result": "0 papers; 4x 429 + 2x SSL timeout"},
|
||||
{"time": "2026-05-20T16:32 CST (background)", "topic": "gaussian_splatting_indoor", "result": "0 papers; 6x 429 (30/60/90/120/150/180s)"},
|
||||
{"time": "2026-05-20T16:50 CST (probe)", "topic": "all:RoomPlan", "result": "HTTP 429 from both direct & proxy"}
|
||||
],
|
||||
"recommended_retry": "Wait >= 2 hours; ideally execute between 04:00-09:00 Beijing Time (arXiv off-peak); or switch to residential proxy. See research/crowdroom_related_papers_2026.md Appendix B."
|
||||
},
|
||||
"topics_meta": {
|
||||
"roomplan_arkit": {"zh": "1. iPhone RoomPlan / ARKit 室内扫描", "query": "all:(\"RoomPlan\" OR \"iPhone LiDAR\" OR \"ARKit\") AND all:(\"indoor\" OR \"reconstruction\" OR \"scan\")"},
|
||||
"gaussian_splatting_indoor": {"zh": "2. 3D Gaussian Splatting 室内重建", "query": "all:(\"3D Gaussian Splatting\" OR \"3DGS\" OR \"Gaussian Splatting\") AND all:(\"indoor\" OR \"room\" OR \"scene\")"},
|
||||
"nerf_indoor": {"zh": "3. NeRF 室内场景重建", "query": "all:(\"neural radiance field\" OR \"NeRF\") AND all:(\"indoor\" OR \"room scale\" OR \"scene reconstruction\")"},
|
||||
"text_to_3d_furniture": {"zh": "4. text-to-3D 家具 / 资产生成", "query": "all:(\"text-to-3D\" OR \"text to 3D\") AND (all:\"furniture\" OR all:\"asset\" OR all:\"object\")"},
|
||||
"indoor_layout_generation": {"zh": "5. 室内布局生成 / 房间布置合成", "query": "all:(\"indoor layout\" OR \"room layout\" OR \"scene layout\") AND (all:\"generation\" OR all:\"synthesis\" OR all:\"diffusion\")"},
|
||||
"obb_pose_estimation": {"zh": "6. 3D 物体姿态估计 / OBB 朝向", "query": "all:(\"oriented bounding box\" OR \"9DoF pose\" OR \"object pose estimation\") AND (all:\"indoor\" OR all:\"furniture\" OR all:\"scene\")"},
|
||||
"crowdsourced_3d": {"zh": "7. 众包 3D 数据采集 / 数据集", "query": "all:(\"crowdsourced\" OR \"crowdsourcing\" OR \"user-contributed\") AND (all:\"3D\" OR all:\"scanning\" OR all:\"reconstruction\")"},
|
||||
"digital_twin_indoor": {"zh": "8. 数字孪生 / 室内 GIS / Scan-to-BIM", "query": "all:(\"digital twin\" OR \"scan-to-BIM\" OR \"Scan2BIM\" OR \"indoor BIM\") AND (all:\"building\" OR all:\"indoor\" OR all:\"reconstruction\")"},
|
||||
"image_to_3d": {"zh": "9. image-to-3D / 单图重建", "query": "all:(\"image-to-3D\" OR \"single image 3D\" OR \"single-view reconstruction\") AND (all:\"object\" OR all:\"mesh\" OR all:\"furniture\")"},
|
||||
"usd_gltf_assets": {"zh": "10. USD / glTF / 3D 资产标准化", "query": "all:(\"glTF\" OR \"USD\" OR \"Universal Scene Description\" OR \"OpenUSD\") AND (all:\"3D\" OR all:\"asset\" OR all:\"scene\")"}
|
||||
},
|
||||
"arxiv": {
|
||||
"roomplan_arkit": [],
|
||||
"gaussian_splatting_indoor": [],
|
||||
"nerf_indoor": [],
|
||||
"text_to_3d_furniture": [],
|
||||
"indoor_layout_generation": [],
|
||||
"obb_pose_estimation": [],
|
||||
"crowdsourced_3d": [],
|
||||
"digital_twin_indoor": [],
|
||||
"image_to_3d": [],
|
||||
"usd_gltf_assets": []
|
||||
},
|
||||
"stats": {
|
||||
"roomplan_arkit": {"raw": 0, "after_date_filter": 0, "status": "429 rate-limit"},
|
||||
"gaussian_splatting_indoor": {"raw": 0, "after_date_filter": 0, "status": "429 rate-limit"},
|
||||
"nerf_indoor": {"raw": 0, "after_date_filter": 0, "status": "not-attempted (killed at topic 3)"},
|
||||
"text_to_3d_furniture": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"indoor_layout_generation": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"obb_pose_estimation": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"crowdsourced_3d": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"digital_twin_indoor": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"image_to_3d": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"},
|
||||
"usd_gltf_assets": {"raw": 0, "after_date_filter": 0, "status": "not-attempted"}
|
||||
},
|
||||
"unique_papers": []
|
||||
}
|
||||
Binary file not shown.
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user