crawl_config_v5.json
6.29 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
{
"task": "T-mtd7r60r",
"agent": "innovator-hunt (Hunter)",
"round": "retrain v5 HUNT (feishu official first-hand + 2 community skills)",
"crawl_date": "2026-08-28",
"crawl_window_utc": "2026-08-28T17:05:00Z - 2026-08-28T17:29:00Z",
"produces": [
"train/hunt/knowhow_v5.md",
"train/hunt/sources_v5.json",
"train/hunt/crawl_config_v5.json",
"train/hunt/cache_v5/ (全部原文落盘 + _cache_manifest.json)"
],
"tools": {
"feishu": "/app/bin/lark-cli docs +fetch (--scope outline 先拿大纲, --doc-format markdown 拿正文)",
"github_tree": "curl https://api.github.com/repos/<owner>/<repo>/git/trees/<branch>?recursive=1",
"github_raw": "curl https://raw.githubusercontent.com/<owner>/<repo>/<branch>/<path>",
"hash": "node crypto.createHash('sha256') 本地计算",
"forbidden": "WebFetch / curl 直抓 open.feishu.cn(登录墙);cat /home/node/.lark-cli/config.json"
},
"rate_limit": {
"per_site_min_interval_seconds": 2,
"applied": "raw.githubusercontent.com 26 次请求每次 sleep 2;api.github.com 3 次(tree x2 + repo x1)间隔不小于 2s",
"robots_txt": {
"checked": true,
"result": "raw.githubusercontent.com 与 github.com 的 /robots.txt 抓取超时(--max-time 40 内无响应,本轮两域 robots 均未取得)",
"handling": "按「无法复检」继续推进,不据此豁免任何礼貌抓取约束:仅抓取公开仓库的文档类文件、无登录绕过、无并发轰炸、单站点间隔 >= 2s;已在 sources_v5.json negative_findings 登记"
}
},
"request_policy": {
"curl_flags": "--retry 1 --max-time 40",
"retry_rules": "失败 -> 换镜像域一次(GitHub 走 cdn.jsdelivr.net/gh)-> WebFetch 一次 -> 标「无法复检」继续;禁止重跑整轮",
"http_200_verification": "逐文件按 <title> 抽取 + 403/404/400 错误体前缀匹配 + 最小字节数过滤,防假 200;两条 400B/428B 的超短文件逐字读过确认为真实正文(profile 模板本就极短),非错误体"
},
"targets": [
{
"id": "A1",
"name": "飞书官方 · MiniMax H3 模型 - 使用手册",
"kind": "feishu_wiki",
"url": "https://vrfi1sk8a0.feishu.cn/wiki/FIWjwgL33ipnkekzk30crmKUnIh",
"grade": "A",
"parse": [
"docs +fetch --scope outline 拿标题树 + block id",
"docs +fetch --doc-format markdown 拿正文全文(147058B 一次取全,无需分段)"
],
"headers": "lark-cli bot 身份 + tenant token(凭据在 /home/node/.lark-cli/config.json,禁止读取/打印)",
"expected_fields": ["document.content", "document.revision_id", "document.document_id"],
"observed": { "revision_id": 1297, "document_id": "ZRFZd89oMoGTyrx2JXWcpPq1n7f", "content_bytes": 147058 },
"cache": "cache_v5/feishu_doc1_H3使用手册_FIWjwgL33ipnkekzk30crmKUnIh.md",
"cache_sha256": "48a7a693c70b4e8fd6567a2f4d3ce049ec3966577a3b2ee0e4db556c752b98b6"
},
{
"id": "A2",
"name": "飞书官方 · MiniMax H3 - 视频扩写 Prompt 写作全能指南",
"kind": "feishu_wiki",
"url": "https://vrfi1sk8a0.feishu.cn/wiki/Vztzwstuqi3MQKkxs8QcGmxinxe",
"grade": "A",
"parse": [
"docs +fetch --scope outline 拿标题树 + block id",
"docs +fetch --doc-format markdown 拿正文全文(41607B 一次取全,无需分段)"
],
"headers": "同 A1",
"observed": { "revision_id": 252, "document_id": "DhOudwGrAoykcRxjBOoc4pLonTf", "content_bytes": 41607 },
"cache": "cache_v5/feishu_doc2_Prompt写作全能指南_Vztzwstuqi3MQKkxs8QcGmxinxe.md",
"cache_sha256": "caa721f6140cc06f1450c961d38372059191fc5e106cef787249a8098f8c0e06"
},
{
"id": "B1",
"name": "GitHub 社区 skill 仓 · chiphoton/MiniMax-H3-Codex-Drama",
"kind": "github_repo",
"repo": "chiphoton/MiniMax-H3-Codex-Drama",
"default_branch": "main",
"grade": "B",
"tree_paths_total": 214,
"fetched_files": 18,
"selection_rule": "只抓提示词规范 / 模板 / 工作流 / 示例等文本核心件,跳过 mp4/webp/png/jpg/svg/二进制与 ComfyUI workflow json",
"fetch_order": [
"api.github.com git/trees/main?recursive=1 -> 全量路径清单(78321B)",
"raw.githubusercontent.com 逐文件抓取(18 文件,全部 HTTP 200)"
],
"cache_root": "cache_v5/github/chiphoton/"
},
{
"id": "B2",
"name": "GitHub 社区 skill 仓 · TFboy1/oh-my-minimaxh3-director",
"kind": "github_repo",
"repo": "TFboy1/oh-my-minimaxh3-director",
"default_branch": "master",
"grade": "B (工程链路) / C (WenWu 自创导演词汇)",
"tree_paths_total": 33,
"fetched_files": 8,
"fetch_order": [
"git/trees/main?recursive=1 -> 404(default_branch 实为 master)",
"api.github.com repos/<repo> -> 读 default_branch=master",
"git/trees/master?recursive=1 -> 全量路径清单(12710B)",
"raw.githubusercontent.com 逐文件抓取(8 文件,全部 HTTP 200)"
],
"lesson": "GitHub API tree 对非 main 默认分支会 404;先读 repos/<repo>.default_branch 再拉 tree",
"cache_root": "cache_v5/github/tfboy1/"
},
{
"id": "REUSE",
"name": "v4 已固化信源(复用禁复抓)",
"kind": "reuse",
"rule": "MiniMax-AI/MiniMax-H3 官方仓 + platform.minimax.io 的 25 条信源本轮一律不重抓;需要引用时在 sources_v5.json 的 reused_from_v4 区块登记,逐字出处走 train/hunt/sources_v4.json 的 quotes(含行号)",
"refetched": false
}
],
"output_rules": [
"所有产物一律 _v5 新文件,禁止覆盖 v1/v2/v3/v4 的任何文件",
"每条 A/B 级数据必须带 source_url 与 crawl_time",
"官方一手结论标注 A 级;社区实践标注 B/C 级且与官方冲突处只登记不采信"
],
"acceptance_checklist": [
{ "item": "sources_v5.json 含两篇飞书文档且带 sha256 + revision_id", "met": true },
{ "item": "缓存原文可被派单方逐字回查(cache_v5/ 34 文件 + _cache_manifest.json)", "met": true },
{ "item": "knowhow_v5.md 每条结论标注信源级别", "met": true },
{ "item": "v1-v4 产物 mtime / bytes 零改动", "met": true, "evidence": "见 sources_v5.json.verification_of_no_overwrite" }
]
}