crawl_config.json
2.72 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
{
"meta": {
"task": "T-mtcng5rb deep-research Hunt Phase 1",
"agent": "innovator-hunt (Hunter)",
"generated_at": "2026-08-28T08:40:00+08:00",
"updated_at": "2026-08-28T08:37Z (run2 增量：网络出口为时变，reachable/unreachable 两个名单不能当常量)",
"purpose": "两轮实测可用的外网抓取策略（SOP 固化），供 Digester / 后续轮次复用"
},
"http": {
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120 Safari/537.36",
"curl_template": "curl -sS -L -A \"$UA\" -o <out> -w \"%{http_code}\" --max-time 40 --retry 1 <url>",
"per_request_timeout_s": 40,
"retry": 1,
"note": "任务书原写 --retry 3 --retry-all-errors --max-time 60，实测在慢主机会把单条 URL 放大到 4 分钟以上（本轮先超时一次），已收敛为 max-time 40 + retry 1 + 失败换镜像域"
},
"frequency": { "inter_request_delay_s": 2, "robots_txt_respect": true },
"reachable_hosts": [
"www.anthropic.com",
"code.claude.com / docs.claude.com（部分页为区域封锁页，需按 <title> 复核）",
"arxiv.org",
"mediabiasfactcheck.com",
"cdn.jsdelivr.net（GitHub 镜像）",
"raw.githubusercontent.com / api.github.com（run2 直连恢复，且镜像保真度已由 4 文件 sha256 双向验证；应先直连、失败再换镜像）",
"blog.google / hapgood.us / fullfact.org / docs.perplexity.ai 除外（run2 实测可达域名补录）"
],
"unreachable_hosts": [
"en.wikipedia.org", "support.google.com",
"www.perplexity.ai 与 docs.perplexity.ai（两轮均超时，标「无法复检」）",
"webliteracy.pressbooks.com", "openai.com（两轮均 403，改走官方 cookbook 仓库镜像）",
"library.csuchico.edu（两轮均 404，CRAAP 官方页）",
"blog.langchain.dev（run2 实测 404）"
],
"github_mirror": {
"pattern": "https://cdn.jsdelivr.net/gh/<org>/<repo>@<branch>/<path>",
"verified_examples": [
"https://cdn.jsdelivr.net/gh/assafelovic/gpt-researcher@master/README.md",
"https://cdn.jsdelivr.net/gh/stanford-oval/storm@main/README.md",
"https://cdn.jsdelivr.net/gh/anthropics/skills@main/README.md"
]
},
"validation_rules": [
"下载后必须校验 <title> 或关键关键词，HTTP 200 不代表内容有效（本轮 3 例假 200：2 例区域封锁页、1 例软 404）",
"同时抓到的两个 URL 若 sha256 一致即为镜像/错误页，只记其一并进 blocked_sources",
"每条缓存落盘即算 sha256 并写入 sources_v1.json，后续轮次只读缓存禁复抓"
],
"target_output": {
"dir": "/app/system/agents/deep-research/train/hunt/",
"files": ["knowhow_v1.md", "sources_v1.json", "crawl_config.json", "sources_raw/*"]
}
}