crawl_config_v5.json 6.29 KB
{
  "task": "T-mtd7r60r",
  "agent": "innovator-hunt (Hunter)",
  "round": "retrain v5 HUNT (feishu official first-hand + 2 community skills)",
  "crawl_date": "2026-08-28",
  "crawl_window_utc": "2026-08-28T17:05:00Z - 2026-08-28T17:29:00Z",
  "produces": [
    "train/hunt/knowhow_v5.md",
    "train/hunt/sources_v5.json",
    "train/hunt/crawl_config_v5.json",
    "train/hunt/cache_v5/  (全部原文落盘 + _cache_manifest.json)"
  ],
  "tools": {
    "feishu": "/app/bin/lark-cli docs +fetch (--scope outline 先拿大纲, --doc-format markdown 拿正文)",
    "github_tree": "curl https://api.github.com/repos/<owner>/<repo>/git/trees/<branch>?recursive=1",
    "github_raw": "curl https://raw.githubusercontent.com/<owner>/<repo>/<branch>/<path>",
    "hash": "node crypto.createHash('sha256') 本地计算",
    "forbidden": "WebFetch / curl 直抓 open.feishu.cn(登录墙);cat /home/node/.lark-cli/config.json"
  },
  "rate_limit": {
    "per_site_min_interval_seconds": 2,
    "applied": "raw.githubusercontent.com 26 次请求每次 sleep 2;api.github.com 3 次(tree x2 + repo x1)间隔不小于 2s",
    "robots_txt": {
      "checked": true,
      "result": "raw.githubusercontent.com 与 github.com 的 /robots.txt 抓取超时(--max-time 40 内无响应,本轮两域 robots 均未取得)",
      "handling": "按「无法复检」继续推进,不据此豁免任何礼貌抓取约束:仅抓取公开仓库的文档类文件、无登录绕过、无并发轰炸、单站点间隔 >= 2s;已在 sources_v5.json negative_findings 登记"
    }
  },
  "request_policy": {
    "curl_flags": "--retry 1 --max-time 40",
    "retry_rules": "失败 -> 换镜像域一次(GitHub 走 cdn.jsdelivr.net/gh)-> WebFetch 一次 -> 标「无法复检」继续;禁止重跑整轮",
    "http_200_verification": "逐文件按 <title> 抽取 + 403/404/400 错误体前缀匹配 + 最小字节数过滤,防假 200;两条 400B/428B 的超短文件逐字读过确认为真实正文(profile 模板本就极短),非错误体"
  },
  "targets": [
    {
      "id": "A1",
      "name": "飞书官方 · MiniMax H3 模型 - 使用手册",
      "kind": "feishu_wiki",
      "url": "https://vrfi1sk8a0.feishu.cn/wiki/FIWjwgL33ipnkekzk30crmKUnIh",
      "grade": "A",
      "parse": [
        "docs +fetch --scope outline  拿标题树 + block id",
        "docs +fetch --doc-format markdown  拿正文全文(147058B 一次取全,无需分段)"
      ],
      "headers": "lark-cli bot 身份 + tenant token(凭据在 /home/node/.lark-cli/config.json,禁止读取/打印)",
      "expected_fields": ["document.content", "document.revision_id", "document.document_id"],
      "observed": { "revision_id": 1297, "document_id": "ZRFZd89oMoGTyrx2JXWcpPq1n7f", "content_bytes": 147058 },
      "cache": "cache_v5/feishu_doc1_H3使用手册_FIWjwgL33ipnkekzk30crmKUnIh.md",
      "cache_sha256": "48a7a693c70b4e8fd6567a2f4d3ce049ec3966577a3b2ee0e4db556c752b98b6"
    },
    {
      "id": "A2",
      "name": "飞书官方 · MiniMax H3 - 视频扩写 Prompt 写作全能指南",
      "kind": "feishu_wiki",
      "url": "https://vrfi1sk8a0.feishu.cn/wiki/Vztzwstuqi3MQKkxs8QcGmxinxe",
      "grade": "A",
      "parse": [
        "docs +fetch --scope outline  拿标题树 + block id",
        "docs +fetch --doc-format markdown  拿正文全文(41607B 一次取全,无需分段)"
      ],
      "headers": "同 A1",
      "observed": { "revision_id": 252, "document_id": "DhOudwGrAoykcRxjBOoc4pLonTf", "content_bytes": 41607 },
      "cache": "cache_v5/feishu_doc2_Prompt写作全能指南_Vztzwstuqi3MQKkxs8QcGmxinxe.md",
      "cache_sha256": "caa721f6140cc06f1450c961d38372059191fc5e106cef787249a8098f8c0e06"
    },
    {
      "id": "B1",
      "name": "GitHub 社区 skill 仓 · chiphoton/MiniMax-H3-Codex-Drama",
      "kind": "github_repo",
      "repo": "chiphoton/MiniMax-H3-Codex-Drama",
      "default_branch": "main",
      "grade": "B",
      "tree_paths_total": 214,
      "fetched_files": 18,
      "selection_rule": "只抓提示词规范 / 模板 / 工作流 / 示例等文本核心件,跳过 mp4/webp/png/jpg/svg/二进制与 ComfyUI workflow json",
      "fetch_order": [
        "api.github.com git/trees/main?recursive=1  ->  全量路径清单(78321B)",
        "raw.githubusercontent.com 逐文件抓取(18 文件,全部 HTTP 200)"
      ],
      "cache_root": "cache_v5/github/chiphoton/"
    },
    {
      "id": "B2",
      "name": "GitHub 社区 skill 仓 · TFboy1/oh-my-minimaxh3-director",
      "kind": "github_repo",
      "repo": "TFboy1/oh-my-minimaxh3-director",
      "default_branch": "master",
      "grade": "B (工程链路) / C (WenWu 自创导演词汇)",
      "tree_paths_total": 33,
      "fetched_files": 8,
      "fetch_order": [
        "git/trees/main?recursive=1  ->  404(default_branch 实为 master)",
        "api.github.com repos/<repo>  ->  读 default_branch=master",
        "git/trees/master?recursive=1  ->  全量路径清单(12710B)",
        "raw.githubusercontent.com 逐文件抓取(8 文件,全部 HTTP 200)"
      ],
      "lesson": "GitHub API tree 对非 main 默认分支会 404;先读 repos/<repo>.default_branch 再拉 tree",
      "cache_root": "cache_v5/github/tfboy1/"
    },
    {
      "id": "REUSE",
      "name": "v4 已固化信源(复用禁复抓)",
      "kind": "reuse",
      "rule": "MiniMax-AI/MiniMax-H3 官方仓 + platform.minimax.io 的 25 条信源本轮一律不重抓;需要引用时在 sources_v5.json 的 reused_from_v4 区块登记,逐字出处走 train/hunt/sources_v4.json 的 quotes(含行号)",
      "refetched": false
    }
  ],
  "output_rules": [
    "所有产物一律 _v5 新文件,禁止覆盖 v1/v2/v3/v4 的任何文件",
    "每条 A/B 级数据必须带 source_url 与 crawl_time",
    "官方一手结论标注 A 级;社区实践标注 B/C 级且与官方冲突处只登记不采信"
  ],
  "acceptance_checklist": [
    { "item": "sources_v5.json 含两篇飞书文档且带 sha256 + revision_id", "met": true },
    { "item": "缓存原文可被派单方逐字回查(cache_v5/ 34 文件 + _cache_manifest.json)", "met": true },
    { "item": "knowhow_v5.md 每条结论标注信源级别", "met": true },
    { "item": "v1-v4 产物 mtime / bytes 零改动", "met": true, "evidence": "见 sources_v5.json.verification_of_no_overwrite" }
  ]
}