crawl_config.json 2.62 KB
{
  "version": "1.0",
  "skill": "researcher",
  "description": "Researcher 技能爬取配置 - 深度研究数据源",
  "crawl_rules": {
    "global": {
      "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
      "timeout": 30000,
      "retry": 3,
      "delay_between_requests": 3000
    }
  },
  "targets": [
    {
      "name": "豆瓣电影短剧榜",
      "source": "douban",
      "base_url": "https://movie.douban.com",
      "endpoints": [
        "/tag/短剧/",
        "/celebrity/followed/",
        "/review/latest/"
      ],
      "selectors": {
        "list_item": ".item",
        "title": ".title",
        "rating": ".rating_num",
        "info": ".info"
      },
      "priority": "P0",
      "max_items": 50
    },
    {
      "name": "腾讯视频短剧",
      "source": "tencent",
      "base_url": "https://v.qq.com",
      "endpoints": [
        "/channel/shortdrama",
        "/rank/shortdrama/"
      ],
      "selectors": {
        "list_item": ".mod-item",
        "title": ".title",
        "play_count": ".num"
      },
      "priority": "P0",
      "max_items": 30
    },
    {
      "name": "爱奇艺短剧",
      "source": "iqiyi",
      "base_url": "https://www.iqiyi.com",
      "endpoints": [
        "/dianying/list?channel=shortplay",
        "/v list?category=shortplay"
      ],
      "selectors": {
        "list_item": ".qy-item",
        "title": ".qy-title",
        "tags": ".tag-list"
      },
      "priority": "P1",
      "max_items": 30
    },
    {
      "name": "云合数据短剧榜",
      "source": "yunhe",
      "base_url": "https://www.yunhe.cn",
      "endpoints": [
        "/short-drama/ranking"
      ],
      "selectors": {
        "list_item": ".ranking-item",
        "title": ".drama-title",
        "views": ".view-count"
      },
      "priority": "P1",
      "max_items": 20
    },
    {
      "name": "知乎剧本创作",
      "source": "zhihu",
      "base_url": "https://www.zhihu.com",
      "endpoints": [
        "/topic/剧本创作",
        "/topic/人物塑造"
      ],
      "selectors": {
        "list_item": ".List-item",
        "title": ".ContentItem-title",
        "excerpt": ".RichText"
      },
      "priority": "P2",
      "max_items": 20
    }
  ],
  "data_fields": {
    "required": [
      "title",
      "source_url",
      "crawl_time"
    ],
    "optional": [
      "description",
      "rating",
      "tags",
      "author",
      "views"
    ]
  },
  "storage": {
    "format": "jsonl",
    "path": "./seed_data.json"
  },
  "notes": "遵守 robots.txt,单站点请求间隔 ≥ 3秒"
}