crawl_config.json 7.86 KB
{
  "version": "1.0.0",
  "agent": "weibo-scraper",
  "updated_at": "2026-04-13T10:35:00+08:00",
  "sources": [
    {
      "id": "weibo_pc_hotsearch",
      "name": "微博 PC 端热搜 API",
      "platform": "weibo",
      "category": "热搜榜",
      "url": "https://weibo.com/ajax/side/hotSearch",
      "method": "GET",
      "priority": 1,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
        "Accept-Encoding": "gzip, deflate, br",
        "Connection": "keep-alive",
        "Referer": "https://weibo.com"
      },
      "parser": {
        "type": "json",
        "root_path": "$.data.realtime",
        "item_selector": "*",
        "fields": {
          "word": { "path": "word", "type": "string", "required": true },
          "word_scheme": { "path": "word_scheme", "type": "string", "required": true },
          "note": { "path": "note", "type": "string", "required": false },
          "num": { "path": "num", "type": "integer", "required": true },
          "raw_hot": { "path": "raw_hot", "type": "integer", "required": false, "default": 0 },
          "rank": { "path": "rank", "type": "integer", "required": true },
          "realpos": { "path": "realpos", "type": "integer", "required": false },
          "label_name": { "path": "label_name", "type": "string", "required": false },
          "flag": { "path": "flag", "type": "integer", "required": true },
          "topic_flag": { "path": "topic_flag", "type": "integer", "required": false, "default": 0 },
          "is_ad": { "path": "is_ad", "type": "boolean", "required": false, "default": false },
          "topic_ad": { "path": "topic_ad", "type": "boolean", "required": false, "default": false }
        },
        "filters": {
          "exclude_ads": "item.is_ad != true && item.topic_ad != true",
          "valid_flag": "[0, 1, 2].includes(item.flag)"
        }
      }
    },
    {
      "id": "weibo_mobile_container",
      "name": "微博移动端容器 API",
      "platform": "weibo",
      "category": "热搜榜",
      "url": "https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot",
      "method": "GET",
      "priority": 2,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148 Safari/604.1",
        "Accept": "application/json, text/plain, */*",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
        "Referer": "https://m.weibo.cn"
      },
      "parser": {
        "type": "json",
        "root_path": "$.data.cards",
        "item_selector": "$..card_group[*]",
        "fields": {
          "word": { "path": "desc", "type": "string", "required": true },
          "word_scheme": { "path": "scheme", "type": "string", "required": false },
          "num": { "path": "desc_extr", "type": "string", "required": false },
          "rank": { "path": "itemid", "type": "string", "required": false }
        },
        "filters": {
          "has_desc": "item.desc != null && item.desc != ''"
        }
      },
      "notes": "移动端卡片结构,作为 PC 端 API 的降级备用方案。"
    },
    {
      "id": "weibo_hottopic",
      "name": "微博话题榜 API",
      "platform": "weibo",
      "category": "话题榜",
      "url": "https://weibo.com/ajax/side/hotTopic",
      "method": "GET",
      "priority": 3,
      "enabled": false,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json, text/plain, */*",
        "Referer": "https://weibo.com"
      },
      "parser": {
        "type": "json",
        "root_path": "$.data",
        "item_selector": "*",
        "fields": {
          "word": { "path": "word", "type": "string", "required": true },
          "num": { "path": "num", "type": "integer", "required": false }
        }
      },
      "notes": "部分情况下需要登录态 Cookie,当前配置中暂不启用。"
    },
    {
      "id": "weibo_news",
      "name": "微博要闻榜 API",
      "platform": "weibo",
      "category": "要闻榜",
      "url": "https://weibo.com/ajax/side/news",
      "method": "GET",
      "priority": 4,
      "enabled": false,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json, text/plain, */*",
        "Referer": "https://weibo.com"
      },
      "parser": {
        "type": "json",
        "root_path": "$.data",
        "item_selector": "*",
        "fields": {
          "word": { "path": "word", "type": "string", "required": true }
        }
      },
      "notes": "与 hotTopic 类似,可能需要登录态,默认关闭。"
    },
    {
      "id": "weibo_html_fallback",
      "name": "微博热搜榜 HTML 备用页",
      "platform": "weibo",
      "category": "热搜榜",
      "url": "https://s.weibo.com/top/summary?cate=realtimehot",
      "method": "GET",
      "priority": 5,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Referer": "https://weibo.com"
      },
      "parser": {
        "type": "html",
        "item_selector": "#pl_top_realtimehot table tbody tr",
        "fields": {
          "rank": { "selector": "td.ranktop", "type": "integer", "required": false },
          "word": { "selector": "td.td_02 a", "type": "string", "required": true, "attr": "text" },
          "word_scheme": { "selector": "td.td_02 a", "type": "string", "required": false, "attr": "href" },
          "num": { "selector": "td.td_03 span", "type": "string", "required": false, "attr": "text" }
        }
      },
      "notes": "当所有 JSON API 失败时的降级方案,通过解析 HTML DOM 获取数据。"
    }
  ],
  "rate_limit": {
    "strategy": "adaptive",
    "min_delay_seconds": 3,
    "max_delay_seconds": 5,
    "requests_per_minute": 20,
    "cooldown_minutes": 1,
    "retry_on_rate_limit": {
      "enabled": true,
      "max_retries": 2,
      "backoff_seconds": 30
    }
  },
  "user_agent_pool": [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
  ],
  "validation": {
    "expected_content_type": "application/json",
    "fallback_content_types": ["text/html"],
    "success_indicator": {
      "path": "$.ok",
      "value": 1
    }
  },
  "output": {
    "schema": "weibo_hotsearch_v1",
    "required_fields": [
      "source.platform",
      "source.category",
      "source.hot_id",
      "content.title",
      "engagement.likes",
      "metrics.hot_rank",
      "metrics.trend",
      "metadata.source_url",
      "metadata.crawl_time"
    ],
    "include_raw_data": true
  },
  "compliance": {
    "robots_txt": "https://weibo.com/robots.txt",
    "crawl_delay_seconds": 10,
    "exclude_private_data": true,
    "exclude_ads": true,
    "data_retention_days": 90
  }
}