{
  "project": "weibo-scraper",
  "description": "Respectful crawler configuration for Weibo hot search/trending topics",
  "respect_robots_txt": true,
  "request_interval_seconds": 3,
  "max_retries": 3,
  "timeout_seconds": 15,
  "headers": {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Accept-Encoding": "gzip, deflate, br",
    "Referer": "https://s.weibo.com/",
    "Connection": "keep-alive"
  },
  "sources": [
    {
      "name": "s_weibo_top_summary",
      "priority": 1,
      "enabled": true,
      "url": "https://s.weibo.com/top/summary",
      "method": "GET",
      "type": "html",
      "parser": {
        "selector": "table tbody tr",
        "fields": {
          "rank": {
            "selector": "td.td-01.ranktop",
            "type": "text",
            "transform": "trim"
          },
          "keyword": {
            "selector": "td.td-02 a",
            "type": "text",
            "transform": "trim"
          },
          "heat_score": {
            "selector": "td.td-02 span",
            "type": "text",
            "transform": "trim"
          },
          "tag": {
            "selector": "td.td-03 i",
            "type": "text",
            "transform": "trim",
            "optional": true
          },
          "source_url": {
            "selector": "td.td-02 a",
            "type": "attribute",
            "attribute": "href",
            "base_url": "https://s.weibo.com"
          }
        }
      },
      "rate_limit": {
        "domain": "s.weibo.com",
        "interval_seconds": 3,
        "max_requests_per_minute": 15
      }
    },
    {
      "name": "weibo_ajax_hotsearch",
      "priority": 2,
      "enabled": true,
      "url": "https://weibo.com/ajax/side/hotSearch",
      "method": "GET",
      "type": "json",
      "parser": {
        "root": "data.realtime",
        "fields": {
          "rank": {
            "path": "rank",
            "type": "number"
          },
          "keyword": {
            "path": "word",
            "type": "text",
            "transform": "trim"
          },
          "heat_score": {
            "path": "raw_hot",
            "type": "number"
          },
          "category": {
            "path": "category",
            "type": "text",
            "optional": true
          },
          "subject_label": {
            "path": "subject_label",
            "type": "text",
            "optional": true
          },
          "icon_desc": {
            "path": "icon_desc",
            "type": "text",
            "optional": true
          },
          "source_url": {
            "static": "https://weibo.com/ajax/side/hotSearch"
          }
        }
      },
      "rate_limit": {
        "domain": "weibo.com",
        "interval_seconds": 3,
        "max_requests_per_minute": 15
      },
      "notes": "Requires valid authenticated cookies (SUB, SUBP) to return full data."
    },
    {
      "name": "m_weibo_container_api",
      "priority": 3,
      "enabled": true,
      "url": "https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot",
      "method": "GET",
      "type": "json",
      "headers": {
        "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1",
        "Referer": "https://m.weibo.cn/p/index?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot",
        "X-Requested-With": "XMLHttpRequest"
      },
      "parser": {
        "root": "data.cards[0].card_group",
        "fields": {
          "rank": {
            "path": "itemid",
            "type": "text",
            "optional": true
          },
          "keyword": {
            "path": "desc",
            "type": "text",
            "transform": "trim"
          },
          "heat_score": {
            "path": "desc_extr",
            "type": "text",
            "transform": "trim",
            "optional": true
          },
          "pic_url": {
            "path": "pic",
            "type": "text",
            "optional": true
          },
          "source_url": {
            "static": "https://m.weibo.cn/api/container/getIndex?containerid=106003type%3D25%26t%3D3%26disable_hot%3D1%26filter_type%3Drealtimehot"
          }
        }
      },
      "rate_limit": {
        "domain": "m.weibo.cn",
        "interval_seconds": 3,
        "max_requests_per_minute": 15
      },
      "notes": "Mobile API. May require anti-bot tokens or cookie rotation."
    },
    {
      "name": "github_snapshot_fallback",
      "priority": 4,
      "enabled": true,
      "url": "https://raw.githubusercontent.com/justjavac/weibo-trending-hot-search/master/raw/2026-04-13.json",
      "method": "GET",
      "type": "json",
      "parser": {
        "root": "",
        "fields": {
          "rank": {
            "path": "rank",
            "type": "number"
          },
          "keyword": {
            "path": "title",
            "type": "text",
            "transform": "trim"
          },
          "heat_score": {
            "path": "hot",
            "type": "text",
            "optional": true
          },
          "source_url": {
            "static": "https://raw.githubusercontent.com/justjavac/weibo-trending-hot-search/master/raw/2026-04-13.json"
          }
        }
      },
      "rate_limit": {
        "domain": "raw.githubusercontent.com",
        "interval_seconds": 2,
        "max_requests_per_minute": 20
      },
      "notes": "Community snapshot. Good for historical seed data. Update URL date for different days."
    }
  ],
  "output": {
    "required_fields": [
      "keyword",
      "source_url",
      "crawl_time"
    ],
    "optional_fields": [
      "rank",
      "heat_score",
      "category",
      "tag",
      "pic_url",
      "subject_label",
      "icon_desc"
    ]
  }
}