crawl_config.json 1.1 KB
{
  "name": "hot-topic-researcher",
  "version": "1.0",
  "description": "热点研究员 - 多源热点抓取与深度研究",
  "data_sources": [
    {
      "name": "Hacker News",
      "type": "html",
      "url": "https://news.ycombinator.com/",
      "selector": ".titleline > a",
      "encoding": "utf-8",
      "interval": 300,
      "category": "tech"
    },
    {
      "name": "BBC News",
      "type": "rss",
      "url": "https://feeds.bbci.co.uk/news/rss.xml",
      "item_path": "//item/title",
      "encoding": "utf-8",
      "interval": 600,
      "category": "world"
    },
    {
      "name": "Google News US",
      "type": "rss",
      "url": "https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB?hl=en-US&gl=US",
      "item_path": "//item/title",
      "encoding": "utf-8",
      "interval": 600,
      "category": "general"
    }
  ],
  "rate_limit": {
    "min_interval_ms": 2000,
    "max_retries": 3,
    "timeout_ms": 10000
  },
  "output": {
    "format": "jsonl",
    "fields": ["title", "source", "url", "category", "timestamp", "hotness_score"]
  }
}