crawl_config.json
1.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
{
"name": "hot-topic-researcher",
"version": "1.0",
"description": "热点研究员 - 多源热点抓取与深度研究",
"data_sources": [
{
"name": "Hacker News",
"type": "html",
"url": "https://news.ycombinator.com/",
"selector": ".titleline > a",
"encoding": "utf-8",
"interval": 300,
"category": "tech"
},
{
"name": "BBC News",
"type": "rss",
"url": "https://feeds.bbci.co.uk/news/rss.xml",
"item_path": "//item/title",
"encoding": "utf-8",
"interval": 600,
"category": "world"
},
{
"name": "Google News US",
"type": "rss",
"url": "https://news.google.com/rss/topics/CAAqJggKIiBDQkFTRWdvSUwyMHZNRGx1YlY4U0FtVnVHZ0pWVXlnQVAB?hl=en-US&gl=US",
"item_path": "//item/title",
"encoding": "utf-8",
"interval": 600,
"category": "general"
}
],
"rate_limit": {
"min_interval_ms": 2000,
"max_retries": 3,
"timeout_ms": 10000
},
"output": {
"format": "jsonl",
"fields": ["title", "source", "url", "category", "timestamp", "hotness_score"]
}
}