crawl_config.json
1.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
{
"name": "trend-hunter",
"description": "热点抓取爬虫配置",
"version": "1.0.0",
"data_sources": [
{
"name": "百度热搜",
"url": "https://top.baidu.com/board?tab=realtime",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml"
},
"parser": {
"type": "html",
"extract_method": "regex",
"pattern": "(\\d+)\\s+(\\d+)\\s+(.*?)热搜指数\\s+(.*?)(?=\\d+\\s+\\d+\\s+|</div>)"
},
"rate_limit": 3,
"enabled": true
},
{
"name": "B站热门",
"url": "https://api.bilibili.com/x/web-interface/popular?ps=50&pn=1",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0",
"Referer": "https://www.bilibili.com"
},
"parser": {
"type": "json",
"path": "data.list"
},
"rate_limit": 2,
"enabled": true
},
{
"name": "今日头条热榜",
"url": "https://www.toutiao.com/hot-event/hot-board/",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0"
},
"parser": {
"type": "json",
"path": "data"
},
"rate_limit": 3,
"enabled": true
},
{
"name": "腾讯新闻热榜",
"url": "https://r.inews.qq.com/gw/event/hot_ranking",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0"
},
"parser": {
"type": "json",
"path": "idlist"
},
"rate_limit": 3,
"enabled": true
}
],
"output": {
"format": "json",
"fields": ["title", "rank", "hot_value", "category", "tags", "description", "source", "crawl_time"]
},
"schedule": {
"interval_minutes": 15,
"max_records": 100
}
}