crawl_config.json
4.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
{
"version": "1.0",
"skill_name": "trend-hunter",
"description": "热点抓取员 - 多源热点数据采集系统",
"crawl_interval": "*/30 * * * *",
"sources": [
{
"name": "github_trending",
"type": "api",
"url": "https://api.github.com/search/repositories",
"method": "GET",
"params": {
"q": "created:>[DATE-7]",
"sort": "stars",
"order": "desc",
"per_page": 30
},
"headers": {
"User-Agent": "Mozilla/5.0 (compatible; TrendHunter/1.0)"
},
"rate_limit": 60,
"parse_rule": "json.items[] | {title: .name, description: .description, stars: .stargazers_count, url: .html_url, category: .language, source: \"github\"}",
"category": "tech",
"enabled": true
},
{
"name": "hacker_news",
"type": "api",
"url": "https://hacker-news.firebaseio.com/v0/topstories.json",
"method": "GET",
"headers": {},
"rate_limit": 100,
"parse_rule": "json | [.[:30][] | {id: ., url: (\"https://hacker-news.firebaseio.com/v0/item/\" + (. | tostring) + \".json\")}]",
"category": "tech",
"enabled": true
},
{
"name": "baidu_trending",
"type": "web",
"url": "https://top.baidu.com/board?tab=realtime",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"Accept": "text/html,application/xhtml+xml",
"Accept-Language": "zh-CN,zh;q=0.9"
},
"rate_limit": 30,
"parse_rule": "css:.item-title::text | json:{title: [], rank: [1..]}",
"category": "general",
"enabled": true,
"note": "需定期更换User-Agent"
},
{
"name": "bilibili_popular",
"type": "web",
"url": "https://api.bilibili.com/x/web-interface/popular?pn=1&ps=20",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0",
"Referer": "https://www.bilibili.com"
},
"rate_limit": 30,
"parse_rule": "json.data.list[] | {title: .title, author: .owner.name, play: .stat.play, category: \"bilibili\"}",
"category": "entertainment",
"enabled": true
},
{
"name": "google_trends",
"type": "web",
"url": "https://trends.google.com/trending?hl=zh-CN&gl=CN",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)"
},
"rate_limit": 30,
"parse_rule": "css:.title::text",
"category": "general",
"enabled": true
}
],
"hotspot_categories": [
{
"id": "military",
"name": "军事",
"keywords": ["军事", "国防", "武器", "军演", "战争", "冲突"],
"weight": 1.2
},
{
"id": "entertainment",
"name": "娱乐",
"keywords": ["明星", "电影", "综艺", "音乐", "八卦", "票房"],
"weight": 1.0
},
{
"id": "sports",
"name": "体育",
"keywords": ["足球", "篮球", "NBA", "奥运", "电竞", "比赛"],
"weight": 1.0
},
{
"id": "tech",
"name": "科技",
"keywords": ["AI", "数码", "手机", "电脑", "互联网", "开源"],
"weight": 1.0
},
{
"id": "finance",
"name": "财经",
"keywords": ["股票", "基金", "A股", "美股", "财经", "经济"],
"weight": 1.0
}
],
"data_format": {
"required_fields": ["title", "category", "source", "source_url", "crawl_time"],
"optional_fields": ["description", "score", "url", "author", "keywords"]
},
"storage": {
"type": "json",
"path": "../../workspace/hunter/trend-hunter/data/hotspots.json",
"max_records": 500,
"dedup": true
},
"rate_limits": {
"global": {
"min_interval": 2000,
"max_per_minute": 30
},
"per_source": {
"default": 2000,
"github_api": 1000,
"bilibili_api": 3000
}
},
"headers": {
"default": {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "application/json, text/html, */*",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"
}
}
}