crawl_config.json
2.77 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
{
"version": "1.0",
"description": "topic-selector 选题筛选器爬取配置",
"data_sources": [
{
"name": "百度热搜",
"url": "https://top.baidu.com/board?tab=realtime",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
},
"update_frequency": "实时",
"status": "active",
"parser": {
"type": "html",
"selector": ".item-wrap .item-title",
"fields": ["rank", "title", "heat"]
}
},
{
"name": "哔哩哔哩热搜",
"url": "https://api.bilibili.com/x/web-interface/popular",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
},
"update_frequency": "实时",
"status": "active",
"parser": {
"type": "json",
"path": "data.list",
"fields": ["title", "desc", "pic", "stat"]
}
}
],
"filter_rules": {
"pre_filters": [
{
"type": "exclude_keywords",
"keywords": ["敏感", "违规", "作弊", "谣言"]
},
{
"type": "min_heat_threshold",
"value": 1000
}
],
"scoring_dimensions": [
{
"name": "热度指数",
"key": "heat_score",
"weight": 0.3,
"metrics": ["search_volume", "read_count", "discussion_count"]
},
{
"name": "话题延展性",
"key": "extendability",
"weight": 0.25,
"metrics": ["angle_count", "controversy_level", "knowledge增量"]
},
{
"name": "受众覆盖",
"key": "audience_coverage",
"weight": 0.25,
"metrics": ["age_group", "purchasing_power", "interest_match"]
},
{
"name": "生命周期",
"key": "lifecycle",
"weight": 0.2,
"metrics": ["trend_phase", "time_sensitivity", "reversal_risk"]
}
],
"industry_categories": [
"科技数码",
"娱乐明星",
"社会民生",
"国际关系",
"财经商业",
"健康养生",
"教育职场"
],
"output_top_n": 20
},
"output_format": {
"type": "json",
"schema": {
"topic_id": "string",
"title": "string",
"category": "string",
"source": "string",
"source_url": "string",
"heat_score": "number",
"extendability_score": "number",
"audience_score": "number",
"lifecycle_score": "number",
"total_score": "number",
"crawl_time": "string"
}
},
"crawl_settings": {
"request_interval_ms": 2000,
"max_retries": 3,
"timeout_ms": 10000,
"respect_robots_txt": true
}
}