crawl_config.json
4.37 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
{
"skill_name": "topic-selector",
"description": "AI互动剧选题筛选 - 数据采集配置",
"version": "1.0",
"created": "2026-03-27",
"data_sources": [
{
"name": "WETRUE短剧数据",
"url": "https://wetrue.com",
"type": "充值榜单",
"priority": "P0",
"update_frequency": "每日",
"crawl_config": {
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
},
"interval": 3000,
"timeout": 15000
},
"parsing_rules": {
"target_selector": ".drama-list, .ranking-list",
"fields": ["rank", "title", "charge_amount", "genre", "platform"]
},
"robots_allowed": true
},
{
"name": "鞭牛士短剧新闻",
"url": "https://www.bianews.com/news/duanju",
"type": "行业新闻",
"priority": "P0",
"update_frequency": "实时",
"crawl_config": {
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
},
"interval": 3000,
"timeout": 15000
},
"parsing_rules": {
"target_selector": ".news-list, article",
"fields": ["title", "summary", "url", "publish_time"]
},
"robots_allowed": true
},
{
"name": "抖音热榜",
"url": "https://www.douyin.com/aweme/v1/web/hot/search/",
"type": "平台趋势",
"priority": "P1",
"update_frequency": "每小时",
"crawl_config": {
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
},
"interval": 5000,
"timeout": 15000
},
"parsing_rules": {
"target_selector": "word",
"fields": ["rank", "word", "hot_value"]
},
"robots_allowed": true
},
{
"name": "快手热榜",
"url": "https://www.kuaishou.com/short-video/lists/hot",
"type": "平台趋势",
"priority": "P1",
"update_frequency": "每小时",
"crawl_config": {
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
},
"interval": 5000,
"timeout": 15000
},
"parsing_rules": {
"target_selector": ".hot-list",
"fields": ["rank", "title", "views"]
},
"robots_allowed": true
},
{
"name": "36kr短剧资讯",
"url": "https://36kr.com/search/news/keyword/%E7%9F%AD%E5%89%A7",
"type": "科技媒体",
"priority": "P1",
"update_frequency": "每日",
"crawl_config": {
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
},
"interval": 3000,
"timeout": 15000
},
"parsing_rules": {
"target_selector": ".search-result, .article-list",
"fields": ["title", "summary", "url", "publish_time"]
},
"robots_allowed": true
}
],
"genre_classification": {
"战神赘婿": ["复仇", "逆袭", "身份反转", "窝囊女婿"],
"甜宠": ["恋爱", "总裁", "豪门", "甜妻"],
"虐恋": ["复仇", "误会", "虐心", "生死"],
"都市": ["职场", "家庭", "情感", "成长"],
"玄幻": ["修仙", "系统", "穿越", "异界"],
"悬疑": ["推理", "惊悚", "侦破", "烧脑"]
},
"quality_metrics": {
"完播率": {"excellent": ">40%", "good": ">30%", "fair": ">20%"},
"付费率": {"excellent": ">5%", "good": ">3%", "fair": ">1%"},
"互动率": {"excellent": ">3%", "good": ">2%", "fair": ">1%"},
"次留率": {"excellent": ">30%", "good": ">25%", "fair": ">20%"}
},
"crawl_schedule": {
"daily": [
{"time": "09:00", "sources": ["WETRUE", "鞭牛士"], "action": "榜单+新闻"},
{"time": "15:00", "sources": ["WETRUE"], "action": "榜单更新"},
{"time": "21:00", "sources": ["WETRUE", "鞭牛士"], "action": "榜单+新闻"}
],
"weekly": [
{"day": "周一", "action": "周度趋势分析报告"}
]
},
"rate_limiting": {
"min_interval_ms": 2000,
"max_requests_per_minute": 30,
"retry_on_failure": true,
"max_retries": 3
},
"data_retention": {
"raw_data_days": 90,
"aggregated_data_days": 365
}
}