crawl_config.json
7.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
{
"config_version": "2.0",
"skill_target": "interactive-video-selector",
"crawl_policy": {
"respect_robots_txt": true,
"request_interval_seconds": 2,
"max_retries": 3,
"user_agent": "Mozilla/5.0 (compatible; LemonAI-Hunter/1.0; +https://lemonai.io/bot)"
},
"data_sources": [
{
"id": "youtube_trending_api",
"name": "YouTube Data API v3 - 热门视频",
"type": "api",
"relevance": "获取带字幕的热门视频Metadata,作为互动剧选题的原始输入素材",
"endpoints": [
{
"url": "https://www.googleapis.com/youtube/v3/videos",
"method": "GET",
"params": {
"part": "snippet,contentDetails,statistics",
"chart": "mostPopular",
"regionCode": "US",
"maxResults": 50,
"videoCategoryId": "24"
},
"headers": {
"Accept": "application/json"
}
},
{
"url": "https://www.googleapis.com/youtube/v3/captions",
"method": "GET",
"params": {
"part": "snippet",
"videoId": "{{videoId}}"
},
"note": "需配合 caption download endpoint 获取实际字幕文本"
}
],
"parse_rules": {
"video_id": "items[].id",
"title": "items[].snippet.title",
"channel_title": "items[].snippet.channelTitle",
"publish_time": "items[].snippet.publishedAt",
"duration": "items[].contentDetails.duration",
"view_count": "items[].statistics.viewCount",
"like_count": "items[].statistics.likeCount",
"tags": "items[].snippet.tags"
},
"rate_limit": {
"requests_per_day": 10000,
"note": "每日配额需通过 Google Cloud Console 申请"
},
"update_frequency": "hourly",
"difficulty": "medium"
},
{
"id": "tiktok_research_api",
"name": "TikTok Research API - 短视频趋势",
"type": "api",
"relevance": "获取热门短视频、标签趋势、互动数据,捕捉短剧/互动剧热门题材和情绪热点",
"endpoints": [
{
"url": "https://open-api.tiktok.com/research/video/query/",
"method": "POST",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer {{access_token}}"
},
"body": {
"query": {
"and": [
{ "operation": "EQ", "field_name": "region_code", "field_values": ["US"] },
{ "operation": "IN", "field_name": "keyword", "field_values": ["short drama", "reelshort", "dramabox", "interactive story"] }
]
},
"start_date": "{{start_date}}",
"end_date": "{{end_date}}",
"max_count": 100
}
}
],
"parse_rules": {
"video_id": "data[].id",
"title": "data[].title",
"username": "data[].username",
"like_count": "data[].like_count",
"comment_count": "data[].comment_count",
"share_count": "data[].share_count",
"view_count": "data[].view_count",
"hashtag_names": "data[].hashtag_names",
"create_time": "data[].create_time"
},
"rate_limit": {
"requests_per_day": 600,
"note": "需申请 TikTok for Business 开发者账号"
},
"update_frequency": "daily",
"difficulty": "high"
},
{
"id": "douban_short_drama_topics",
"name": "豆瓣 - 短剧/互动剧话题讨论",
"type": "web_scraper",
"relevance": "爬取豆瓣小组和话题中的短剧推荐、口碑分析、题材偏好讨论,补充用户侧的定性洞察",
"endpoints": [
{
"url": "https://www.douban.com/group/search?cat=1013&q=短剧",
"method": "GET",
"headers": {
"Accept": "text/html"
}
},
{
"url": "https://www.douban.com/group/search?cat=1013&q=互动剧",
"method": "GET",
"headers": {
"Accept": "text/html"
}
}
],
"parse_rules": {
"post_title": "css:.title a",
"post_url": "css:.title a@href",
"post_summary": "css:.content",
"reply_count": "css:.pl",
"post_time": "css:.time"
},
"rate_limit": {
"request_interval_seconds": 3,
"note": "需遵守豆瓣 robots.txt,避免高频爬取"
},
"update_frequency": "daily",
"difficulty": "medium"
},
{
"id": "reddit_short_drama",
"name": "Reddit r/ShortDrama & r/DramaBox - 海外用户讨论",
"type": "api",
"relevance": "获取海外短剧观众的真实反馈、爆款剧推荐、平台内容偏好",
"endpoints": [
{
"url": "https://www.reddit.com/r/ShortDrama/hot.json",
"method": "GET",
"headers": {
"Accept": "application/json",
"User-Agent": "LemonAI-Hunter/1.0"
}
},
{
"url": "https://www.reddit.com/r/ReelShort/hot.json",
"method": "GET",
"headers": {
"Accept": "application/json",
"User-Agent": "LemonAI-Hunter/1.0"
}
}
],
"parse_rules": {
"post_title": "data.children[].data.title",
"post_url": "data.children[].data.url",
"permalink": "data.children[].data.permalink",
"score": "data.children[].data.score",
"num_comments": "data.children[].data.num_comments",
"created_utc": "data.children[].data.created_utc",
"selftext": "data.children[].data.selftext"
},
"rate_limit": {
"requests_per_minute": 30,
"note": "Reddit API 对未认证请求有限制"
},
"update_frequency": "daily",
"difficulty": "low"
},
{
"id": "zhihu_short_drama",
"name": "知乎 - 短剧/互动剧选题与行业分析",
"type": "web_scraper",
"relevance": "获取中文互联网关于短剧爆款逻辑、互动剧发展前景、平台策略的专业分析",
"endpoints": [
{
"url": "https://www.zhihu.com/search?type=content&q=短剧%20爆款",
"method": "GET",
"headers": {
"Accept": "text/html",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}
},
{
"url": "https://www.zhihu.com/search?type=content&q=互动剧%20选题",
"method": "GET",
"headers": {
"Accept": "text/html",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}
}
],
"parse_rules": {
"question_title": "css:.ContentItem-title",
"question_url": "css:.ContentItem-title a@href",
"answer_excerpt": "css:.RichContent-inner",
"vote_count": "css:.VoteButton--up"
},
"rate_limit": {
"request_interval_seconds": 3,
"note": "知乎反爬严格,建议配合登录态或降低频率"
},
"update_frequency": "weekly",
"difficulty": "medium"
}
],
"output_schema": {
"required_fields": [
"source_url",
"crawl_time",
"title",
"platform",
"category_tag",
"duration_seconds",
"view_count",
"subtitle_extract",
"interactive_suitability_score",
"ai_producibility_score",
"grade",
"recommendation"
]
}
}