crawl_config.json
2.62 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
{
"version": "1.0",
"skill": "researcher",
"description": "Researcher 技能爬取配置 - 深度研究数据源",
"crawl_rules": {
"global": {
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"timeout": 30000,
"retry": 3,
"delay_between_requests": 3000
}
},
"targets": [
{
"name": "豆瓣电影短剧榜",
"source": "douban",
"base_url": "https://movie.douban.com",
"endpoints": [
"/tag/短剧/",
"/celebrity/followed/",
"/review/latest/"
],
"selectors": {
"list_item": ".item",
"title": ".title",
"rating": ".rating_num",
"info": ".info"
},
"priority": "P0",
"max_items": 50
},
{
"name": "腾讯视频短剧",
"source": "tencent",
"base_url": "https://v.qq.com",
"endpoints": [
"/channel/shortdrama",
"/rank/shortdrama/"
],
"selectors": {
"list_item": ".mod-item",
"title": ".title",
"play_count": ".num"
},
"priority": "P0",
"max_items": 30
},
{
"name": "爱奇艺短剧",
"source": "iqiyi",
"base_url": "https://www.iqiyi.com",
"endpoints": [
"/dianying/list?channel=shortplay",
"/v list?category=shortplay"
],
"selectors": {
"list_item": ".qy-item",
"title": ".qy-title",
"tags": ".tag-list"
},
"priority": "P1",
"max_items": 30
},
{
"name": "云合数据短剧榜",
"source": "yunhe",
"base_url": "https://www.yunhe.cn",
"endpoints": [
"/short-drama/ranking"
],
"selectors": {
"list_item": ".ranking-item",
"title": ".drama-title",
"views": ".view-count"
},
"priority": "P1",
"max_items": 20
},
{
"name": "知乎剧本创作",
"source": "zhihu",
"base_url": "https://www.zhihu.com",
"endpoints": [
"/topic/剧本创作",
"/topic/人物塑造"
],
"selectors": {
"list_item": ".List-item",
"title": ".ContentItem-title",
"excerpt": ".RichText"
},
"priority": "P2",
"max_items": 20
}
],
"data_fields": {
"required": [
"title",
"source_url",
"crawl_time"
],
"optional": [
"description",
"rating",
"tags",
"author",
"views"
]
},
"storage": {
"format": "jsonl",
"path": "./seed_data.json"
},
"notes": "遵守 robots.txt,单站点请求间隔 ≥ 3秒"
}