crawl_config.json
3.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
{
"skill_name": "lemonv_drama",
"version": "1.0.0",
"crawl_time": "2026-04-13T02:40:00Z",
"sources": [
{
"id": "zhonghuadiancan",
"name": "中华典藏古籍",
"type": "web_scraper",
"endpoint": "https://www.zhonghuadiancan.com/",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0"
},
"parse_rules": {
"list_selector": ".book-item",
"fields": {
"title": "h3",
"author": ".author"
}
},
"rate_limit": {
"requests_per_minute": 10,
"sleep_ms": 3000,
"respect_robots_txt": true
},
"priority": 1
},
{
"id": "gushiwen",
"name": "古诗文网",
"type": "web_scraper",
"endpoint": "https://so.gushiwen.cn/guwen/",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0"
},
"parse_rules": {
"list_selector": ".sons",
"fields": {
"title": "b",
"dynasty": ".source"
}
},
"rate_limit": {
"requests_per_minute": 10,
"sleep_ms": 3000,
"respect_robots_txt": true
},
"priority": 2
},
{
"id": "douban_gufeng",
"name": "豆瓣古风小说分类",
"type": "web_scraper",
"endpoint": "https://book.douban.com/tag/古风",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0"
},
"parse_rules": {
"list_selector": ".subject-item",
"fields": {
"title": "h2 a",
"rating": ".rating_nums"
}
},
"rate_limit": {
"requests_per_minute": 5,
"sleep_ms": 3000,
"respect_robots_txt": true
},
"priority": 3
},
{
"id": "bilibili_gufeng",
"name": "B站古风区热门",
"type": "api",
"endpoint": "https://api.bilibili.com/x/web-interface/ranking/v2",
"method": "GET",
"params": {
"rid": 168,
"type": "all"
},
"headers": {
"User-Agent": "Mozilla/5.0",
"Referer": "https://www.bilibili.com"
},
"parse_rules": {
"list_path": "data.list",
"fields": {
"title": "title",
"bvid": "bvid",
"author": "owner.name",
"view": "stat.view"
}
},
"rate_limit": {
"requests_per_minute": 10,
"sleep_ms": 2000
},
"priority": 4
},
{
"id": "wikipedia_chinese_history",
"name": "维基百科中国历史页面",
"type": "web_scraper",
"endpoint": "https://zh.wikipedia.org/wiki/中国历史",
"method": "GET",
"headers": {
"User-Agent": "Mozilla/5.0"
},
"parse_rules": {
"list_selector": "#mw-content-text h2",
"fields": {
"title": "span.mw-headline"
}
},
"rate_limit": {
"requests_per_minute": 10,
"sleep_ms": 3000,
"respect_robots_txt": true
},
"priority": 5
}
],
"data_schema": {
"required_fields": [
"drama_id",
"title",
"era",
"genre",
"source_url",
"crawl_time",
"source_type",
"source_content"
],
"optional_fields": [
"characters",
"scenes",
"plots",
"endings",
"image_prompts",
"word_count"
]
},
"output_format": {
"file": "seed_data.json",
"encoder": "json",
"append_timestamp": true
}
}