crawl_config.json
3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
{
"skill_name": "researcher",
"description": "深度研究技能爬取配置",
"data_sources": [
{
"name": "维基百科 API",
"type": "api",
"base_url": "https://zh.wikipedia.org/w/api.php",
"endpoints": [
{
"path": "/w/api.php",
"method": "GET",
"params": {
"action": "query",
"prop": "extracts|info|images",
"exintro": true,
"explaintext": true,
"format": "json",
"titles": "{query}"
}
}
],
"headers": {
"User-Agent": "LemonAI-Hunter/1.0 (research purpose)"
},
"rate_limit": {
"requests_per_second": 2,
"delay_ms": 500
}
},
{
"name": "百度百科",
"type": "web",
"base_url": "https://baike.baidu.com",
"endpoints": [
{
"path": "/item/{query}",
"method": "GET"
}
],
"headers": {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
},
"rate_limit": {
"requests_per_second": 0.5,
"delay_ms": 2000
},
"selectors": {
"title": ".title .lemma-title",
"content": ".lemma-text",
"image": ".summary-pic img"
}
},
{
"name": "新闻源",
"type": "web",
"sources": [
{
"name": "新浪新闻",
"base_url": "https://search.sina.com.cn",
"search_path": "/?q={query}&c=news"
},
{
"name": "腾讯新闻",
"base_url": "https://news.qq.com",
"search_path": "/cgi-bin/search?q={query}"
},
{
"name": "澎湃新闻",
"base_url": "https://www.thepaper.cn",
"search_path": "/searchPageResult?keys={query}"
}
],
"headers": {
"User-Agent": "Mozilla/5.0"
},
"rate_limit": {
"delay_ms": 5000
}
},
{
"name": "社交媒体",
"type": "web",
"sources": [
{
"name": "微博热搜",
"base_url": "https://weibo.com",
"api": "https://weibo.com/ajax/side/hotSearch"
},
{
"name": "知乎",
"base_url": "https://www.zhihu.com",
"search_path": "/search?q={query}&type=content"
}
],
"headers": {
"User-Agent": "Mozilla/5.0"
},
"rate_limit": {
"delay_ms": 10000
},
"note": "需登录态或模拟浏览器"
}
],
"crawl_rules": {
"respect_robots_txt": true,
"max_depth": 3,
"retry_on_failure": 3,
"timeout_ms": 30000
},
"output": {
"format": "jsonl",
"fields": [
"query",
"source_type",
"source_url",
"crawl_time",
"title",
"content",
"summary",
"tags",
"confidence"
],
"required_fields": ["query", "source_url", "crawl_time"]
},
"seed_queries": [
"人物",
"事件",
"组织",
"作品",
"历史"
]
}