crawl_config.json
3.53 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
{
"version": "1.0",
"skill_name": "community-manager",
"description": "社区运营与马甲运营知识库的爬虫配置。由于社区平台规则更新频繁且部分数据需登录态，本配置以公开榜单和规则页面为主数据源。",
"crawl_policy": {
"respect_robots_txt": true,
"request_interval_seconds": 3,
"max_retries": 2,
"user_agent": "Mozilla/5.0 (compatible; ArenaSK-Hunter/1.0)"
},
"sources": [
{
"name": "weibo_hot_search",
"platform": "微博",
"url": "https://s.weibo.com/top/summary",
"type": "html_parse",
"update_frequency": "realtime",
"fields": ["rank", "title", "hot_value", "category"],
"headers": {
"Accept": "text/html,application/xhtml+xml",
"Accept-Language": "zh-CN,zh;q=0.9"
},
"parser": {
"selector": "#pl_top_realtimehot tbody tr",
"field_mapping": {
"rank": "td.td-01.ranktop | text",
"title": "td.td-02 a | text",
"hot_value": "td.td-02 span | text"
}
}
},
{
"name": "bilibili_ranking",
"platform": "B站",
"url": "https://api.bilibili.com/x/web-interface/ranking/v2?rid=0&type=all",
"type": "api_json",
"update_frequency": "daily",
"fields": ["bvid", "title", "pic", "author", "view_count", "danmaku_count", "like_count"],
"headers": {
"Referer": "https://www.bilibili.com",
"User-Agent": "Mozilla/5.0"
},
"parser": {
"root_path": "data.list",
"field_mapping": {
"bvid": "bvid",
"title": "title",
"pic": "pic",
"author": "owner.name",
"view_count": "stat.view",
"danmaku_count": "stat.danmaku",
"like_count": "stat.like"
}
}
},
{
"name": "douyin_hot_search",
"platform": "抖音",
"url": "https://www.douyin.com/hot",
"type": "html_parse_js_rendered",
"update_frequency": "realtime",
"fields": ["rank", "title", "hot_value", "label"],
"note": "需 Selenium/Playwright 等无头浏览器渲染 JS 后抓取",
"parser": {
"selector": "[data-e2e='hot-list'] li",
"field_mapping": {
"rank": ".rank | text",
"title": ".title | text",
"hot_value": ".hot-value | text"
}
}
},
{
"name": "xiaohongshu_explore",
"platform": "小红书",
"url": "https://www.xiaohongshu.com/explore",
"type": "rpa_browser",
"update_frequency": "daily",
"fields": ["note_id", "title", "author", "likes", "cover_image"],
"note": "小红书核心内容需登录态，建议通过 RPA 配合 cookies 获取首页推荐流"
},
{
"name": "platform_rules",
"platform": "通用",
"type": "manual_reference",
"urls": [
"https://www.xiaohongshu.com/protocols/agreement",
"https://www.douyin.com/aboutus/rules.html",
"https://www.bilibili.com/blackboard/account-cn.html",
"https://weibo.com/signup/v5/protocol"
],
"update_frequency": "monthly",
"note": "平台规则页面主要用于人工核对和引用，机器爬取后需人工校验"
}
],
"data_schema": {
"required_fields": ["source_name", "crawl_time", "platform"],
"optional_fields": ["rank", "title", "author", "view_count", "hot_value", "url"]
},
"fallback": {
"description": "当公开榜单因反爬升级不可用时，可切换至第三方数据平台 API（如新榜、蝉妈妈、飞瓜数据），但需付费授权。"
}
}