crawl_config.yaml
3.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
# YouTube 热门视频爬虫配置
# 支持 API 和 yt-dlp 两种方案
# 基础配置
crawler:
name: "youtube-trending-crawler"
version: "1.0.0"
description: "获取 YouTube 当日最热门视频列表"
# 方案选择: api | ytdlp | auto (优先API,失败时降级)
mode: "auto"
# YouTube Data API 配置
api:
enabled: true
base_url: "https://www.googleapis.com/youtube/v3"
endpoints:
trending:
path: "/videos"
method: "GET"
params:
part: "snippet,statistics,contentDetails"
chart: "mostPopular"
maxResults: 50 # API 限制: 1-50
# 认证配置 (从环境变量读取)
auth:
type: "api_key"
api_key_env: "YOUTUBE_API_KEY"
# 地区配置
regions:
- code: "US"
name: "United States"
- code: "GB"
name: "United Kingdom"
- code: "JP"
name: "Japan"
- code: "KR"
name: "South Korea"
- code: "CN"
name: "China (may require VPN)"
# 视频分类 (可选)
categories:
- id: "10"
name: "Music"
- id: "20"
name: "Gaming"
- id: "24"
name: "Entertainment"
- id: "28"
name: "Science & Technology"
# yt-dlp 配置 (无 API 方案)
ytdlp:
enabled: true
urls:
default: "https://www.youtube.com/feed/trending"
music: "https://www.youtube.com/feed/trending?bp=4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
gaming: "https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29udGVudF9mZWVk"
movies: "https://www.youtube.com/feed/trending?bp=4gIKGgh0cmFpbGVycw%3D%3D"
options:
extract_flat: true
playlistend: 50
quiet: true
no_warnings: true
# 模拟浏览器
user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
# Cookie 配置 (可选,用于减少验证)
cookies_from_browser: null # 可设置为 "chrome", "firefox" 等
# 输出字段配置
output_fields:
required:
- name: "videoId"
type: "string"
description: "YouTube 视频 ID"
- name: "url"
type: "string"
description: "完整视频链接"
- name: "title"
type: "string"
description: "视频标题"
- name: "channelTitle"
type: "string"
description: "频道名称"
- name: "viewCount"
type: "integer"
description: "播放量"
- name: "thumbnails"
type: "object"
description: "缩略图 URLs (default/medium/high)"
optional:
- name: "publishedAt"
type: "datetime"
description: "发布时间"
- name: "duration"
type: "integer"
description: "视频时长(秒)"
- name: "likeCount"
type: "integer"
description: "点赞数"
- name: "commentCount"
type: "integer"
description: "评论数"
- name: "categoryId"
type: "string"
description: "分类 ID"
- name: "tags"
type: "array"
description: "视频标签"
# 爬虫行为配置
behavior:
# 请求间隔 (秒)
delay: 2
# 超时时间 (秒)
timeout: 30
# 重试次数
retries: 3
# 缓存时间 (分钟)
cache_ttl: 15
# 输出配置
output:
format: "json" # json | csv
pretty_print: true
# 字段映射 (用于标准化不同方案的输出)
field_mapping:
api:
videoId: "id"
url: "https://youtube.com/watch?v={id}"
title: "snippet.title"
channelTitle: "snippet.channelTitle"
viewCount: "statistics.viewCount"
thumbnails: "snippet.thumbnails"
publishedAt: "snippet.publishedAt"
ytdlp:
videoId: "id"
url: "webpage_url"
title: "title"
channelTitle: "uploader"
viewCount: "view_count"
thumbnails: "thumbnail"
duration: "duration"
# 错误处理
error_handling:
# API 配额耗尽时自动切换
auto_fallback: true
# 降级顺序
fallback_chain: ["api", "ytdlp"]
# 告警阈值
alert_on_failures: 3