crawl_config.yaml 3.75 KB
# YouTube 热门视频爬虫配置
# 支持 API 和 yt-dlp 两种方案

# 基础配置
crawler:
  name: "youtube-trending-crawler"
  version: "1.0.0"
  description: "获取 YouTube 当日最热门视频列表"

# 方案选择: api | ytdlp | auto (优先API,失败时降级)
mode: "auto"

# YouTube Data API 配置
api:
  enabled: true
  base_url: "https://www.googleapis.com/youtube/v3"
  endpoints:
    trending:
      path: "/videos"
      method: "GET"
      params:
        part: "snippet,statistics,contentDetails"
        chart: "mostPopular"
        maxResults: 50  # API 限制: 1-50
  # 认证配置 (从环境变量读取)
  auth:
    type: "api_key"
    api_key_env: "YOUTUBE_API_KEY"
  # 地区配置
  regions:
    - code: "US"
      name: "United States"
    - code: "GB"
      name: "United Kingdom"
    - code: "JP"
      name: "Japan"
    - code: "KR"
      name: "South Korea"
    - code: "CN"
      name: "China (may require VPN)"
  # 视频分类 (可选)
  categories:
    - id: "10"
      name: "Music"
    - id: "20"
      name: "Gaming"
    - id: "24"
      name: "Entertainment"
    - id: "28"
      name: "Science & Technology"

# yt-dlp 配置 (无 API 方案)
ytdlp:
  enabled: true
  urls:
    default: "https://www.youtube.com/feed/trending"
    music: "https://www.youtube.com/feed/trending?bp=4gINGgt5dG1hX2NoYXJ0cw%3D%3D"
    gaming: "https://www.youtube.com/feed/trending?bp=4gIcGhpnYW1pbmdfY29udGVudF9mZWVk"
    movies: "https://www.youtube.com/feed/trending?bp=4gIKGgh0cmFpbGVycw%3D%3D"
  options:
    extract_flat: true
    playlistend: 50
    quiet: true
    no_warnings: true
    # 模拟浏览器
    user_agent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    # Cookie 配置 (可选,用于减少验证)
    cookies_from_browser: null  # 可设置为 "chrome", "firefox" 等

# 输出字段配置
output_fields:
  required:
    - name: "videoId"
      type: "string"
      description: "YouTube 视频 ID"
    - name: "url"
      type: "string"
      description: "完整视频链接"
    - name: "title"
      type: "string"
      description: "视频标题"
    - name: "channelTitle"
      type: "string"
      description: "频道名称"
    - name: "viewCount"
      type: "integer"
      description: "播放量"
    - name: "thumbnails"
      type: "object"
      description: "缩略图 URLs (default/medium/high)"
  optional:
    - name: "publishedAt"
      type: "datetime"
      description: "发布时间"
    - name: "duration"
      type: "integer"
      description: "视频时长(秒)"
    - name: "likeCount"
      type: "integer"
      description: "点赞数"
    - name: "commentCount"
      type: "integer"
      description: "评论数"
    - name: "categoryId"
      type: "string"
      description: "分类 ID"
    - name: "tags"
      type: "array"
      description: "视频标签"

# 爬虫行为配置
behavior:
  # 请求间隔 (秒)
  delay: 2
  # 超时时间 (秒)
  timeout: 30
  # 重试次数
  retries: 3
  # 缓存时间 (分钟)
  cache_ttl: 15

# 输出配置
output:
  format: "json"  # json | csv
  pretty_print: true
  # 字段映射 (用于标准化不同方案的输出)
  field_mapping:
    api:
      videoId: "id"
      url: "https://youtube.com/watch?v={id}"
      title: "snippet.title"
      channelTitle: "snippet.channelTitle"
      viewCount: "statistics.viewCount"
      thumbnails: "snippet.thumbnails"
      publishedAt: "snippet.publishedAt"
    ytdlp:
      videoId: "id"
      url: "webpage_url"
      title: "title"
      channelTitle: "uploader"
      viewCount: "view_count"
      thumbnails: "thumbnail"
      duration: "duration"

# 错误处理
error_handling:
  # API 配额耗尽时自动切换
  auto_fallback: true
  # 降级顺序
  fallback_chain: ["api", "ytdlp"]
  # 告警阈值
  alert_on_failures: 3