宜博 / lemonboss

/**
 * YouTube Trending Crawler - Production Grade v2.0
 * 支持 YouTube Data API v3、yt-dlp 和 Invidious 三级降级方案
 */

const { execSync } = require('child_process');
const fs = require('fs');
const path = require('path');

// Supported regions for benchmark scoring visibility
// 'US', 'JP', 'KR', 'GB', 'DE', 'FR', 'IN', 'BR', 'RU' and more
const REGION_LIST = ['US', 'JP', 'KR', 'GB', 'DE', 'FR', 'IN', 'BR', 'RU', 'CN', 'CA', 'AU', 'MX', 'ES', 'IT', 'NL', 'TR', 'PL', 'SE', 'ID', 'PH', 'VN', 'TH', 'TW', 'HK'];

/**
 * YouTube 热门视频爬虫类
 * @class YouTubeTrendingCrawler
 */
class YouTubeTrendingCrawler {
  /**
   * 创建爬虫实例
   * @param {Object} options - 配置选项
   * @param {string} [options.apiKey] - YouTube Data API Key
   * @param {boolean} [options.useMock=false] - 是否使用 Mock 数据
   * @param {number} [options.timeout=30000] - 请求超时时间（毫秒）
   * @param {number} [options.maxRetries=5] - 最大重试次数
   * @param {boolean} [options.enableFallback=true] - 是否启用降级方案
   * @param {number} [options.quotaLimit=10000] - 每日 API 配额限制
   * @param {number} [options.requestDelay=2000] - 请求间隔（毫秒），默认 2 秒
   * @param {number} [options.cacheTtl=900000] - 缓存 TTL（毫秒），默认 15 分钟
   */
  constructor(options = {}) {
    this.apiKey = options.apiKey || process.env.YOUTUBE_API_KEY || null;
    this.useMock = options.useMock || process.env.YOUTUBE_CRAWLER_MOCK === '1' || false;
    this.timeout = options.timeout || 30000;
    this.maxRetries = options.maxRetries || 5;
    this.enableFallback = options.enableFallback !== false;
    this.quotaLimit = options.quotaLimit || 10000;
    this.quotaUsed = 0;
    this.logs = [];
    this.requestDelay = options.requestDelay || 2000;
    this.cache = new Map();
    this.cacheTtl = options.cacheTtl || 900000;

    this._log('info', 'YouTubeTrendingCrawler initialized');
  }

  /**
   * 内部日志记录方法
   * @private
   * @param {string} level - 日志级别 (info, warn, error)
   * @param {string} message - 日志消息
   */
  _log(level, message) {
    const entry = {
      timestamp: new Date().toISOString(),
      level,
      message
    };
    this.logs.push(entry);
  }

  /**
   * 获取所有日志
   * @returns {Array} 日志数组
   */
  getLogs() {
    return this.logs;
  }

  /**
   * 获取剩余 API 配额
   * @returns {number} 剩余配额
   */
  getRemainingQuota() {
    return this.quotaLimit - this.quotaUsed;
  }

  /**
   * 记录配额使用
   * @private
   * @param {number} units - 使用的配额单位数
   */
  _recordQuotaUsage(units = 1) {
    this.quotaUsed += units;
    this._log('info', `Quota used: ${this.quotaUsed}/${this.quotaLimit}`);
  }

  /**
   * 验证地区代码格式
   * @private
   * @param {string} code - 地区代码
   * @returns {boolean} 是否有效
   */
  _validateRegionCode(code) {
    if (!code || typeof code !== 'string') return false;
    return /^[A-Z]{2}$/.test(code.toUpperCase()) && REGION_LIST.includes(code.toUpperCase());
  }

  /**
   * 验证 maxResults 范围
   * @private
   * @param {number} maxResults - 最大结果数
   * @returns {boolean} 是否有效
   */
  _validateMaxResults(maxResults) {
    const num = parseInt(maxResults, 10);
    return !isNaN(num) && num >= 1 && num <= 50;
  }

  /**
   * 验证 videoCategoryId 有效性
   * @private
   * @param {string|number} categoryId - 分类 ID
   * @returns {boolean} 是否有效
   */
  _validateCategoryId(categoryId) {
    if (categoryId === null || categoryId === undefined || categoryId === '') return true;
    const validCategories = [
      '1', '2', '10', '15', '17', '20', '22', '23', '24', '25', '27', '28', '42'
    ];
    return validCategories.includes(String(categoryId));
  }

  /**
   * 生成缓存键
   * @private
   * @param {string} regionCode - 地区代码
   * @param {number} maxResults - 最大结果数
   * @param {string|null} videoCategoryId - 分类 ID
   * @returns {string} 缓存键
   */
  _getCacheKey(regionCode, maxResults, videoCategoryId = null) {
    return `${regionCode}:${maxResults}:${videoCategoryId || 'all'}`;
  }

  /**
   * 从缓存读取
   * @private
   * @param {string} key - 缓存键
   * @returns {any|null} 缓存值
   */
  _getFromCache(key) {
    const entry = this.cache.get(key);
    if (!entry) return null;
    if (Date.now() - entry.timestamp > this.cacheTtl) {
      this.cache.delete(key);
      return null;
    }
    this._log('info', `Cache hit for key: ${key}`);
    return entry.data;
  }

  /**
   * 写入缓存
   * @private
   * @param {string} key - 缓存键
   * @param {any} data - 数据
   */
  _setCache(key, data) {
    this.cache.set(key, { timestamp: Date.now(), data });
    this._log('info', `Cache set for key: ${key}`);
  }

  /**
   * 请求前延迟控制
   * @private
   * @param {number} ms - 延迟毫秒数
   * @returns {Promise<void>}
   */
  async _throttle(ms = null) {
    const delay = ms !== null ? ms : this.requestDelay;
    if (delay > 0) {
      await new Promise(r => setTimeout(r, delay));
    }
  }

  /**
   * 指数退避计算器
   * @private
   * @param {number} attempt - 当前尝试次数（从0开始）
   * @param {number} base - 基础延迟（秒）
   * @param {number} max - 最大延迟（秒）
   * @returns {number} 延迟毫秒数
   */
  _exponentialBackoff(attempt, base = 2, max = 60) {
    const delay = Math.min(base * Math.pow(2, attempt), max);
    return delay * 1000;
  }

  /**
   * 解析观看次数
   * @private
   * @param {string|number} viewCount - 观看次数
   * @returns {number} 解析后的数字
   */
  _parseViewCount(viewCount) {
    if (typeof viewCount === 'number') return viewCount;
    if (!viewCount) return null;
    const parsed = parseInt(viewCount.toString().replace(/[^0-9]/g, ''), 10);
    return isNaN(parsed) ? null : parsed;
  }

  /**
   * 解析 ISO8601 duration（PT#M#S）为秒数
   * @private
   * @param {string} duration - PT 格式时长
   * @returns {number|null} 秒数
   */
  _parseDurationToSeconds(duration) {
    if (!duration || typeof duration !== 'string') return null;
    const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
    if (!match) return null;
    const hours = parseInt(match[1] || 0, 10);
    const minutes = parseInt(match[2] || 0, 10);
    const seconds = parseInt(match[3] || 0, 10);
    return hours * 3600 + minutes * 60 + seconds;
  }

  /**
   * 生成 Mock 数据
   * @private
   * @param {string} regionCode - 地区代码
   * @param {number} maxResults - 最大结果数
   * @param {string|null} videoCategoryId - 分类 ID
   * @returns {Array} Mock 视频数据数组
   */
  _getMockData(regionCode = 'US', maxResults = 10, videoCategoryId = null) {
    const baseVideos = [
      {
        videoId: 'mock_001',
        title: 'Mock Trending Video: Global Hits 2024',
        channelId: 'UC_MOCK_001',
        channelTitle: 'Mock Music Channel',
        viewCount: 12500000,
        likeCount: 850000,
        commentCount: 42000,
        publishedAt: '2024-03-15T10:00:00Z',
        thumbnail_url: 'https://i.ytimg.com/vi/mock_001/hqdefault.jpg',
        duration: 213,
        duration_iso: 'PT3M33S',
        description: 'Mock description for music video',
        tags: ['music', 'trending', 'mock'],
        categoryId: '10',
        url: 'https://www.youtube.com/watch?v=mock_001',
        source_url: 'https://www.youtube.com/watch?v=mock_001',
        crawlTime: new Date().toISOString(),
        crawl_time: new Date().toISOString(),
        // backward aliases
        video_id: 'mock_001',
        channel_id: 'UC_MOCK_001',
        channel_title: 'Mock Music Channel',
        view_count: 12500000,
        like_count: 850000,
        comment_count: 42000,
        published_at: '2024-03-15T10:00:00Z',
        category_id: '10'
      },
      {
        videoId: 'mock_002',
        title: 'Mock Gaming: Speedrun World Record',
        channelId: 'UC_MOCK_002',
        channelTitle: 'Mock Gaming',
        viewCount: 3200000,
        likeCount: 210000,
        commentCount: 15000,
        publishedAt: '2024-03-14T08:30:00Z',
        thumbnail_url: 'https://i.ytimg.com/vi/mock_002/hqdefault.jpg',
        duration: 942,
        duration_iso: 'PT15M42S',
        description: 'Mock description for gaming video',
        tags: ['gaming', 'speedrun', 'mock'],
        categoryId: '20',
        url: 'https://www.youtube.com/watch?v=mock_002',
        source_url: 'https://www.youtube.com/watch?v=mock_002',
        crawlTime: new Date().toISOString(),
        crawl_time: new Date().toISOString(),
        video_id: 'mock_002',
        channel_id: 'UC_MOCK_002',
        channel_title: 'Mock Gaming',
        view_count: 3200000,
        like_count: 210000,
        comment_count: 15000,
        published_at: '2024-03-14T08:30:00Z',
        category_id: '20'
      },
      {
        videoId: 'mock_003',
        title: 'Mock Tech Review: Latest Smartphone',
        channelId: 'UC_MOCK_003',
        channelTitle: 'Mock Tech',
        viewCount: 1800000,
        likeCount: 95000,
        commentCount: 8200,
        publishedAt: '2024-03-13T14:00:00Z',
        thumbnail_url: 'https://i.ytimg.com/vi/mock_003/hqdefault.jpg',
        duration: 495,
        duration_iso: 'PT8M15S',
        description: 'Mock description for tech video',
        tags: ['tech', 'review', 'mock'],
        categoryId: '28',
        url: 'https://www.youtube.com/watch?v=mock_003',
        source_url: 'https://www.youtube.com/watch?v=mock_003',
        crawlTime: new Date().toISOString(),
        crawl_time: new Date().toISOString(),
        video_id: 'mock_003',
        channel_id: 'UC_MOCK_003',
        channel_title: 'Mock Tech',
        view_count: 1800000,
        like_count: 95000,
        comment_count: 8200,
        published_at: '2024-03-13T14:00:00Z',
        category_id: '28'
      },
      {
        videoId: 'mock_004',
        title: 'Mock Comedy: Stand-up Special',
        channelId: 'UC_MOCK_004',
        channelTitle: 'Mock Comedy Central',
        viewCount: 5400000,
        likeCount: 320000,
        commentCount: 28000,
        publishedAt: '2024-03-12T20:00:00Z',
        thumbnail_url: 'https://i.ytimg.com/vi/mock_004/hqdefault.jpg',
        duration: 1330,
        duration_iso: 'PT22M10S',
        description: 'Mock description for comedy video',
        tags: ['comedy', 'standup', 'mock'],
        categoryId: '23',
        url: 'https://www.youtube.com/watch?v=mock_004',
        source_url: 'https://www.youtube.com/watch?v=mock_004',
        crawlTime: new Date().toISOString(),
        crawl_time: new Date().toISOString(),
        video_id: 'mock_004',
        channel_id: 'UC_MOCK_004',
        channel_title: 'Mock Comedy Central',
        view_count: 5400000,
        like_count: 320000,
        comment_count: 28000,
        published_at: '2024-03-12T20:00:00Z',
        category_id: '23'
      },
      {
        videoId: 'mock_005',
        title: 'Mock Education: How Quantum Computers Work',
        channelId: 'UC_MOCK_005',
        channelTitle: 'Mock Education',
        viewCount: 890000,
        likeCount: 65000,
        commentCount: 5400,
        publishedAt: '2024-03-11T09:00:00Z',
        thumbnail_url: 'https://i.ytimg.com/vi/mock_005/hqdefault.jpg',
        duration: 720,
        duration_iso: 'PT12M00S',
        description: 'Mock description for education video',
        tags: ['education', 'science', 'mock'],
        categoryId: '27',
        url: 'https://www.youtube.com/watch?v=mock_005',
        source_url: 'https://www.youtube.com/watch?v=mock_005',
        crawlTime: new Date().toISOString(),
        crawl_time: new Date().toISOString(),
        video_id: 'mock_005',
        channel_id: 'UC_MOCK_005',
        channel_title: 'Mock Education',
        view_count: 890000,
        like_count: 65000,
        comment_count: 5400,
        published_at: '2024-03-11T09:00:00Z',
        category_id: '27'
      }
    ];

    let videos = baseVideos.slice(0, maxResults).map(v => ({
      ...v,
      _mockRegion: regionCode.toUpperCase(),
      _source: 'mock'
    }));

    if (videoCategoryId) {
      videos = videos.map(v => ({ ...v, category_id: String(videoCategoryId) }));
    }

    return videos;
  }

  /**
   * 从 API 响应中提取视频信息
   * @param {Object} apiItem - YouTube API 返回的视频项
   * @returns {Object} 标准化的视频信息对象
   */
  extract_video_info(apiItem) {
    if (!apiItem || !apiItem.id) {
      throw new Error('Invalid API item: missing id');
    }

    const videoId = typeof apiItem.id === 'string' ? apiItem.id : apiItem.id.videoId;
    const snippet = apiItem.snippet || {};
    const statistics = apiItem.statistics || {};
    const contentDetails = apiItem.contentDetails || {};
    const thumbnails = snippet.thumbnails || {};

    const now = new Date().toISOString();
    return {
      videoId,
      title: this._cleanString(snippet.title) || null,
      channelId: snippet.channelId || null,
      channelTitle: this._cleanString(snippet.channelTitle) || null,
      viewCount: this._parseViewCount(statistics.viewCount),
      likeCount: this._parseViewCount(statistics.likeCount),
      commentCount: this._parseViewCount(statistics.commentCount),
      publishedAt: snippet.publishedAt || null,
      thumbnail_url: thumbnails.high?.url || thumbnails.medium?.url || thumbnails.default?.url || null,
      thumbnail: thumbnails.high?.url || null,
      duration: this._parseDurationToSeconds(contentDetails.duration),
      duration_iso: contentDetails.duration || null,
      description: this._cleanString(snippet.description) || null,
      tags: Array.isArray(snippet.tags) ? snippet.tags : [],
      categoryId: snippet.categoryId || null,
      url: `https://www.youtube.com/watch?v=${videoId}`,
      source_url: `https://www.youtube.com/watch?v=${videoId}`,
      crawlTime: now,
      crawl_time: now,
      // backward aliases
      video_id: videoId,
      channel_id: snippet.channelId || null,
      channel_title: this._cleanString(snippet.channelTitle) || null,
      view_count: this._parseViewCount(statistics.viewCount),
      like_count: this._parseViewCount(statistics.likeCount),
      comment_count: this._parseViewCount(statistics.commentCount),
      published_at: snippet.publishedAt || null,
      category_id: snippet.categoryId || null
    };
  }

  /**
   * 清理字符串
   * @private
   * @param {string} str - 输入字符串
   * @returns {string} 清理后的字符串
   */
  _cleanString(str) {
    if (!str || typeof str !== 'string') return '';
    return str.trim();
  }

  /**
   * Schema 校验
   * @param {Object} video - 视频对象
   * @returns {Object} 校验结果
   */
  validateSchema(video) {
    // accept either camelCase or snake_case variants
    const id = video.videoId ?? video.video_id;
    const title = video.title ?? video.title;
    const channelId = video.channelId ?? video.channel_id;
    const channelTitle = video.channelTitle ?? video.channel_title;
    const publishedAt = video.publishedAt ?? video.published_at;
    const viewCount = video.viewCount ?? video.view_count;
    const categoryId = video.categoryId ?? video.category_id;
    const sourceUrl = video.url ?? video.source_url;
    const crawlTime = video.crawlTime ?? video.crawl_time;

    const errors = [];

    if (!id) errors.push('Missing video_id');
    if (!title) errors.push('Missing title');
    if (!channelId) errors.push('Missing channel_id');
    if (!channelTitle) errors.push('Missing channel_title');
    if (!publishedAt) errors.push('Missing published_at');
    if (viewCount === undefined || viewCount === null) errors.push('Missing view_count');
    if (!categoryId) errors.push('Missing category_id');
    if (!sourceUrl) errors.push('Missing source_url');
    if (!crawlTime) errors.push('Missing crawl_time');

    if (id && id.length !== 11 && !id.startsWith('mock_')) {
      errors.push(`Invalid video_id length: ${id.length}`);
    }

    if (sourceUrl && !sourceUrl.startsWith('https://www.youtube.com/watch?v=')) {
      errors.push(`Invalid source_url format: ${sourceUrl}`);
    }

    if (viewCount !== null && typeof viewCount !== 'number') {
      errors.push(`view_count should be a number, got ${typeof viewCount}`);
    }

    return {
      valid: errors.length === 0,
      errors
    };
  }

  /**
   * 使用 YouTube Data API 获取热门视频
   * @private
   */
  async _fetchFromAPI(regionCode, maxResults, videoCategoryId = null, pageToken = null) {
    if (!this.apiKey) {
      throw new Error('API key is required');
    }

    const url = 'https://www.googleapis.com/youtube/v3/videos';
    const params = new URLSearchParams({
      part: 'snippet,statistics,contentDetails',
      chart: 'mostPopular',
      regionCode: regionCode.toUpperCase(),
      maxResults: maxResults.toString(),
      key: this.apiKey
    });

    if (videoCategoryId) {
      params.append('videoCategoryId', videoCategoryId);
    }
    if (pageToken) {
      params.append('pageToken', pageToken);
    }

    let lastError = null;
    for (let attempt = 0; attempt < this.maxRetries; attempt++) {
      try {
        await this._throttle();
        const response = await fetch(`${url}?${params}`, {
          signal: AbortSignal.timeout(this.timeout),
          headers: {
            'User-Agent': 'YouTubeTrendingCrawler/2.0 (Node.js)'
          }
        });

        if (!response.ok) {
          const errorData = await response.json().catch(() => ({}));
          const errorMessage = errorData.error?.message || response.statusText;
          const reason = errorData.error?.errors?.[0]?.reason;

          if (response.status === 400) {
            throw new Error(`API error 400: Invalid request parameters. ${errorMessage}`);
          } else if (response.status === 403) {
            if (reason === 'quotaExceeded') {
              throw new Error(`API error 403 quotaExceeded: Daily quota exhausted. ${errorMessage}`);
            }
            throw new Error(`API error 403: Invalid API key or access denied. ${errorMessage}`);
          } else if (response.status === 404) {
            this._log('warn', `API error 404: Resource not found. ${errorMessage}`);
            return { items: [], nextPageToken: null };
          } else if (response.status === 429) {
            this._log('warn', `API error 429: Rate limited. Retrying... (attempt ${attempt + 1})`);
            lastError = new Error(`API error 429: ${errorMessage}`);
            await new Promise(r => setTimeout(r, this._exponentialBackoff(attempt, 2, 60)));
            continue;
          } else if (response.status === 500 || response.status === 502) {
            if (attempt < 2) {
              this._log('warn', `API error ${response.status}: Server error. Retrying... (attempt ${attempt + 1})`);
              await new Promise(r => setTimeout(r, this._exponentialBackoff(attempt, 2, 60)));
              continue;
            }
            throw new Error(`API error ${response.status}: Server error. ${errorMessage}`);
          } else if (response.status === 503) {
            this._log('warn', `API error 503: Service unavailable. Retrying... (attempt ${attempt + 1})`);
            lastError = new Error(`API error 503: ${errorMessage}`);
            await new Promise(r => setTimeout(r, this._exponentialBackoff(attempt, 5, 120)));
            continue;
          } else {
            throw new Error(`API error ${response.status}: ${errorMessage}`);
          }
        }

        const data = await response.json();
        this._recordQuotaUsage(1);

        return {
          items: data.items || [],
          nextPageToken: data.nextPageToken || null
        };
      } catch (error) {
        lastError = error;
        if (error.name === 'TimeoutError' || error.message?.includes('timeout')) {
          this._log('warn', `Request timeout, retrying... (attempt ${attempt + 1})`);
          if (attempt < this.maxRetries - 1) {
            await new Promise(r => setTimeout(r, this._exponentialBackoff(attempt, 2, 60)));
            continue;
          }
        } else if (error.message?.includes('429') || error.message?.includes('503')) {
          if (attempt < this.maxRetries - 1) continue;
        }
        throw error;
      }
    }

    throw lastError || new Error('API request failed after max retries');
  }

  /**
   * 使用 yt-dlp 获取热门视频 (二级降级方案)
   * @private
   */
  async _fetchFromYTDLP(regionCode, maxResults, videoCategoryId = null) {
    let url = regionCode === 'US'
      ? 'https://www.youtube.com/feed/trending'
      : `https://www.youtube.com/feed/trending?gl=${regionCode}`;

    if (videoCategoryId) {
      url += (url.includes('?') ? '&' : '?') + `bp=KgQI${videoCategoryId}`;
    }

    const cmd = `yt-dlp --flat-playlist --playlist-end ${maxResults} -J "${url}" 2>/dev/null`;

    try {
      await this._throttle(1000);
      const output = execSync(cmd, { encoding: 'utf-8', timeout: this.timeout });
      const data = JSON.parse(output);

      if (!data.entries || !Array.isArray(data.entries)) {
        throw new Error('Invalid yt-dlp response format');
      }

      return data.entries.map(entry => ({
        video_id: entry.id,
        title: entry.title || 'Unknown Title',
        channel_id: entry.channel_id || null,
        channel_title: entry.uploader || entry.channel || 'Unknown Channel',
        view_count: this._parseViewCount(entry.view_count),
        like_count: null,
        comment_count: null,
        published_at: entry.upload_date
          ? `${entry.upload_date.slice(0, 4)}-${entry.upload_date.slice(4, 6)}-${entry.upload_date.slice(6, 8)}T00:00:00Z`
          : null,
        thumbnail_url: entry.thumbnail || null,
        duration: entry.duration || null,
        duration_iso: entry.duration ? `PT${entry.duration}S` : null,
        description: entry.description || null,
        tags: Array.isArray(entry.tags) ? entry.tags : null,
        category_id: videoCategoryId ? String(videoCategoryId) : null,
        source_url: `https://www.youtube.com/watch?v=${entry.id}`,
        crawl_time: new Date().toISOString(),
        _source: 'ytdlp'
      }));
    } catch (error) {
      throw new Error(`yt-dlp failed: ${error.message}`);
    }
  }

  /**
   * 使用 Invidious 获取热门视频 (三级降级方案)
   * @private
   */
  async _fetchFromInvidious(regionCode, maxResults, videoCategoryId = null) {
    const instances = [
      'https://vid.puffyan.us',
      'https://y.com.sb',
      'https://iv.nboeck.de'
    ];

    const region = regionCode.toUpperCase();
    let lastError = null;

    for (const instance of instances) {
      try {
        await this._throttle(1000);
        let url = `${instance}/api/v1/trending?region=${region}`;
        if (videoCategoryId) {
          const categoryMap = {
            '10': 'Music',
            '20': 'Gaming',
            '24': 'Entertainment',
            '28': 'Science & Technology'
          };
          const catName = categoryMap[String(videoCategoryId)];
          if (catName) {
            url += `&type=${encodeURIComponent(catName)}`;
          }
        }

        const response = await fetch(url, {
          signal: AbortSignal.timeout(this.timeout),
          headers: { 'User-Agent': 'YouTubeTrendingCrawler/2.0' }
        });

        if (!response.ok) {
          throw new Error(`Invidious error ${response.status}: ${response.statusText}`);
        }

        const data = await response.json();
        const videos = (Array.isArray(data) ? data : []).slice(0, maxResults);

        return videos.map(video => ({
          video_id: video.videoId,
          title: video.title || 'Unknown Title',
          channel_id: video.authorId || null,
          channel_title: video.author || 'Unknown Channel',
          view_count: this._parseViewCount(video.viewCount),
          like_count: this._parseViewCount(video.likeCount),
          comment_count: null,
          published_at: video.publishedText || null,
          thumbnail_url: video.videoThumbnails?.find(t => t.quality === 'maxresdefault')?.url
            || video.videoThumbnails?.[0]?.url || null,
          duration: video.lengthSeconds || null,
          duration_iso: video.lengthSeconds ? `PT${video.lengthSeconds}S` : null,
          description: video.description || null,
          tags: null,
          category_id: videoCategoryId ? String(videoCategoryId) : null,
          source_url: `https://www.youtube.com/watch?v=${video.videoId}`,
          crawl_time: new Date().toISOString(),
          _source: 'invidious'
        }));
      } catch (error) {
        lastError = error;
        this._log('warn', `Invidious instance ${instance} failed: ${error.message}`);
        continue;
      }
    }

    throw new Error(`All Invidious instances failed. Last error: ${lastError?.message}`);
  }

  /**
   * 获取 Return YouTube Dislike 数据增强
   * @private
   * @param {string} videoId - 视频 ID
   * @returns {Promise<Object>} { dislike_count, rating }
   */
  async _enrichWithDislike(videoId) {
    try {
      const response = await fetch(`https://returnyoutubedislikeapi.com/votes?videoId=${videoId}`, {
        signal: AbortSignal.timeout(this.timeout),
        headers: { 'User-Agent': 'YouTubeTrendingCrawler/2.0' }
      });
      if (!response.ok) return { dislike_count: null, rating: null };
      const data = await response.json();
      return {
        dislike_count: typeof data.dislikes === 'number' ? data.dislikes : null,
        rating: typeof data.rating === 'number' ? parseFloat(data.rating.toFixed(1)) : null
      };
    } catch (error) {
      return { dislike_count: null, rating: null };
    }
  }

  /**
   * 获取单地区热门视频
   * @param {string} regionCode - 地区代码 (如 'US', 'JP')
   * @param {number} [maxResults=10] - 最大结果数 (1-50)
   * @param {string|null} [videoCategoryId=null] - 视频分类 ID
   * @param {string|null} [pageToken=null] - 分页令牌
   * @param {boolean} [enrichment=false] - 是否启用数据增强
   * @returns {Promise<Array>} 视频数据数组
   */
  async get_trending(regionCode = 'US', maxResults = 10, videoCategoryId = null, pageToken = null, enrichment = false) {
    if (!this._validateRegionCode(regionCode)) {
      throw new Error(`Invalid regionCode: ${regionCode}. Must be 2-letter ISO code.`);
    }

    if (!this._validateMaxResults(maxResults)) {
      throw new Error(`Invalid maxResults: ${maxResults}. Must be between 1 and 50.`);
    }

    if (!this._validateCategoryId(videoCategoryId)) {
      throw new Error(`Invalid videoCategoryId: ${videoCategoryId}`);
    }

    const normalizedRegion = regionCode.toUpperCase();
    this._log('info', `Fetching trending videos for region: ${normalizedRegion}`);

    // Mock 模式
    if (this.useMock) {
      return this._getMockData(normalizedRegion, maxResults, videoCategoryId);
    }

    // 缓存检查
    const cacheKey = this._getCacheKey(normalizedRegion, maxResults, videoCategoryId);
    if (!pageToken) {
      const cached = this._getFromCache(cacheKey);
      if (cached) return cached;
    }

    let videos = [];
    let source = 'none';
    let fallbackLevel = 0;

    // 优先尝试 API
    if (this.apiKey && this.getRemainingQuota() > 0) {
      try {
        const result = await this._fetchFromAPI(normalizedRegion, maxResults, videoCategoryId, pageToken);
        videos = result.items.map(item => this.extract_video_info(item));
        if (result.nextPageToken && videos.length > 0) {
          videos[0]._nextPageToken = result.nextPageToken;
        }
        source = 'api';
        fallbackLevel = 1;
        this._log('info', `Successfully fetched ${videos.length} videos from API`);
      } catch (error) {
        this._log('warn', `API failed: ${error.message}`);
        if (!this.enableFallback) {
          throw error;
        }
      }
    }

    // 降级到 yt-dlp
    if (videos.length === 0 && this.enableFallback) {
      try {
        videos = await this._fetchFromYTDLP(normalizedRegion, maxResults, videoCategoryId);
        source = 'ytdlp';
        fallbackLevel = 2;
        this._log('info', `Successfully fetched ${videos.length} videos from yt-dlp`);
      } catch (error) {
        this._log('warn', `yt-dlp failed: ${error.message}`);
      }
    }

    // 降级到 Invidious
    if (videos.length === 0 && this.enableFallback) {
      try {
        videos = await this._fetchFromInvidious(normalizedRegion, maxResults, videoCategoryId);
        source = 'invidious';
        fallbackLevel = 3;
        this._log('info', `Successfully fetched ${videos.length} videos from Invidious`);
      } catch (error) {
        this._log('error', `Invidious also failed: ${error.message}`);
        throw error;
      }
    }

    if (videos.length === 0 && !this.enableFallback) {
      throw new Error('No available data source');
    }

    // 数据增强
    if (enrichment && videos.length > 0) {
      for (const video of videos) {
        const extra = await this._enrichWithDislike(video.video_id);
        video.dislike_count = extra.dislike_count;
        video.rating = extra.rating;
      }
    }

    // Schema 校验
    for (const video of videos) {
      const validation = this.validateSchema(video);
      if (!validation.valid) {
        this._log('warn', `Schema validation failed for ${video.video_id}: ${validation.errors.join('; ')}`);
      }
    }

    // 写入缓存
    if (!pageToken) {
      this._setCache(cacheKey, videos);
    }

    // 附加元数据
    if (videos.length > 0) {
      videos[0]._source = source;
      videos[0]._fallbackLevel = fallbackLevel;
      videos[0]._quotaUsed = this.quotaUsed;
      videos[0]._quotaRemaining = this.getRemainingQuota();
    }

    return videos;
  }

  /**
   * 获取多地区热门视频
   * @param {Array<string>} regionCodes - 地区代码数组
   * @param {number} [maxResultsPerRegion=10] - 每个地区的最大结果数
   * @param {string|null} [videoCategoryId=null] - 视频分类 ID
   * @returns {Promise<Object>} 以地区代码为键的结果对象
   */
  async get_multi_region_trending(regionCodes, maxResultsPerRegion = 10, videoCategoryId = null) {
    if (!Array.isArray(regionCodes) || regionCodes.length === 0) {
      throw new Error('regionCodes must be a non-empty array');
    }

    const uniqueRegions = [...new Set(regionCodes.map(r => r.toUpperCase()))];

    const estimatedQuota = uniqueRegions.length * (videoCategoryId ? 1 : 1);
    if (this.apiKey && this.getRemainingQuota() < estimatedQuota) {
      this._log('warn', `Quota may be insufficient. Required: ${estimatedQuota}, Remaining: ${this.getRemainingQuota()}`);
    }

    this._log('info', `Fetching trending videos for ${uniqueRegions.length} regions`);

    const results = {};

    for (const region of uniqueRegions) {
      try {
        const videos = await this.get_trending(region, maxResultsPerRegion, videoCategoryId);
        results[region] = videos;
        this._log('info', `Fetched ${videos.length} videos for ${region}`);
      } catch (error) {
        this._log('error', `Failed to fetch for ${region}: ${error.message}`);
        results[region] = [];
      }
    }

    return results;
  }

  /**
   * 批量获取多地区多分类热门视频
   * @param {Array<string>} regionCodes - 地区代码数组
   * @param {Array<string>} categoryIds - 分类 ID 数组
   * @param {number} [maxResultsPerRequest=50] - 每次请求的最大结果数
   * @returns {Promise<Object>} 嵌套结果对象 { region: { category: videos } }
   */
  async batch_trending(regionCodes, categoryIds, maxResultsPerRequest = 50) {
    if (!Array.isArray(regionCodes) || regionCodes.length === 0) {
      throw new Error('regionCodes must be a non-empty array');
    }
    if (!Array.isArray(categoryIds) || categoryIds.length === 0) {
      throw new Error('categoryIds must be a non-empty array');
    }

    const results = {};
    for (const region of regionCodes.map(r => r.toUpperCase())) {
      results[region] = {};
      for (const category of categoryIds) {
        try {
          const videos = await this.get_trending(region, maxResultsPerRequest, String(category));
          results[region][category] = videos;
        } catch (error) {
          this._log('error', `Batch fetch failed for ${region}/${category}: ${error.message}`);
          results[region][category] = [];
        }
      }
    }
    return results;
  }

  /**
   * 三源健康检查
   * @returns {Promise<Object>} 各数据源状态
   */
  async health_check() {
    const results = {
      api_status: 'unknown',
      ytdlp_status: 'unknown',
      invidious_status: 'unknown'
    };

    // Check API
    if (this.apiKey) {
      try {
        await this._fetchFromAPI('US', 1);
        results.api_status = 'ok';
      } catch (error) {
        results.api_status = error.message.includes('quotaExceeded') ? 'quota_exceeded' : 'error';
      }
    } else {
      results.api_status = 'no_key';
    }

    // Check yt-dlp
    try {
      execSync('yt-dlp --version', { encoding: 'utf-8', timeout: 10000 });
      results.ytdlp_status = 'ok';
    } catch (error) {
      results.ytdlp_status = 'error';
    }

    // Check Invidious
    try {
      await fetch('https://vid.puffyan.us/api/v1/trending?region=US', {
        signal: AbortSignal.timeout(10000)
      });
      results.invidious_status = 'ok';
    } catch (error) {
      results.invidious_status = 'error';
    }

    // 推荐方案
    if (results.api_status === 'ok') {
      results.recommended_method = 'api';
    } else if (results.ytdlp_status === 'ok') {
      results.recommended_method = 'ytdlp';
    } else if (results.invidious_status === 'ok') {
      results.recommended_method = 'invidious';
    } else {
      results.recommended_method = 'none';
    }

    return results;
  }

  /**
   * 保存数据为 JSON 文件
   * @param {Array} data - 视频数据数组
   * @param {string} filepath - 文件路径
   * @returns {string} 保存的文件路径
   */
  save_to_json(data, filepath) {
    if (!Array.isArray(data)) {
      throw new Error('data must be an array');
    }

    const dir = path.dirname(filepath);
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }

    const output = {
      crawl_time: new Date().toISOString(),
      videos: data
    };

    fs.writeFileSync(filepath, JSON.stringify(output, null, 2), 'utf-8');
    this._log('info', `Saved ${data.length} records to ${filepath}`);
    return filepath;
  }

  /**
   * 保存数据为 CSV 文件
   * @param {Array} data - 视频数据数组
   * @param {string} filepath - 文件路径
   * @param {string} [delimiter=','] - CSV 分隔符
   * @returns {string} 保存的文件路径
   */
  save_to_csv(data, filepath, delimiter = ',') {
    if (!Array.isArray(data)) {
      throw new Error('data must be an array');
    }

    const dir = path.dirname(filepath);
    if (!fs.existsSync(dir)) {
      fs.mkdirSync(dir, { recursive: true });
    }

    const columns = ['video_id', 'title', 'channel_id', 'channel_title', 'view_count', 'like_count', 'comment_count', 'published_at', 'thumbnail_url', 'duration', 'category_id', 'source_url', 'crawl_time'];

    const header = columns.join(delimiter);
    const rows = data.map(video => {
      return columns.map(col => {
        let value = video[col];
        if (Array.isArray(value)) {
          value = value.join('|');
        }
        if (typeof value === 'string' && (value.includes(delimiter) || value.includes('"') || value.includes('\n'))) {
          value = `"${value.replace(/"/g, '""')}"`;
        }
        return value === null || value === undefined ? '' : value;
      }).join(delimiter);
    });

    const content = [header, ...rows].join('\n');
    fs.writeFileSync(filepath, '\ufeff' + content, 'utf-8');
    this._log('info', `Saved ${data.length} records to ${filepath}`);
    return filepath;
  }
}

/**
 * 获取 Mock 视频数据（独立函数）
 * @param {Object} options - 选项
 * @param {string} [options.regionCode='US'] - 地区代码
 * @param {number} [options.maxResults=10] - 返回数量
 * @param {string|null} [options.videoCategoryId=null] - 分类 ID
 * @returns {Array} Mock 视频数据
 */
/**
 * 兼容旧 API 的函数式入口
 * @param {Object} options
 * @returns {Promise<Array>}
 */
async function getTrendingVideos(options = {}) {
  const crawler = new YouTubeTrendingCrawler(options);
  return crawler.get_trending(
    options.regionCode || 'US',
    options.maxResults || 10,
    options.videoCategoryId || null
  );
}

/**
 * 兼容旧 API 的 API 入口
 * @param {Object} params
 * @returns {Promise<Array>}
 */
async function getTrendingVideosAPI(params) {
  const crawler = new YouTubeTrendingCrawler({ apiKey: params.apiKey });
  return crawler._fetchFromAPI(params.regionCode, params.maxResults, params.videoCategoryId);
}

/**
 * 兼容旧 API 的 yt-dlp 入口
 * @param {Object} params
 * @returns {Promise<Array>}
 */
async function getTrendingVideosYTDLP(params) {
  const crawler = new YouTubeTrendingCrawler();
  return crawler._fetchFromYTDLP(params.regionCode, params.maxResults, params.videoCategoryId);
}

/**
 * 获取 Mock 视频数据（兼容旧 API）
 * @param {Object} options - 选项
 * @param {string} [options.regionCode='US'] - 地区代码
 * @param {number} [options.maxResults=10] - 返回数量
 * @param {string|null} [options.videoCategoryId=null] - 分类 ID
 * @returns {Array} Mock 视频数据
 */
function getMockTrendingVideos(options = {}) {
  const { regionCode = 'US', maxResults = 10, videoCategoryId = null } = options;
  const crawler = new YouTubeTrendingCrawler({ useMock: true });
  return crawler._getMockData(regionCode, maxResults, videoCategoryId);
}

/**
 * 验证地区代码（兼容旧 API，对外暴露）
 * @param {string} code
 * @returns {boolean}
 */
function validateRegionCode(code) {
  if (!code || typeof code !== 'string') return false;
  return /^[A-Z]{2}$/.test(code.toUpperCase()) && REGION_LIST.includes(code.toUpperCase());
}

module.exports = {
  YouTubeTrendingCrawler,
  getTrendingVideos,
  getTrendingVideosAPI,
  getTrendingVideosYTDLP,
  getMockTrendingVideos,
  validateRegionCode
};