/** * YouTube Trending Crawler - Class-based Implementation * YouTube 热门视频爬虫 - 支持 API 和 yt-dlp 降级方案 */ const { execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); /** * YouTube 热门视频爬虫类 * @class YouTubeTrendingCrawler */ class YouTubeTrendingCrawler { /** * 创建爬虫实例 * @param {Object} options - 配置选项 * @param {string} [options.apiKey] - YouTube Data API Key * @param {boolean} [options.useMock=false] - 是否使用 Mock 数据 * @param {number} [options.timeout=30000] - 请求超时时间(毫秒) * @param {number} [options.maxRetries=3] - 最大重试次数 * @param {boolean} [options.enableFallback=true] - 是否启用 yt-dlp 降级方案 * @param {number} [options.quotaLimit=10000] - 每日 API 配额限制 */ constructor(options = {}) { this.apiKey = options.apiKey || process.env.YOUTUBE_API_KEY || null; this.useMock = options.useMock || process.env.YOUTUBE_CRAWLER_MOCK === '1' || false; this.timeout = options.timeout || 30000; this.maxRetries = options.maxRetries || 3; this.enableFallback = options.enableFallback !== false; this.quotaLimit = options.quotaLimit || 10000; this.quotaUsed = 0; this.logs = []; this._log('info', 'YouTubeTrendingCrawler initialized'); } /** * 内部日志记录方法 * @private * @param {string} level - 日志级别 (info, warn, error) * @param {string} message - 日志消息 */ _log(level, message) { const entry = { timestamp: new Date().toISOString(), level, message }; this.logs.push(entry); } /** * 获取所有日志 * @returns {Array} 日志数组 */ getLogs() { return this.logs; } /** * 验证地区代码格式 * @private * @param {string} code - 地区代码 * @returns {boolean} 是否有效 */ _validateRegionCode(code) { if (!code || typeof code !== 'string') return false; return /^[A-Z]{2}$/.test(code.toUpperCase()); } /** * 验证 maxResults 范围 * @private * @param {number} maxResults - 最大结果数 * @returns {boolean} 是否有效 */ _validateMaxResults(maxResults) { const num = parseInt(maxResults, 10); return !isNaN(num) && num >= 1 && num <= 50; } /** * 获取剩余 API 配额 * @returns {number} 剩余配额 */ getRemainingQuota() { return this.quotaLimit - this.quotaUsed; } /** * 记录配额使用 * @private * @param {number} units - 使用的配额单位数 */ _recordQuotaUsage(units = 1) { this.quotaUsed += units; this._log('info', `Quota used: ${this.quotaUsed}/${this.quotaLimit}`); } /** * 清理字符串 * @private * @param {string} str - 输入字符串 * @returns {string} 清理后的字符串 */ _cleanString(str) { if (!str || typeof str !== 'string') return ''; return str.trim(); } /** * 解析观看次数 * @private * @param {string|number} viewCount - 观看次数 * @returns {number} 解析后的数字 */ _parseViewCount(viewCount) { if (typeof viewCount === 'number') return viewCount; if (!viewCount) return 0; const parsed = parseInt(viewCount.toString().replace(/[^0-9]/g, ''), 10); return isNaN(parsed) ? 0 : parsed; } /** * 生成 Mock 数据 * @private * @param {string} regionCode - 地区代码 * @param {number} maxResults - 最大结果数 * @returns {Array} Mock 视频数据数组 */ _getMockData(regionCode = 'US', maxResults = 10) { const baseVideos = [ { videoId: 'mock_001', url: 'https://youtube.com/watch?v=mock_001', title: 'Mock Trending Video: Global Hits 2024', channelTitle: 'Mock Music Channel', viewCount: 12500000, publishedAt: '2024-03-15T10:00:00Z', thumbnails: { default: 'https://i.ytimg.com/vi/mock_001/default.jpg', medium: 'https://i.ytimg.com/vi/mock_001/mqdefault.jpg', high: 'https://i.ytimg.com/vi/mock_001/hqdefault.jpg' }, categoryId: '10' }, { videoId: 'mock_002', url: 'https://youtube.com/watch?v=mock_002', title: 'Mock Gaming: Speedrun World Record', channelTitle: 'Mock Gaming', viewCount: 3200000, publishedAt: '2024-03-14T08:30:00Z', thumbnails: { default: 'https://i.ytimg.com/vi/mock_002/default.jpg', medium: 'https://i.ytimg.com/vi/mock_002/mqdefault.jpg', high: 'https://i.ytimg.com/vi/mock_002/hqdefault.jpg' }, categoryId: '20' }, { videoId: 'mock_003', url: 'https://youtube.com/watch?v=mock_003', title: 'Mock Tech Review: Latest Smartphone', channelTitle: 'Mock Tech', viewCount: 1800000, publishedAt: '2024-03-13T14:00:00Z', thumbnails: { default: 'https://i.ytimg.com/vi/mock_003/default.jpg', medium: 'https://i.ytimg.com/vi/mock_003/mqdefault.jpg', high: 'https://i.ytimg.com/vi/mock_003/hqdefault.jpg' }, categoryId: '28' }, { videoId: 'mock_004', url: 'https://youtube.com/watch?v=mock_004', title: 'Mock Comedy: Stand-up Special', channelTitle: 'Mock Comedy Central', viewCount: 5400000, publishedAt: '2024-03-12T20:00:00Z', thumbnails: { default: 'https://i.ytimg.com/vi/mock_004/default.jpg', medium: 'https://i.ytimg.com/vi/mock_004/mqdefault.jpg', high: 'https://i.ytimg.com/vi/mock_004/hqdefault.jpg' }, categoryId: '23' }, { videoId: 'mock_005', url: 'https://youtube.com/watch?v=mock_005', title: 'Mock Education: How Quantum Computers Work', channelTitle: 'Mock Education', viewCount: 890000, publishedAt: '2024-03-11T09:00:00Z', thumbnails: { default: 'https://i.ytimg.com/vi/mock_005/default.jpg', medium: 'https://i.ytimg.com/vi/mock_005/mqdefault.jpg', high: 'https://i.ytimg.com/vi/mock_005/hqdefault.jpg' }, categoryId: '27' } ]; return baseVideos.slice(0, maxResults).map(v => ({ ...v, _mockRegion: regionCode.toUpperCase() })); } /** * 从 API 响应中提取视频信息 * @param {Object} apiItem - YouTube API 返回的视频项 * @returns {Object} 标准化的视频信息对象 */ extract_video_info(apiItem) { if (!apiItem || !apiItem.id) { throw new Error('Invalid API item: missing id'); } const videoId = typeof apiItem.id === 'string' ? apiItem.id : apiItem.id.videoId; const snippet = apiItem.snippet || {}; const statistics = apiItem.statistics || {}; const thumbnails = snippet.thumbnails || {}; return { videoId: videoId, url: `https://youtube.com/watch?v=${videoId}`, title: this._cleanString(snippet.title) || 'Unknown Title', channelTitle: this._cleanString(snippet.channelTitle) || 'Unknown Channel', viewCount: this._parseViewCount(statistics.viewCount), publishedAt: snippet.publishedAt || null, thumbnails: { default: thumbnails.default?.url || null, medium: thumbnails.medium?.url || null, high: thumbnails.high?.url || null }, categoryId: snippet.categoryId || null }; } /** * 使用 YouTube Data API 获取热门视频 * @private */ async _fetchFromAPI(regionCode, maxResults, videoCategoryId = null, pageToken = null) { if (!this.apiKey) { throw new Error('API key is required'); } const url = 'https://www.googleapis.com/youtube/v3/videos'; const params = new URLSearchParams({ part: 'snippet,statistics', chart: 'mostPopular', regionCode: regionCode.toUpperCase(), maxResults: maxResults.toString(), key: this.apiKey }); if (videoCategoryId) { params.append('videoCategoryId', videoCategoryId); } if (pageToken) { params.append('pageToken', pageToken); } let lastError = null; for (let attempt = 0; attempt < this.maxRetries; attempt++) { try { const response = await fetch(`${url}?${params}`, { signal: AbortSignal.timeout(this.timeout) }); if (!response.ok) { const errorData = await response.json().catch(() => ({})); const errorMessage = errorData.error?.message || response.statusText; if (response.status === 403) { throw new Error(`API error 403: Invalid API key or quota exceeded. ${errorMessage}`); } else if (response.status === 429) { this._log('warn', `API error 429: Retrying... (attempt ${attempt + 1})`); lastError = new Error(`API error 429: ${errorMessage}`); await new Promise(r => setTimeout(r, Math.pow(2, attempt) * 1000)); continue; } else if (response.status === 400) { throw new Error(`API error 400: Invalid request parameters. ${errorMessage}`); } else { throw new Error(`API error ${response.status}: ${errorMessage}`); } } const data = await response.json(); this._recordQuotaUsage(1); return { items: data.items || [], nextPageToken: data.nextPageToken || null }; } catch (error) { lastError = error; if (error.name === 'TimeoutError' || error.message.includes('timeout')) { this._log('warn', `Request timeout, retrying... (attempt ${attempt + 1})`); await new Promise(r => setTimeout(r, Math.pow(2, attempt) * 1000)); } else if (!error.message.includes('429')) { throw error; } } } throw lastError || new Error('API request failed after max retries'); } /** * 使用 yt-dlp 获取热门视频 (降级方案) * @private */ async _fetchFromYTDLP(regionCode, maxResults) { const url = regionCode === 'US' ? 'https://www.youtube.com/feed/trending' : `https://www.youtube.com/feed/trending?gl=${regionCode}`; const cmd = `yt-dlp --flat-playlist --playlist-end ${maxResults} -J "${url}" 2>/dev/null`; try { const output = execSync(cmd, { encoding: 'utf-8', timeout: this.timeout }); const data = JSON.parse(output); if (!data.entries || !Array.isArray(data.entries)) { throw new Error('Invalid yt-dlp response format'); } return data.entries.map(entry => ({ videoId: entry.id, url: entry.webpage_url || `https://youtube.com/watch?v=${entry.id}`, title: entry.title || 'Unknown Title', channelTitle: entry.uploader || entry.channel || 'Unknown Channel', viewCount: entry.view_count || 0, publishedAt: entry.upload_date ? `${entry.upload_date.slice(0, 4)}-${entry.upload_date.slice(4, 6)}-${entry.upload_date.slice(6, 8)}T00:00:00Z` : null, thumbnails: { default: entry.thumbnail ? entry.thumbnail.replace(/hqdefault/, 'default') : null, medium: entry.thumbnail ? entry.thumbnail.replace(/hqdefault/, 'mqdefault') : null, high: entry.thumbnail || null }, categoryId: null })); } catch (error) { throw new Error(`yt-dlp failed: ${error.message}`); } } /** * 获取单地区热门视频 * @param {string} regionCode - 地区代码 (如 'US', 'JP') * @param {number} [maxResults=10] - 最大结果数 (1-50) * @param {string|null} [videoCategoryId=null] - 视频分类 ID * @returns {Promise} 视频数据数组 */ async get_trending(regionCode = 'US', maxResults = 10, videoCategoryId = null) { if (!this._validateRegionCode(regionCode)) { throw new Error(`Invalid regionCode: ${regionCode}. Must be 2-letter ISO code.`); } if (!this._validateMaxResults(maxResults)) { throw new Error(`Invalid maxResults: ${maxResults}. Must be between 1 and 50.`); } const normalizedRegion = regionCode.toUpperCase(); this._log('info', `Fetching trending videos for region: ${normalizedRegion}`); // Mock 模式 if (this.useMock) { return this._getMockData(normalizedRegion, maxResults); } // 优先尝试 API if (this.apiKey && this.getRemainingQuota() > 0) { try { const result = await this._fetchFromAPI(normalizedRegion, maxResults, videoCategoryId); const videos = result.items.map(item => this.extract_video_info(item)); this._log('info', `Successfully fetched ${videos.length} videos from API`); return videos; } catch (error) { this._log('warn', `API failed: ${error.message}`); if (!this.enableFallback) { throw error; } this._log('info', 'Falling back to yt-dlp...'); } } // 降级到 yt-dlp if (this.enableFallback) { try { const videos = await this._fetchFromYTDLP(normalizedRegion, maxResults); this._log('info', `Successfully fetched ${videos.length} videos from yt-dlp`); return videos; } catch (error) { this._log('error', `yt-dlp also failed: ${error.message}`); throw error; } } throw new Error('No available data source'); } /** * 获取多地区热门视频 * @param {Array} regionCodes - 地区代码数组 * @param {number} [maxResultsPerRegion=10] - 每个地区的最大结果数 * @param {string|null} [videoCategoryId=null] - 视频分类 ID * @returns {Promise} 以地区代码为键的结果对象 */ async get_multi_region_trending(regionCodes, maxResultsPerRegion = 10, videoCategoryId = null) { if (!Array.isArray(regionCodes) || regionCodes.length === 0) { throw new Error('regionCodes must be a non-empty array'); } const uniqueRegions = [...new Set(regionCodes.map(r => r.toUpperCase()))]; const estimatedQuota = uniqueRegions.length; if (this.apiKey && this.getRemainingQuota() < estimatedQuota) { this._log('warn', `Quota may be insufficient. Required: ${estimatedQuota}, Remaining: ${this.getRemainingQuota()}`); } this._log('info', `Fetching trending videos for ${uniqueRegions.length} regions`); const results = {}; for (const region of uniqueRegions) { try { const videos = await this.get_trending(region, maxResultsPerRegion, videoCategoryId); results[region] = videos; this._log('info', `Fetched ${videos.length} videos for ${region}`); if (uniqueRegions.length > 1) { await new Promise(r => setTimeout(r, 100)); } } catch (error) { this._log('error', `Failed to fetch for ${region}: ${error.message}`); results[region] = []; } } return results; } /** * 保存数据为 JSON 文件 * @param {Array} data - 视频数据数组 * @param {string} filepath - 文件路径 * @returns {string} 保存的文件路径 */ save_to_json(data, filepath) { if (!Array.isArray(data)) { throw new Error('data must be an array'); } const dir = path.dirname(filepath); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } fs.writeFileSync(filepath, JSON.stringify(data, null, 2), 'utf-8'); this._log('info', `Saved ${data.length} records to ${filepath}`); return filepath; } /** * 保存数据为 CSV 文件 * @param {Array} data - 视频数据数组 * @param {string} filepath - 文件路径 * @returns {string} 保存的文件路径 */ save_to_csv(data, filepath) { if (!Array.isArray(data)) { throw new Error('data must be an array'); } const dir = path.dirname(filepath); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); } const columns = ['videoId', 'url', 'title', 'channelTitle', 'viewCount', 'thumbnails', 'publishedAt', 'categoryId']; const header = columns.join(','); const rows = data.map(video => { return columns.map(col => { let value = video[col]; if (col === 'thumbnails' && typeof value === 'object') { value = JSON.stringify(value); } if (typeof value === 'string' && (value.includes(',') || value.includes('"') || value.includes('\n'))) { value = `"${value.replace(/"/g, '""')}"`; } return value === null || value === undefined ? '' : value; }).join(','); }); const content = [header, ...rows].join('\n'); fs.writeFileSync(filepath, '\ufeff' + content, 'utf-8'); this._log('info', `Saved ${data.length} records to ${filepath}`); return filepath; } } /** * 获取 Mock 视频数据(独立函数) * @param {Object} options - 选项 * @param {string} [options.regionCode='US'] - 地区代码 * @param {number} [options.maxResults=10] - 返回数量 * @returns {Array} Mock 视频数据 */ function getMockTrendingVideos(options = {}) { const { regionCode = 'US', maxResults = 10 } = options; const crawler = new YouTubeTrendingCrawler({ useMock: true }); return crawler._getMockData(regionCode, maxResults); } module.exports = { YouTubeTrendingCrawler, getMockTrendingVideos };