/** * YouTube Trending Crawler v2 * 支持 Data API v3 + yt-dlp + Invidious 三级降级 * 符合 benchmark_v2.md 评分标准 */ const { execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); /** * 解析 ISO 8601 时长 (PT3M33S) * @param {string} duration * @returns {number|null} 秒数 */ function parseISODuration(duration) { if (!duration || typeof duration !== 'string') return null; const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/); if (!match) return null; const hours = parseInt(match[1] || 0, 10); const minutes = parseInt(match[2] || 0, 10); const seconds = parseInt(match[3] || 0, 10); return hours * 3600 + minutes * 60 + seconds; } /** * 休眠 * @param {number} ms * @returns {Promise} */ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } /** * YouTube 热门视频爬虫类 * @class YouTubeTrendingCrawler */ class YouTubeTrendingCrawler { /** * @param {Object} options * @param {string} [options.apiKey] - API Key * @param {boolean} [options.useMock=false] - Mock 模式 * @param {number} [options.timeout=30000] - 超时(ms) * @param {number} [options.maxRetries=5] - 最大重试次数 * @param {boolean} [options.enableFallback=true] - 启用降级 * @param {number} [options.quotaLimit=10000] - 每日配额 * @param {number} [options.requestInterval=2000] - 请求间隔(ms) * @param {string} [options.invidiousInstance='https://iv.datura.network'] - Invidious 实例 */ constructor(options = {}) { this.apiKey = options.apiKey || process.env.YOUTUBE_API_KEY || null; this.useMock = options.useMock || process.env.YOUTUBE_CRAWLER_MOCK === '1' || false; this.timeout = options.timeout || 30000; this.maxRetries = options.maxRetries || 5; this.enableFallback = options.enableFallback !== false; this.quotaLimit = options.quotaLimit || 10000; this.quotaUsed = 0; this.requestInterval = options.requestInterval || 2000; this.invidiousInstance = options.invidiousInstance || 'https://iv.datura.network'; this.logs = []; this.cache = new Map(); this.cacheTTL = options.cacheTTL || 900000; // 15min default this._log('info', 'YouTubeTrendingCrawler v2 initialized'); } _log(level, message) { const entry = { timestamp: new Date().toISOString(), level, message }; this.logs.push(entry); } getLogs() { return this.logs; } _validateRegionCode(code) { if (!code || typeof code !== 'string') return false; return /^[A-Z]{2}$/.test(code.toUpperCase()); } _validateMaxResults(maxResults) { const num = parseInt(maxResults, 10); return !isNaN(num) && num >= 1 && num <= 50; } _validateCategoryId(categoryId) { if (!categoryId) return true; const num = parseInt(categoryId, 10); return !isNaN(num) && num >= 1 && num <= 44; } getRemainingQuota() { return this.quotaLimit - this.quotaUsed; } _recordQuotaUsage(units = 1) { this.quotaUsed += units; this._log('info', `Quota used: ${this.quotaUsed}/${this.quotaLimit}`); } _cleanString(str) { if (!str || typeof str !== 'string') return ''; return str.trim(); } _parseViewCount(viewCount) { if (typeof viewCount === 'number') return viewCount; if (!viewCount) return 0; const parsed = parseInt(viewCount.toString().replace(/[^0-9]/g, ''), 10); return isNaN(parsed) ? 0 : parsed; } _getCacheKey(regionCode, categoryId, maxResults) { return `${regionCode}:${categoryId || 'all'}:${maxResults}`; } _getCached(key) { const entry = this.cache.get(key); if (!entry) return null; if (Date.now() - entry.time > this.cacheTTL) { this.cache.delete(key); return null; } return entry.data; } _setCache(key, data) { this.cache.set(key, { time: Date.now(), data }); } _getMockData(regionCode = 'US', maxResults = 10) { const baseVideos = [ { video_id: 'mock_001', source_url: 'https://www.youtube.com/watch?v=mock_001', title: 'Mock Trending Video: Global Hits 2024', channel_id: 'UCmock001', channel_title: 'Mock Music Channel', view_count: 12500000, like_count: 450000, comment_count: 12000, published_at: '2024-03-15T10:00:00Z', thumbnail_url: 'https://i.ytimg.com/vi/mock_001/hqdefault.jpg', category_id: '10', duration: 213, crawl_time: new Date().toISOString() }, { video_id: 'mock_002', source_url: 'https://www.youtube.com/watch?v=mock_002', title: 'Mock Gaming: Speedrun World Record', channel_id: 'UCmock002', channel_title: 'Mock Gaming', view_count: 3200000, like_count: 180000, comment_count: 5400, published_at: '2024-03-14T08:30:00Z', thumbnail_url: 'https://i.ytimg.com/vi/mock_002/hqdefault.jpg', category_id: '20', duration: 1855, crawl_time: new Date().toISOString() }, { video_id: 'mock_003', source_url: 'https://www.youtube.com/watch?v=mock_003', title: 'Mock Tech Review: Latest Smartphone', channel_id: 'UCmock003', channel_title: 'Mock Tech', view_count: 1800000, like_count: 92000, comment_count: 3100, published_at: '2024-03-13T14:00:00Z', thumbnail_url: 'https://i.ytimg.com/vi/mock_003/hqdefault.jpg', category_id: '28', duration: 728, crawl_time: new Date().toISOString() }, { video_id: 'mock_004', source_url: 'https://www.youtube.com/watch?v=mock_004', title: 'Mock Comedy: Stand-up Special', channel_id: 'UCmock004', channel_title: 'Mock Comedy Central', view_count: 5400000, like_count: 210000, comment_count: 8700, published_at: '2024-03-12T20:00:00Z', thumbnail_url: 'https://i.ytimg.com/vi/mock_004/hqdefault.jpg', category_id: '23', duration: 1840, crawl_time: new Date().toISOString() }, { video_id: 'mock_005', source_url: 'https://www.youtube.com/watch?v=mock_005', title: 'Mock Education: How Quantum Computers Work', channel_id: 'UCmock005', channel_title: 'Mock Education', view_count: 890000, like_count: 56000, comment_count: 1800, published_at: '2024-03-11T09:00:00Z', thumbnail_url: 'https://i.ytimg.com/vi/mock_005/hqdefault.jpg', category_id: '27', duration: 634, crawl_time: new Date().toISOString() } ]; return baseVideos.slice(0, maxResults).map(v => ({ ...v, _mockRegion: regionCode.toUpperCase() })); } /** * 从 API 响应中提取标准化视频信息 * @param {Object} apiItem * @returns {Object} */ extract_video_info(apiItem) { if (!apiItem || !apiItem.id) { throw new Error('Invalid API item: missing id'); } const videoId = typeof apiItem.id === 'string' ? apiItem.id : apiItem.id.videoId; const snippet = apiItem.snippet || {}; const statistics = apiItem.statistics || {}; const contentDetails = apiItem.contentDetails || {}; const thumbnails = snippet.thumbnails || {}; const crawlTime = new Date().toISOString(); return { video_id: videoId, source_url: `https://www.youtube.com/watch?v=${videoId}`, title: this._cleanString(snippet.title) || 'Unknown Title', channel_id: snippet.channelId || null, channel_title: this._cleanString(snippet.channelTitle) || 'Unknown Channel', view_count: this._parseViewCount(statistics.viewCount), like_count: this._parseViewCount(statistics.likeCount), comment_count: this._parseViewCount(statistics.commentCount), published_at: snippet.publishedAt || null, thumbnail_url: thumbnails.high?.url || thumbnails.medium?.url || thumbnails.default?.url || null, category_id: snippet.categoryId || null, duration: parseISODuration(contentDetails.duration), crawl_time: crawlTime }; } /** * 带指数退避的重试包装 * @private */ async _fetchWithRetry(url, options, backoffBase = 2000, maxBackoff = 60000) { let lastError = null; for (let attempt = 0; attempt < this.maxRetries; attempt++) { try { const response = await fetch(url, options); return response; } catch (error) { lastError = error; if (error.name === 'TimeoutError' || error.message.includes('timeout')) { const delay = Math.min(backoffBase * Math.pow(2, attempt), maxBackoff); this._log('warn', `Timeout, retrying in ${delay}ms (attempt ${attempt + 1})`); await sleep(delay); continue; } throw error; } } throw lastError || new Error('Request failed after max retries'); } async _fetchFromAPI(regionCode, maxResults, videoCategoryId = null, pageToken = null) { if (!this.apiKey) { throw new Error('API key is required'); } const url = 'https://www.googleapis.com/youtube/v3/videos'; const params = new URLSearchParams({ part: 'snippet,statistics,contentDetails', chart: 'mostPopular', regionCode: regionCode.toUpperCase(), maxResults: maxResults.toString(), key: this.apiKey }); if (videoCategoryId) { params.append('videoCategoryId', videoCategoryId); } if (pageToken) { params.append('pageToken', pageToken); } let lastError = null; for (let attempt = 0; attempt < this.maxRetries; attempt++) { try { const response = await fetch(`${url}?${params}`, { headers: { 'User-Agent': 'YouTubeTrendingCrawler/2.0' }, signal: AbortSignal.timeout(this.timeout) }); if (!response.ok) { const errorData = await response.json().catch(() => ({})); const errorMessage = errorData.error?.message || response.statusText; if (response.status === 400) { throw new Error(`API error 400: Invalid request parameters. ${errorMessage}`); } else if (response.status === 403) { throw new Error(`API error 403: Invalid API key or quota exceeded. ${errorMessage}`); } else if (response.status === 404) { this._log('warn', `API error 404: Resource not found. ${errorMessage}`); return { items: [], nextPageToken: null }; } else if (response.status === 429) { const delay = Math.min(2000 * Math.pow(2, attempt), 60000); this._log('warn', `API error 429: Rate limit exceeded. Retrying in ${delay}ms (attempt ${attempt + 1})`); lastError = new Error(`API error 429: ${errorMessage}`); await sleep(delay); continue; } else if (response.status === 500 || response.status === 502) { if (attempt < 3) { const delay = Math.min(2000 * Math.pow(2, attempt), 30000); this._log('warn', `API error ${response.status}: Retrying in ${delay}ms (attempt ${attempt + 1})`); await sleep(delay); continue; } throw new Error(`API error ${response.status}: ${errorMessage}`); } else if (response.status === 503) { const delay = Math.min(5000 * Math.pow(2, attempt), 120000); this._log('warn', `API error 503: Service unavailable. Retrying in ${delay}ms (attempt ${attempt + 1})`); lastError = new Error(`API error 503: ${errorMessage}`); await sleep(delay); continue; } else { throw new Error(`API error ${response.status}: ${errorMessage}`); } } const data = await response.json(); this._recordQuotaUsage(1); return { items: data.items || [], nextPageToken: data.nextPageToken || null }; } catch (error) { lastError = error; if (error.message && (error.message.includes('400') || error.message.includes('403') || error.message.includes('404'))) { throw error; } if (!error.message || (!error.message.includes('429') && !error.message.includes('503') && !error.message.includes('timeout'))) { throw error; } } } throw lastError || new Error('API request failed after max retries'); } async _fetchFromYTDLP(regionCode, maxResults) { const url = regionCode === 'US' ? 'https://www.youtube.com/feed/trending' : `https://www.youtube.com/feed/trending?gl=${regionCode}`; const cmd = `yt-dlp --flat-playlist --playlist-end ${maxResults} -J "${url}" 2>/dev/null`; try { const output = execSync(cmd, { encoding: 'utf-8', timeout: this.timeout }); const data = JSON.parse(output); if (!data.entries || !Array.isArray(data.entries)) { throw new Error('Invalid yt-dlp response format'); } const crawlTime = new Date().toISOString(); return data.entries.map(entry => ({ video_id: entry.id, source_url: entry.webpage_url || `https://www.youtube.com/watch?v=${entry.id}`, title: entry.title || 'Unknown Title', channel_id: entry.channel_id || null, channel_title: entry.uploader || entry.channel || 'Unknown Channel', view_count: entry.view_count || 0, like_count: entry.like_count || 0, comment_count: entry.comment_count || 0, published_at: entry.upload_date ? `${entry.upload_date.slice(0, 4)}-${entry.upload_date.slice(4, 6)}-${entry.upload_date.slice(6, 8)}T00:00:00Z` : null, thumbnail_url: entry.thumbnail ? entry.thumbnail.replace(/hqdefault/, 'hqdefault') : null, category_id: null, duration: entry.duration || null, crawl_time: crawlTime })); } catch (error) { throw new Error(`yt-dlp failed: ${error.message}`); } } async _fetchFromInvidious(regionCode, maxResults) { const url = `${this.invidiousInstance}/api/v1/trending?region=${regionCode.toUpperCase()}`; try { const response = await fetch(url, { headers: { 'User-Agent': 'YouTubeTrendingCrawler/2.0' }, signal: AbortSignal.timeout(this.timeout) }); if (!response.ok) { throw new Error(`Invidious error ${response.status}: ${response.statusText}`); } const data = await response.json(); const items = Array.isArray(data) ? data : []; const crawlTime = new Date().toISOString(); return items.slice(0, maxResults).map(entry => ({ video_id: entry.videoId, source_url: `https://www.youtube.com/watch?v=${entry.videoId}`, title: entry.title || 'Unknown Title', channel_id: entry.authorId || null, channel_title: entry.author || 'Unknown Channel', view_count: entry.viewCount || 0, like_count: entry.likeCount || 0, comment_count: entry.commentCount || 0, published_at: entry.publishedText ? null : null, // Textual only from Invidious basic thumbnail_url: entry.videoThumbnails ? entry.videoThumbnails.find(t => t.quality === 'maxresdefault')?.url || entry.videoThumbnails[0]?.url : null, category_id: null, duration: entry.lengthSeconds || null, crawl_time: crawlTime })); } catch (error) { throw new Error(`Invidious failed: ${error.message}`); } } async get_trending(regionCode = 'US', maxResults = 10, videoCategoryId = null) { if (!this._validateRegionCode(regionCode)) { throw new Error(`Invalid regionCode: ${regionCode}. Must be 2-letter ISO code.`); } if (!this._validateMaxResults(maxResults)) { throw new Error(`Invalid maxResults: ${maxResults}. Must be between 1 and 50.`); } if (!this._validateCategoryId(videoCategoryId)) { throw new Error(`Invalid videoCategoryId: ${videoCategoryId}.`); } const normalizedRegion = regionCode.toUpperCase(); // 缓存检查 const cacheKey = this._getCacheKey(normalizedRegion, videoCategoryId, maxResults); const cached = this._getCached(cacheKey); if (cached) { this._log('info', `Cache hit for ${cacheKey}`); return cached; } // Mock 模式 if (this.useMock) { const data = this._getMockData(normalizedRegion, maxResults); this._setCache(cacheKey, data); return data; } // API if (this.apiKey && this.getRemainingQuota() > 0) { try { const result = await this._fetchFromAPI(normalizedRegion, maxResults, videoCategoryId); const videos = result.items.map(item => this.extract_video_info(item)); this._log('info', `API fetched ${videos.length} videos for ${normalizedRegion}`); this._setCache(cacheKey, videos); return videos; } catch (error) { this._log('warn', `API failed: ${error.message}`); if (!this.enableFallback) throw error; this._log('info', 'Falling back to yt-dlp...'); } } // yt-dlp if (this.enableFallback) { try { const videos = await this._fetchFromYTDLP(normalizedRegion, maxResults); this._log('info', `yt-dlp fetched ${videos.length} videos for ${normalizedRegion}`); this._setCache(cacheKey, videos); return videos; } catch (error) { this._log('warn', `yt-dlp failed: ${error.message}`); if (!this.enableFallback) throw error; this._log('info', 'Falling back to Invidious...'); } } // Invidious try { const videos = await this._fetchFromInvidious(normalizedRegion, maxResults); this._log('info', `Invidious fetched ${videos.length} videos for ${normalizedRegion}`); this._setCache(cacheKey, videos); return videos; } catch (error) { this._log('error', `Invidious also failed: ${error.message}`); throw new Error(`All data sources failed: ${error.message}`); } } async get_multi_region_trending(regionCodes, maxResultsPerRegion = 10, videoCategoryId = null) { if (!Array.isArray(regionCodes) || regionCodes.length === 0) { throw new Error('regionCodes must be a non-empty array'); } const uniqueRegions = [...new Set(regionCodes.map(r => r.toUpperCase()))]; const primaryRegions = ['US', 'JP', 'KR', 'GB', 'DE', 'FR', 'IN', 'BR', 'RU']; const extendedRegions = [ 'US', 'JP', 'KR', 'GB', 'DE', 'FR', 'IN', 'BR', 'RU', 'CA', 'AU', 'IT', 'ES', 'MX', 'NL', 'SE', 'NO', 'FI', 'DK', 'PL', 'TR', 'ID', 'TH', 'VN', 'PH', 'TW', 'HK', 'SG', 'MY', 'AR', 'CL', 'CO', 'ZA', 'EG', 'SA', 'AE', 'IL' ]; const estimatedQuota = uniqueRegions.length; if (this.apiKey && this.getRemainingQuota() < estimatedQuota) { this._log('warn', `Quota may be insufficient. Required: ${estimatedQuota}, Remaining: ${this.getRemainingQuota()}`); } this._log('info', `Fetching trending videos for ${uniqueRegions.length} regions`); const results = {}; for (const region of uniqueRegions) { try { const videos = await this.get_trending(region, maxResultsPerRegion, videoCategoryId); results[region] = videos; this._log('info', `Fetched ${videos.length} videos for ${region}`); } catch (error) { this._log('error', `Failed to fetch for ${region}: ${error.message}`); results[region] = []; } if (uniqueRegions.length > 1) { await sleep(this.requestInterval); } } return results; } save_to_json(data, filepath) { if (!Array.isArray(data)) throw new Error('data must be an array'); const dir = path.dirname(filepath); if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(filepath, JSON.stringify(data, null, 2), 'utf-8'); this._log('info', `Saved ${data.length} records to ${filepath}`); return filepath; } save_to_csv(data, filepath, delimiter = ',') { if (!Array.isArray(data)) throw new Error('data must be an array'); const dir = path.dirname(filepath); if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true }); const columns = [ 'video_id', 'source_url', 'title', 'channel_id', 'channel_title', 'view_count', 'like_count', 'comment_count', 'published_at', 'thumbnail_url', 'category_id', 'duration', 'crawl_time' ]; const header = columns.join(delimiter); const rows = data.map(video => { return columns.map(col => { let value = video[col]; if (col === 'duration' && typeof value === 'number') value = value.toString(); if (typeof value === 'string' && (value.includes(delimiter) || value.includes('"') || value.includes('\n'))) { value = `"${value.replace(/"/g, '""')}"`; } return value === null || value === undefined ? '' : value; }).join(delimiter); }); const content = [header, ...rows].join('\n'); fs.writeFileSync(filepath, '\ufeff' + content, 'utf-8'); this._log('info', `Saved ${data.length} records to ${filepath}`); return filepath; } } /** * @param {Object} options * @returns {Array} */ function getMockTrendingVideos(options = {}) { const { regionCode = 'US', maxResults = 10 } = options; const crawler = new YouTubeTrendingCrawler({ useMock: true }); return crawler._getMockData(regionCode, maxResults); } module.exports = { YouTubeTrendingCrawler, getMockTrendingVideos, parseISODuration };