/** * YouTube Trending Crawler v2.0 * 支持 API / yt-dlp / Invidious 三级降级 */ const { execSync } = require('child_process'); const fs = require('fs'); const path = require('path'); /** * Parse ISO 8601 duration (PT#M#S) to seconds */ function parseDuration(duration) { if (!duration || typeof duration !== 'string') return null; const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/); if (!match) return null; const hours = parseInt(match[1] || 0, 10); const minutes = parseInt(match[2] || 0, 10); const seconds = parseInt(match[3] || 0, 10); return hours * 3600 + minutes * 60 + seconds; } /** * Sleep utility */ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } /** * Exponential backoff delay */ function getBackoffDelay(attempt, base = 2000, max = 60000) { const delay = base * Math.pow(2, attempt) + Math.random() * 1000; return Math.min(delay, max); } class YouTubeTrendingCrawler { constructor(options = {}) { this.apiKey = options.apiKey || process.env.YOUTUBE_API_KEY || null; this.timeout = options.timeout || 30000; this.maxRetries = options.maxRetries || 5; this.enableFallback = options.enableFallback !== false; this.quotaLimit = options.quotaLimit || 10000; this.quotaUsed = 0; this.requestDelayMs = (options.requestDelay || 2) * 1000; this.useMock = options.useMock || false; this.cache = new Map(); this.cacheTTL = (options.cacheTTL || 900) * 1000; this.logs = []; this.userAgent = options.userAgent || 'YouTubeTrendingCrawler/2.0 (Node.js)'; } _log(level, message) { const entry = { timestamp: new Date().toISOString(), level, message }; this.logs.push(entry); if (level === 'error') console.error(`[${level}] ${message}`); else console.log(`[${level}] ${message}`); } _getCacheKey(regionCode, videoCategoryId, maxResults) { return `${regionCode}:${videoCategoryId || 'all'}:${maxResults}`; } _getCached(key) { const entry = this.cache.get(key); if (!entry) return null; if (Date.now() - entry.time > this.cacheTTL) { this.cache.delete(key); return null; } return entry.data; } _setCache(key, data) { this.cache.set(key, { data, time: Date.now() }); } validateRegionCode(code) { if (!code || typeof code !== 'string') return false; return /^[A-Z]{2}$/i.test(code); } validateCategoryId(id) { if (id === null || id === undefined) return true; const valid = [1, 2, 10, 15, 17, 20, 22, 23, 24, 25, 27, 28, 42]; return valid.includes(Number(id)); } async _fetchWithRetry(url, options = {}, retryOpts = {}) { const maxRetries = retryOpts.maxRetries ?? this.maxRetries; const baseDelay = retryOpts.baseDelay ?? 2000; const maxDelay = retryOpts.maxDelay ?? 60000; for (let attempt = 0; attempt <= maxRetries; attempt++) { try { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), this.timeout); const response = await fetch(url, { ...options, signal: controller.signal, headers: { 'User-Agent': this.userAgent, ...(options.headers || {}) } }); clearTimeout(timer); return response; } catch (err) { if (attempt === maxRetries) throw err; const delay = getBackoffDelay(attempt, baseDelay, maxDelay); this._log('warn', `Request failed (attempt ${attempt + 1}), retrying after ${delay}ms`); await sleep(delay); } } throw new Error('Max retries exceeded'); } async _makeAPIRequest(endpoint, params) { const url = new URL(`https://www.googleapis.com/youtube/v3/${endpoint}`); Object.entries(params).forEach(([k, v]) => { if (v !== null && v !== undefined) url.searchParams.append(k, v); }); const response = await this._fetchWithRetry(url.toString(), { method: 'GET' }); if (!response.ok) { const error = await response.json().catch(() => ({})); const reason = error.error?.errors?.[0]?.reason || ''; const err = new Error(`API error ${response.status}: ${error.error?.message || response.statusText}`); err.status = response.status; err.reason = reason; err.isQuotaExceeded = response.status === 403 && reason === 'quotaExceeded'; err.isRateLimit = response.status === 429; throw err; } this.quotaUsed += 1; return response.json(); } extractVideoInfo(item) { if (!item || !item.id) return null; const snippet = item.snippet || {}; const statistics = item.statistics || {}; const contentDetails = item.contentDetails || {}; const thumbnails = snippet.thumbnails || {}; const publishedAt = snippet.publishedAt || null; const duration = contentDetails.duration || null; const thumbnailUrl = thumbnails.high?.url || thumbnails.medium?.url || thumbnails.default?.url || null; return { video_id: item.id, title: snippet.title || null, channel_id: snippet.channelId || null, channel_title: snippet.channelTitle || null, published_at: publishedAt, view_count: statistics.viewCount ? parseInt(statistics.viewCount, 10) : null, like_count: statistics.likeCount ? parseInt(statistics.likeCount, 10) : null, comment_count: statistics.commentCount ? parseInt(statistics.commentCount, 10) : null, category_id: snippet.categoryId || null, duration: duration, duration_seconds: parseDuration(duration), thumbnail_url: thumbnailUrl, tags: Array.isArray(snippet.tags) ? snippet.tags : [], source_url: `https://www.youtube.com/watch?v=${item.id}`, crawl_time: new Date().toISOString() }; } validateVideoSchema(video) { if (!video || typeof video !== 'object') return false; const required = ['video_id', 'title', 'channel_id', 'channel_title', 'published_at', 'view_count', 'category_id', 'source_url', 'crawl_time']; return required.every(k => video[k] !== undefined); } async getTrendingAPI({ regionCode = 'US', maxResults = 10, videoCategoryId = null, pageToken = null }) { if (!this.apiKey) { throw Object.assign(new Error('API key is required'), { status: 400, reason: 'key_required' }); } const params = { part: 'snippet,statistics,contentDetails', chart: 'mostPopular', regionCode: regionCode.toUpperCase(), maxResults: Math.min(Math.max(1, maxResults), 50).toString(), key: this.apiKey }; if (videoCategoryId !== null && videoCategoryId !== undefined) { params.videoCategoryId = String(videoCategoryId); } if (pageToken) { params.pageToken = pageToken; } await sleep(this.requestDelayMs); const data = await this._makeAPIRequest('videos', params); const videos = (data.items || []) .map(item => this.extractVideoInfo(item)) .filter(v => v !== null); return { videos, next_page_token: data.nextPageToken || null, total_results: data.pageInfo?.totalResults || videos.length, quota_used: this.quotaUsed, source: 'api' }; } async getTrendingYTDLP({ regionCode = 'US', maxResults = 10, videoCategoryId = null }) { const region = regionCode.toUpperCase(); let url = `https://www.youtube.com/feed/trending`; if (region !== 'US') { url += `?gl=${region}`; } if (videoCategoryId) { url = `https://www.youtube.com/feed/trending?gl=${region}`; } const cmd = `yt-dlp --flat-playlist --playlist-end ${maxResults} -J "${url}" 2>/dev/null`; try { const output = execSync(cmd, { encoding: 'utf-8', timeout: this.timeout }); const data = JSON.parse(output); if (!data.entries || !Array.isArray(data.entries)) { throw new Error('Invalid yt-dlp response format'); } const videos = data.entries.map(entry => { const videoId = entry.id; const uploadDate = entry.upload_date; return { video_id: videoId, title: entry.title || 'Unknown', channel_id: entry.channel_id || null, channel_title: entry.uploader || entry.channel || 'Unknown', published_at: uploadDate ? `${uploadDate.slice(0, 4)}-${uploadDate.slice(4, 6)}-${uploadDate.slice(6, 8)}T00:00:00Z` : null, view_count: entry.view_count ? parseInt(entry.view_count, 10) : null, like_count: null, comment_count: null, category_id: null, duration: entry.duration ? `PT${Math.floor(entry.duration / 60)}M${entry.duration % 60}S` : null, duration_seconds: entry.duration || null, thumbnail_url: entry.thumbnail || null, tags: entry.tags || [], source_url: `https://www.youtube.com/watch?v=${videoId}`, crawl_time: new Date().toISOString() }; }); return { videos, source: 'ytdlp' }; } catch (error) { throw Object.assign(new Error(`yt-dlp failed: ${error.message}`), { source: 'ytdlp' }); } } async getTrendingInvidious({ regionCode = 'US', maxResults = 10, videoCategoryId = null }) { const region = regionCode.toUpperCase(); const instances = [ 'https://vid.puffyan.us', 'https://y.com.sb', 'https://iv.nboeck.de' ]; for (const instance of instances) { try { let url = `${instance}/api/v1/trending?region=${region}`; if (videoCategoryId) { const catMap = { 10: 'music', 20: 'gaming', 24: 'movies', 28: 'tech' }; const type = catMap[videoCategoryId] || ''; if (type) url += `&type=${type}`; } const response = await this._fetchWithRetry(url, { method: 'GET' }, { maxRetries: 2, baseDelay: 1000 }); if (!response.ok) continue; const data = await response.json(); const items = Array.isArray(data) ? data : (data.videos || []); const videos = items.slice(0, maxResults).map(item => ({ video_id: item.videoId || item.videoIdFallback || '', title: item.title || 'Unknown', channel_id: item.authorId || null, channel_title: item.author || 'Unknown', published_at: item.publishedText || null, view_count: item.viewCount ? parseInt(item.viewCount, 10) : null, like_count: item.likeCount ? parseInt(item.likeCount, 10) : null, comment_count: null, category_id: String(videoCategoryId) || null, duration: item.lengthSeconds ? `PT${Math.floor(item.lengthSeconds / 60)}M${item.lengthSeconds % 60}S` : null, duration_seconds: item.lengthSeconds || null, thumbnail_url: item.videoThumbnails?.[0]?.url || item.thumbnail || null, tags: item.keywords || [], source_url: `https://www.youtube.com/watch?v=${item.videoId || ''}`, crawl_time: new Date().toISOString() })); return { videos, source: 'invidious' }; } catch (err) { this._log('warn', `Invidious instance ${instance} failed: ${err.message}`); } } throw new Error('All Invidious instances failed'); } async getTrending(regionCode = 'US', maxResults = 10, videoCategoryId = null, pageToken = null) { if (!this.validateRegionCode(regionCode)) { const err = new Error(`Invalid regionCode: ${regionCode}. Must be 2-letter ISO code.`); err.status = 400; err.reason = 'invalid_region'; throw err; } const maxRes = Math.min(Math.max(1, maxResults), 50); if (maxResults < 1 || maxResults > 50) { const err = new Error(`Invalid maxResults: ${maxResults}. Must be between 1 and 50.`); err.status = 400; err.reason = 'invalid_parameter'; throw err; } if (!this.validateCategoryId(videoCategoryId)) { const err = new Error(`Invalid videoCategoryId: ${videoCategoryId}`); err.status = 400; err.reason = 'invalid_category'; throw err; } const normalizedRegion = regionCode.toUpperCase(); if (this.useMock || process.env.YOUTUBE_CRAWLER_MOCK === '1') { return this._getMockData(normalizedRegion, maxRes, videoCategoryId); } const cacheKey = this._getCacheKey(normalizedRegion, videoCategoryId, maxRes); const cached = this._getCached(cacheKey); if (cached) { return { ...cached, cached: true }; } const sources = []; if (this.apiKey) sources.push('api'); if (this.enableFallback) { if (!sources.includes('ytdlp')) sources.push('ytdlp'); if (!sources.includes('invidious')) sources.push('invidious'); } let lastError = null; let fallbackLevel = 0; for (const source of sources) { fallbackLevel += 1; try { let result; if (source === 'api') { result = await this.getTrendingAPI({ regionCode: normalizedRegion, maxResults: maxRes, videoCategoryId, pageToken }); } else if (source === 'ytdlp') { result = await this.getTrendingYTDLP({ regionCode: normalizedRegion, maxResults: maxRes, videoCategoryId }); } else if (source === 'invidious') { result = await this.getTrendingInvidious({ regionCode: normalizedRegion, maxResults: maxRes, videoCategoryId }); } result.fallback_level = fallbackLevel; this._setCache(cacheKey, result); return result; } catch (error) { lastError = error; this._log('warn', `Source ${source} failed: ${error.message}`); if (error.status === 403 && error.reason === 'quotaExceeded') { continue; } if (error.status === 429) { continue; } if (error.status === 503) { continue; } if (error.message && error.message.includes('yt-dlp failed')) { continue; } if (source === 'api' && this.enableFallback) { continue; } break; } } const finalErr = lastError || new Error('All sources failed'); finalErr.fallback_level = fallbackLevel; throw finalErr; } _getMockData(regionCode, maxResults, videoCategoryId) { const baseVideos = [ { video_id: 'mock_001', title: 'Mock Trending Video: Global Hits 2024', channel_id: 'UC_MOCK_001', channel_title: 'Mock Music Channel', published_at: '2024-03-15T10:00:00Z', view_count: 12500000, like_count: 450000, comment_count: 12000, category_id: '10', duration: 'PT3M45S', duration_seconds: 225, thumbnail_url: 'https://img.youtube.com/vi/mock_001/hqdefault.jpg', tags: ['mock', 'music'], source_url: 'https://www.youtube.com/watch?v=mock_001', crawl_time: new Date().toISOString() }, { video_id: 'mock_002', title: 'Mock Gaming: Speedrun World Record', channel_id: 'UC_MOCK_002', channel_title: 'Mock Gaming', published_at: '2024-03-14T08:30:00Z', view_count: 3200000, like_count: 180000, comment_count: 5400, category_id: '20', duration: 'PT15M20S', duration_seconds: 920, thumbnail_url: 'https://img.youtube.com/vi/mock_002/hqdefault.jpg', tags: ['mock', 'gaming'], source_url: 'https://www.youtube.com/watch?v=mock_002', crawl_time: new Date().toISOString() }, { video_id: 'mock_003', title: 'Mock Tech Review: Latest Smartphone', channel_id: 'UC_MOCK_003', channel_title: 'Mock Tech', published_at: '2024-03-13T14:00:00Z', view_count: 1800000, like_count: 95000, comment_count: 3200, category_id: '28', duration: 'PT12M00S', duration_seconds: 720, thumbnail_url: 'https://img.youtube.com/vi/mock_003/hqdefault.jpg', tags: ['mock', 'tech'], source_url: 'https://www.youtube.com/watch?v=mock_003', crawl_time: new Date().toISOString() }, { video_id: 'mock_004', title: 'Mock Comedy: Stand-up Special', channel_id: 'UC_MOCK_004', channel_title: 'Mock Comedy Central', published_at: '2024-03-12T20:00:00Z', view_count: 5400000, like_count: 210000, comment_count: 8500, category_id: '23', duration: 'PT8M30S', duration_seconds: 510, thumbnail_url: 'https://img.youtube.com/vi/mock_004/hqdefault.jpg', tags: ['mock', 'comedy'], source_url: 'https://www.youtube.com/watch?v=mock_004', crawl_time: new Date().toISOString() }, { video_id: 'mock_005', title: 'Mock Education: How Quantum Computers Work', channel_id: 'UC_MOCK_005', channel_title: 'Mock Education', published_at: '2024-03-11T09:00:00Z', view_count: 890000, like_count: 67000, comment_count: 2100, category_id: '27', duration: 'PT18M45S', duration_seconds: 1125, thumbnail_url: 'https://img.youtube.com/vi/mock_005/hqdefault.jpg', tags: ['mock', 'education'], source_url: 'https://www.youtube.com/watch?v=mock_005', crawl_time: new Date().toISOString() } ]; let videos = baseVideos.slice(0, maxResults).map(v => ({ ...v, _mockRegion: regionCode })); if (videoCategoryId) { videos = videos.filter(v => v.category_id === String(videoCategoryId)); if (videos.length === 0) { videos = [{ video_id: `mock_cat_${videoCategoryId}`, title: `Mock Category ${videoCategoryId} Video`, channel_id: 'UC_MOCK_CAT', channel_title: 'Mock Channel', published_at: '2024-03-10T10:00:00Z', view_count: 1000000, like_count: 50000, comment_count: 2000, category_id: String(videoCategoryId), duration: 'PT10M00S', duration_seconds: 600, thumbnail_url: `https://img.youtube.com/vi/mock_cat_${videoCategoryId}/hqdefault.jpg`, tags: ['mock'], source_url: `https://www.youtube.com/watch?v=mock_cat_${videoCategoryId}`, crawl_time: new Date().toISOString(), _mockRegion: regionCode }]; } } return { videos, total_results: videos.length, quota_used: 0, source: 'mock', cached: false, fallback_level: 1 }; } async getMultiRegionTrending(regions, maxResultsPerRegion = 10, videoCategoryId = null) { const results = {}; for (const region of regions) { try { results[region] = await this.getTrending(region, maxResultsPerRegion, videoCategoryId); } catch (err) { results[region] = { error: err.message, videos: [] }; } } return results; } async getBatchCategories(regionCode, categories, maxResults = 10) { const results = {}; for (const cat of categories) { try { results[cat] = await this.getTrending(regionCode, maxResults, cat); } catch (err) { results[cat] = { error: err.message, videos: [] }; } } return results; } getRemainingQuota() { return this.quotaLimit - this.quotaUsed; } getQuotaStatus() { return { quota_used: this.quotaUsed, quota_limit: this.quotaLimit, quota_remaining: this.getRemainingQuota(), usage_percentage: (this.quotaUsed / this.quotaLimit) * 100 }; } getLogs() { return [...this.logs]; } saveToJSON(data, filepath) { const dir = path.dirname(filepath); if (dir && dir !== '.') fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(filepath, JSON.stringify(data, null, 2), 'utf-8'); return filepath; } saveToCSV(videos, filepath, options = {}) { const delimiter = options.delimiter || ','; const includeHeaders = options.includeHeaders !== false; const dir = path.dirname(filepath); if (dir && dir !== '.') fs.mkdirSync(dir, { recursive: true }); const headers = [ 'video_id', 'title', 'channel_id', 'channel_title', 'published_at', 'view_count', 'like_count', 'comment_count', 'category_id', 'duration', 'duration_seconds', 'thumbnail_url', 'tags', 'source_url', 'crawl_time' ]; const escape = (val) => { if (val === null || val === undefined) return ''; const str = String(val); if (str.includes(delimiter) || str.includes('"') || str.includes('\n')) { return `"${str.replace(/"/g, '""')}"`; } return str; }; let csv = ''; if (includeHeaders) { csv += headers.join(delimiter) + '\n'; } for (const v of videos) { const row = headers.map(h => escape(v[h])).join(delimiter); csv += row + '\n'; } fs.writeFileSync(filepath, csv, 'utf-8'); return filepath; } async healthCheck() { const statuses = { api: 'unknown', ytdlp: 'unknown', invidious: 'unknown' }; if (this.apiKey) { try { await this._fetchWithRetry( `https://www.googleapis.com/youtube/v3/videos?part=snippet&chart=mostPopular®ionCode=US&maxResults=1&key=${this.apiKey}`, { method: 'GET' }, { maxRetries: 1 } ); statuses.api = 'ok'; } catch (e) { statuses.api = e.status >= 400 ? 'error' : 'ok'; } } else { statuses.api = 'error'; } try { execSync('yt-dlp --version', { encoding: 'utf-8', timeout: 10000 }); statuses.ytdlp = 'ok'; } catch (e) { statuses.ytdlp = 'error'; } try { await this._fetchWithRetry('https://vid.puffyan.us/api/v1/trending?region=US', { method: 'GET' }, { maxRetries: 1 }); statuses.invidious = 'ok'; } catch (e) { statuses.invidious = 'error'; } const recommended = statuses.api === 'ok' ? 'api' : (statuses.ytdlp === 'ok' ? 'ytdlp' : (statuses.invidious === 'ok' ? 'invidious' : 'none')); return { ...statuses, recommended_method: recommended }; } } module.exports = { YouTubeTrendingCrawler, parseDuration };