/** * YouTube 数据抓取专家 * 通过 YouTube Data API v3 实现视频、频道、搜索数据的获取 * @version 1.0.0 */ const https = require('https'); const { URL } = require('url'); // ============================================ // 配置 // ============================================ const CONFIG = { apiBaseUrl: 'https://www.googleapis.com/youtube/v3', quotaLimit: 10000, quotaWarningThreshold: 8000, batchSize: 50, maxRetries: 3, baseDelay: 1000, maxConcurrent: 5 }; // 配额追踪 let quotaUsage = { search: 0, videos: 0, channels: 0, playlistItems: 0, total: 0 }; // ============================================ // 核心 API 请求函数 // ============================================ /** * 发送 YouTube API 请求 * @param {string} endpoint - API 端点 * @param {Object} params - 请求参数 * @returns {Promise} API 响应 */ async function apiRequest(endpoint, params = {}) { const apiKey = process.env.YOUTUBE_API_KEY; if (!apiKey) { throw new Error('YOUTUBE_API_KEY environment variable is required'); } const url = new URL(`${CONFIG.apiBaseUrl}/${endpoint}`); url.searchParams.append('key', apiKey); Object.entries(params).forEach(([key, value]) => { if (value !== undefined && value !== null) { url.searchParams.append(key, value); } }); return new Promise((resolve, reject) => { const req = https.get(url.toString(), { headers: { 'Accept': 'application/json', 'User-Agent': 'YouTube-Data-Expert/1.0' } }, (res) => { let data = ''; res.on('data', chunk => data += chunk); res.on('end', () => { try { const json = JSON.parse(data); if (json.error) { const error = new Error(`API Error: ${json.error.message}`); error.code = json.error.code; error.details = json.error.errors; reject(error); } else { resolve(json); } } catch (e) { reject(new Error(`Parse Error: ${e.message}`)); } }); }); req.on('error', reject); req.setTimeout(30000, () => { req.destroy(); reject(new Error('Request timeout')); }); }); } /** * 带重试的 API 请求 * @param {string} endpoint - API 端点 * @param {Object} params - 请求参数 * @param {string} quotaType - 配额类型 * @returns {Promise} API 响应 */ async function apiRequestWithRetry(endpoint, params, quotaType) { let lastError; for (let attempt = 0; attempt < CONFIG.maxRetries; attempt++) { try { // 检查配额 if (quotaUsage.total >= CONFIG.quotaLimit) { throw new Error(`API quota exceeded (${quotaUsage.total}/${CONFIG.quotaLimit} units)`); } const result = await apiRequest(endpoint, params); trackQuota(quotaType); return result; } catch (error) { lastError = error; // 根据错误码处理 if (error.code === 403) { // 配额耗尽或权限不足,不再重试 throw error; } else if (error.code === 429) { // 速率限制,指数退避 const delay = CONFIG.baseDelay * Math.pow(2, attempt); await sleep(delay); } else if (error.code >= 500) { // 服务器错误,延迟重试 await sleep(3000); } else if (error.code === 400) { // 参数错误,检查后重试一次 if (attempt === 0) { await sleep(1000); } else { throw error; } } else { // 其他错误,直接抛出 throw error; } } } throw lastError; } /** * 记录配额使用 * @param {string} type - 配额类型 */ function trackQuota(type) { const costs = { 'search': 100, 'videos': 1, 'channels': 1, 'playlistItems': 1, 'playlists': 1, 'commentThreads': 1 }; const cost = costs[type] || 1; quotaUsage[type] = (quotaUsage[type] || 0) + cost; quotaUsage.total += cost; // 配额警告 if (quotaUsage.total >= CONFIG.quotaWarningThreshold) { console.warn(`⚠️ Quota warning: ${quotaUsage.total}/${CONFIG.quotaLimit} units used`); } } /** * 延迟函数 * @param {number} ms - 毫秒 */ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } // ============================================ // 视频相关功能 // ============================================ /** * 批量获取视频详情 * @param {string[]} videoIds - 视频ID数组 * @param {string[]} parts - 要获取的字段 * @returns {Promise} 视频详情数组 */ async function getVideos(videoIds, parts = ['snippet', 'statistics', 'contentDetails']) { if (!videoIds || videoIds.length === 0) { return []; } const results = []; // 分批处理,每批最多50个 for (let i = 0; i < videoIds.length; i += CONFIG.batchSize) { const batch = videoIds.slice(i, i + CONFIG.batchSize); try { const response = await apiRequestWithRetry('videos', { part: parts.join(','), id: batch.join(',') }, 'videos'); if (response.items) { results.push(...response.items.map(parseVideoData)); } } catch (error) { console.error(`Failed to fetch videos batch ${i / CONFIG.batchSize + 1}: ${error.message}`); // 继续处理其他批次 } } return results; } /** * 获取单个视频详情 * @param {string} videoId - 视频ID * @param {string[]} parts - 要获取的字段 * @returns {Promise} 视频详情 */ async function getVideo(videoId, parts = ['snippet', 'statistics', 'contentDetails']) { const videos = await getVideos([videoId], parts); return videos[0] || null; } /** * 解析视频数据 * @param {Object} item - API 返回的视频数据 * @returns {Object} 清洗后的视频数据 */ function parseVideoData(item) { const snippet = item.snippet || {}; const statistics = item.statistics || {}; const contentDetails = item.contentDetails || {}; return { id: item.id, title: snippet.title || '', description: snippet.description || '', publishedAt: snippet.publishedAt, channelId: snippet.channelId, channelTitle: snippet.channelTitle, thumbnails: snippet.thumbnails, tags: snippet.tags || [], categoryId: snippet.categoryId, duration: parseDuration(contentDetails.duration), dimension: contentDetails.dimension, definition: contentDetails.definition, caption: contentDetails.caption === 'true', viewCount: parseInt(statistics.viewCount, 10) || 0, likeCount: parseInt(statistics.likeCount, 10) || 0, commentCount: parseInt(statistics.commentCount, 10) || 0 }; } /** * 解析 ISO 8601 时长为秒数 * @param {string} duration - ISO 8601 格式时长 (PT15M33S) * @returns {number} 秒数 */ function parseDuration(duration) { if (!duration) return 0; const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/); if (!match) return 0; const hours = parseInt(match[1] || 0, 10); const minutes = parseInt(match[2] || 0, 10); const seconds = parseInt(match[3] || 0, 10); return hours * 3600 + minutes * 60 + seconds; } // ============================================ // 频道相关功能 // ============================================ /** * 批量获取频道详情 * @param {string[]} channelIds - 频道ID数组 * @param {string[]} parts - 要获取的字段 * @returns {Promise} 频道详情数组 */ async function getChannels(channelIds, parts = ['snippet', 'statistics', 'contentDetails']) { if (!channelIds || channelIds.length === 0) { return []; } const results = []; for (let i = 0; i < channelIds.length; i += CONFIG.batchSize) { const batch = channelIds.slice(i, i + CONFIG.batchSize); try { const response = await apiRequestWithRetry('channels', { part: parts.join(','), id: batch.join(',') }, 'channels'); if (response.items) { results.push(...response.items.map(parseChannelData)); } } catch (error) { console.error(`Failed to fetch channels batch ${i / CONFIG.batchSize + 1}: ${error.message}`); } } return results; } /** * 获取单个频道详情 * @param {string} channelId - 频道ID * @param {string[]} parts - 要获取的字段 * @returns {Promise} 频道详情 */ async function getChannel(channelId, parts = ['snippet', 'statistics', 'contentDetails']) { const channels = await getChannels([channelId], parts); return channels[0] || null; } /** * 解析频道数据 * @param {Object} item - API 返回的频道数据 * @returns {Object} 清洗后的频道数据 */ function parseChannelData(item) { const snippet = item.snippet || {}; const statistics = item.statistics || {}; const contentDetails = item.contentDetails || {}; const brandingSettings = item.brandingSettings || {}; const subscriberCount = statistics.hiddenSubscriberCount ? null : parseInt(statistics.subscriberCount, 10) || 0; const videoCount = parseInt(statistics.videoCount, 10) || 0; const viewCount = parseInt(statistics.viewCount, 10) || 0; return { id: item.id, title: snippet.title || '', description: snippet.description || '', publishedAt: snippet.publishedAt, thumbnails: snippet.thumbnails, country: snippet.country, language: snippet.defaultLanguage, subscriberCount, subscriberCountHidden: statistics.hiddenSubscriberCount || false, estimatedSubscriberCount: statistics.hiddenSubscriberCount ? Math.round(viewCount / Math.max(videoCount, 1) * 10) : null, viewCount, videoCount, viewsPerVideo: videoCount > 0 ? Math.round(viewCount / videoCount) : 0, uploadsPlaylistId: contentDetails.relatedPlaylists?.uploads, keywords: brandingSettings.channel?.keywords?.split(' ') || [], url: `https://youtube.com/channel/${item.id}` }; } // ============================================ // 搜索功能 // ============================================ /** * 搜索视频 * @param {Object} options - 搜索选项 * @returns {Promise} 搜索结果 */ async function searchVideos(options = {}) { const { q, channelId, maxResults = 25, order = 'relevance', publishedAfter, publishedBefore, pageToken } = options; const params = { part: 'snippet', type: 'video', maxResults: Math.min(maxResults, CONFIG.batchSize), order }; if (q) params.q = q; if (channelId) params.channelId = channelId; if (publishedAfter) params.publishedAfter = publishedAfter; if (publishedBefore) params.publishedBefore = publishedBefore; if (pageToken) params.pageToken = pageToken; const response = await apiRequestWithRetry('search', params, 'search'); return { items: (response.items || []).map(item => ({ videoId: item.id.videoId, title: item.snippet.title, description: item.snippet.description, publishedAt: item.snippet.publishedAt, channelId: item.snippet.channelId, channelTitle: item.snippet.channelTitle, thumbnails: item.snippet.thumbnails, url: `https://youtube.com/watch?v=${item.id.videoId}` })), nextPageToken: response.nextPageToken, totalResults: response.pageInfo?.totalResults || 0 }; } /** * 搜索频道 * @param {Object} options - 搜索选项 * @returns {Promise} 搜索结果 */ async function searchChannels(options = {}) { const { q, maxResults = 25 } = options; const response = await apiRequestWithRetry('search', { part: 'snippet', type: 'channel', q, maxResults: Math.min(maxResults, CONFIG.batchSize) }, 'search'); return { items: (response.items || []).map(item => ({ channelId: item.id.channelId, title: item.snippet.title, description: item.snippet.description, publishedAt: item.snippet.publishedAt, thumbnails: item.snippet.thumbnails, url: `https://youtube.com/channel/${item.id.channelId}` })), nextPageToken: response.nextPageToken, totalResults: response.pageInfo?.totalResults || 0 }; } // ============================================ // 播放列表功能 // ============================================ /** * 获取播放列表中的视频 * @param {string} playlistId - 播放列表ID * @param {number} maxResults - 最大结果数 * @param {string} pageToken - 分页令牌 * @returns {Promise} 视频列表 */ async function getPlaylistItems(playlistId, maxResults = 50, pageToken) { const params = { part: 'snippet,contentDetails', playlistId, maxResults: Math.min(maxResults, CONFIG.batchSize) }; if (pageToken) params.pageToken = pageToken; const response = await apiRequestWithRetry('playlistItems', params, 'playlistItems'); return { items: (response.items || []).map(item => ({ playlistItemId: item.id, videoId: item.contentDetails?.videoId, title: item.snippet.title, description: item.snippet.description, publishedAt: item.snippet.publishedAt, position: item.snippet.position, thumbnails: item.snippet.thumbnails })), nextPageToken: response.nextPageToken, totalResults: response.pageInfo?.totalResults || 0 }; } /** * 获取频道上传列表 * @param {string} uploadsPlaylistId - 上传播放列表ID * @param {number} maxResults - 最大结果数 * @returns {Promise} 视频列表 */ async function getChannelUploads(uploadsPlaylistId, maxResults = 50) { if (!uploadsPlaylistId) { throw new Error('uploadsPlaylistId is required'); } const results = []; let pageToken; let fetched = 0; while (fetched < maxResults) { const response = await getPlaylistItems( uploadsPlaylistId, Math.min(CONFIG.batchSize, maxResults - fetched), pageToken ); results.push(...response.items); fetched += response.items.length; pageToken = response.nextPageToken; if (!pageToken || response.items.length === 0) break; } return { items: results, totalFetched: results.length }; } // ============================================ // 数据导出功能 // ============================================ /** * 导出为 JSON * @param {Array} data - 数据数组 * @param {Object} metadata - 元数据 * @returns {Object} JSON 对象 */ function exportToJSON(data, metadata = {}) { return { metadata: { exported_at: new Date().toISOString(), total_items: data.length, quota_used: quotaUsage.total, ...metadata }, data }; } /** * 导出为 CSV * @param {Array} data - 数据数组 * @returns {string} CSV 字符串 */ function exportToCSV(data) { if (!data || data.length === 0) return ''; const headers = Object.keys(data[0]); const rows = data.map(item => headers.map(header => { const value = item[header]; if (value === null || value === undefined) return ''; if (typeof value === 'object') return JSON.stringify(value); const str = String(value); // 处理包含逗号或引号的字段 if (str.includes(',') || str.includes('"') || str.includes('\n')) { return `"${str.replace(/"/g, '""')}"`; } return str; }).join(',') ); return [headers.join(','), ...rows].join('\n'); } // ============================================ // 配额和工具函数 // ============================================ /** * 获取配额使用报告 * @returns {Object} 配额使用详情 */ function getQuotaReport() { return { used: quotaUsage, remaining: CONFIG.quotaLimit - quotaUsage.total, limit: CONFIG.quotaLimit, percentage: ((quotaUsage.total / CONFIG.quotaLimit) * 100).toFixed(2) + '%' }; } /** * 重置配额计数 */ function resetQuota() { quotaUsage = { search: 0, videos: 0, channels: 0, playlistItems: 0, total: 0 }; } /** * 验证视频ID格式 * @param {string} videoId - 视频ID * @returns {boolean} 是否有效 */ function isValidVideoId(videoId) { if (!videoId || typeof videoId !== 'string') return false; return /^[a-zA-Z0-9_-]{11}$/.test(videoId); } /** * 验证频道ID格式 * @param {string} channelId - 频道ID * @returns {boolean} 是否有效 */ function isValidChannelId(channelId) { if (!channelId || typeof channelId !== 'string') return false; return channelId.startsWith('UC') && channelId.length >= 20; } // ============================================ // 主入口函数 // ============================================ /** * 完整分析流程 * @param {Object} options - 分析选项 * @returns {Promise} 分析结果 */ async function analyze(options = {}) { const { target, type = 'video' } = options; console.log('🚀 Starting YouTube Data Analysis...\n'); // 重置配额 resetQuota(); try { if (type === 'video') { console.log(`📹 Fetching video: ${target}`); const video = await getVideo(target); console.log('✅ Video data retrieved\n'); return exportToJSON([video], { type: 'video' }); } else if (type === 'channel') { console.log(`📺 Fetching channel: ${target}`); const channel = await getChannel(target); console.log('✅ Channel data retrieved\n'); return exportToJSON([channel], { type: 'channel' }); } else if (type === 'search') { console.log(`🔍 Searching: ${target}`); const results = await searchVideos({ q: target, maxResults: 25 }); console.log(`✅ Found ${results.items.length} videos\n`); return exportToJSON(results.items, { type: 'search', query: target, totalResults: results.totalResults }); } } catch (error) { console.error('❌ Analysis failed:', error.message); throw error; } } // ============================================ // CLI 入口 // ============================================ async function main() { const args = process.argv.slice(2); const command = args[0]; const target = args[1]; if (!command || !target) { console.log('Usage: node youtube_data_expert.js '); console.log(''); console.log('Commands:'); console.log(' video - Get video details'); console.log(' channel - Get channel details'); console.log(' search - Search videos'); console.log(''); console.log('Environment:'); console.log(' YOUTUBE_API_KEY - Required'); process.exit(1); } try { let result; switch (command) { case 'video': result = await getVideo(target); break; case 'channel': result = await getChannel(target); break; case 'search': result = await searchVideos({ q: target, maxResults: 10 }); break; default: console.error(`Unknown command: ${command}`); process.exit(1); } console.log(JSON.stringify(result, null, 2)); console.log('\n📊 Quota usage:', getQuotaReport().percentage); } catch (error) { console.error('Error:', error.message); process.exit(1); } } if (require.main === module) { main(); } // ============================================ // 导出模块 // ============================================ module.exports = { // 核心 API 函数 getVideo, getVideos, getChannel, getChannels, searchVideos, searchChannels, getPlaylistItems, getChannelUploads, // 数据导出 exportToJSON, exportToCSV, // 配额管理 getQuotaReport, resetQuota, // 验证工具 isValidVideoId, isValidChannelId, // 主入口 analyze, // 配置 CONFIG };