宜博 / lemonboss

/**
 * YouTube 数据抓取专家
 * 通过 YouTube Data API v3 实现视频、频道、搜索数据的获取
 * @version 1.0.0
 */

const https = require('https');
const { URL } = require('url');

// ============================================
// 配置
// ============================================
const CONFIG = {
  apiBaseUrl: 'https://www.googleapis.com/youtube/v3',
  quotaLimit: 10000,
  quotaWarningThreshold: 8000,
  batchSize: 50,
  maxRetries: 3,
  baseDelay: 1000,
  maxConcurrent: 5
};

// 配额追踪
let quotaUsage = {
  search: 0,
  videos: 0,
  channels: 0,
  playlistItems: 0,
  total: 0
};

// ============================================
// 核心 API 请求函数
// ============================================

/**
 * 发送 YouTube API 请求
 * @param {string} endpoint - API 端点
 * @param {Object} params - 请求参数
 * @returns {Promise<Object>} API 响应
 */
async function apiRequest(endpoint, params = {}) {
  const apiKey = process.env.YOUTUBE_API_KEY;
  if (!apiKey) {
    throw new Error('YOUTUBE_API_KEY environment variable is required');
  }

  const url = new URL(`${CONFIG.apiBaseUrl}/${endpoint}`);
  url.searchParams.append('key', apiKey);

  Object.entries(params).forEach(([key, value]) => {
    if (value !== undefined && value !== null) {
      url.searchParams.append(key, value);
    }
  });

  return new Promise((resolve, reject) => {
    const req = https.get(url.toString(), {
      headers: {
        'Accept': 'application/json',
        'User-Agent': 'YouTube-Data-Expert/1.0'
      }
    }, (res) => {
      let data = '';
      res.on('data', chunk => data += chunk);
      res.on('end', () => {
        try {
          const json = JSON.parse(data);
          if (json.error) {
            const error = new Error(`API Error: ${json.error.message}`);
            error.code = json.error.code;
            error.details = json.error.errors;
            reject(error);
          } else {
            resolve(json);
          }
        } catch (e) {
          reject(new Error(`Parse Error: ${e.message}`));
        }
      });
    });

    req.on('error', reject);
    req.setTimeout(30000, () => {
      req.destroy();
      reject(new Error('Request timeout'));
    });
  });
}

/**
 * 带重试的 API 请求
 * @param {string} endpoint - API 端点
 * @param {Object} params - 请求参数
 * @param {string} quotaType - 配额类型
 * @returns {Promise<Object>} API 响应
 */
async function apiRequestWithRetry(endpoint, params, quotaType) {
  let lastError;

  for (let attempt = 0; attempt < CONFIG.maxRetries; attempt++) {
    try {
      // 检查配额
      if (quotaUsage.total >= CONFIG.quotaLimit) {
        throw new Error(`API quota exceeded (${quotaUsage.total}/${CONFIG.quotaLimit} units)`);
      }

      const result = await apiRequest(endpoint, params);
      trackQuota(quotaType);
      return result;
    } catch (error) {
      lastError = error;

      // 根据错误码处理
      if (error.code === 403) {
        // 配额耗尽或权限不足，不再重试
        throw error;
      } else if (error.code === 429) {
        // 速率限制，指数退避
        const delay = CONFIG.baseDelay * Math.pow(2, attempt);
        await sleep(delay);
      } else if (error.code >= 500) {
        // 服务器错误，延迟重试
        await sleep(3000);
      } else if (error.code === 400) {
        // 参数错误，检查后重试一次
        if (attempt === 0) {
          await sleep(1000);
        } else {
          throw error;
        }
      } else {
        // 其他错误，直接抛出
        throw error;
      }
    }
  }

  throw lastError;
}

/**
 * 记录配额使用
 * @param {string} type - 配额类型
 */
function trackQuota(type) {
  const costs = {
    'search': 100,
    'videos': 1,
    'channels': 1,
    'playlistItems': 1,
    'playlists': 1,
    'commentThreads': 1
  };
  const cost = costs[type] || 1;
  quotaUsage[type] = (quotaUsage[type] || 0) + cost;
  quotaUsage.total += cost;

  // 配额警告
  if (quotaUsage.total >= CONFIG.quotaWarningThreshold) {
    console.warn(`⚠️  Quota warning: ${quotaUsage.total}/${CONFIG.quotaLimit} units used`);
  }
}

/**
 * 延迟函数
 * @param {number} ms - 毫秒
 */
function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

// ============================================
// 视频相关功能
// ============================================

/**
 * 批量获取视频详情
 * @param {string[]} videoIds - 视频ID数组
 * @param {string[]} parts - 要获取的字段
 * @returns {Promise<Array>} 视频详情数组
 */
async function getVideos(videoIds, parts = ['snippet', 'statistics', 'contentDetails']) {
  if (!videoIds || videoIds.length === 0) {
    return [];
  }

  const results = [];

  // 分批处理，每批最多50个
  for (let i = 0; i < videoIds.length; i += CONFIG.batchSize) {
    const batch = videoIds.slice(i, i + CONFIG.batchSize);

    try {
      const response = await apiRequestWithRetry('videos', {
        part: parts.join(','),
        id: batch.join(',')
      }, 'videos');

      if (response.items) {
        results.push(...response.items.map(parseVideoData));
      }
    } catch (error) {
      console.error(`Failed to fetch videos batch ${i / CONFIG.batchSize + 1}: ${error.message}`);
      // 继续处理其他批次
    }
  }

  return results;
}

/**
 * 获取单个视频详情
 * @param {string} videoId - 视频ID
 * @param {string[]} parts - 要获取的字段
 * @returns {Promise<Object|null>} 视频详情
 */
async function getVideo(videoId, parts = ['snippet', 'statistics', 'contentDetails']) {
  const videos = await getVideos([videoId], parts);
  return videos[0] || null;
}

/**
 * 解析视频数据
 * @param {Object} item - API 返回的视频数据
 * @returns {Object} 清洗后的视频数据
 */
function parseVideoData(item) {
  const snippet = item.snippet || {};
  const statistics = item.statistics || {};
  const contentDetails = item.contentDetails || {};

  return {
    id: item.id,
    title: snippet.title || '',
    description: snippet.description || '',
    publishedAt: snippet.publishedAt,
    channelId: snippet.channelId,
    channelTitle: snippet.channelTitle,
    thumbnails: snippet.thumbnails,
    tags: snippet.tags || [],
    categoryId: snippet.categoryId,
    duration: parseDuration(contentDetails.duration),
    dimension: contentDetails.dimension,
    definition: contentDetails.definition,
    caption: contentDetails.caption === 'true',
    viewCount: parseInt(statistics.viewCount, 10) || 0,
    likeCount: parseInt(statistics.likeCount, 10) || 0,
    commentCount: parseInt(statistics.commentCount, 10) || 0
  };
}

/**
 * 解析 ISO 8601 时长为秒数
 * @param {string} duration - ISO 8601 格式时长 (PT15M33S)
 * @returns {number} 秒数
 */
function parseDuration(duration) {
  if (!duration) return 0;

  const match = duration.match(/PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?/);
  if (!match) return 0;

  const hours = parseInt(match[1] || 0, 10);
  const minutes = parseInt(match[2] || 0, 10);
  const seconds = parseInt(match[3] || 0, 10);

  return hours * 3600 + minutes * 60 + seconds;
}

// ============================================
// 频道相关功能
// ============================================

/**
 * 批量获取频道详情
 * @param {string[]} channelIds - 频道ID数组
 * @param {string[]} parts - 要获取的字段
 * @returns {Promise<Array>} 频道详情数组
 */
async function getChannels(channelIds, parts = ['snippet', 'statistics', 'contentDetails']) {
  if (!channelIds || channelIds.length === 0) {
    return [];
  }

  const results = [];

  for (let i = 0; i < channelIds.length; i += CONFIG.batchSize) {
    const batch = channelIds.slice(i, i + CONFIG.batchSize);

    try {
      const response = await apiRequestWithRetry('channels', {
        part: parts.join(','),
        id: batch.join(',')
      }, 'channels');

      if (response.items) {
        results.push(...response.items.map(parseChannelData));
      }
    } catch (error) {
      console.error(`Failed to fetch channels batch ${i / CONFIG.batchSize + 1}: ${error.message}`);
    }
  }

  return results;
}

/**
 * 获取单个频道详情
 * @param {string} channelId - 频道ID
 * @param {string[]} parts - 要获取的字段
 * @returns {Promise<Object|null>} 频道详情
 */
async function getChannel(channelId, parts = ['snippet', 'statistics', 'contentDetails']) {
  const channels = await getChannels([channelId], parts);
  return channels[0] || null;
}

/**
 * 解析频道数据
 * @param {Object} item - API 返回的频道数据
 * @returns {Object} 清洗后的频道数据
 */
function parseChannelData(item) {
  const snippet = item.snippet || {};
  const statistics = item.statistics || {};
  const contentDetails = item.contentDetails || {};
  const brandingSettings = item.brandingSettings || {};

  const subscriberCount = statistics.hiddenSubscriberCount
    ? null
    : parseInt(statistics.subscriberCount, 10) || 0;

  const videoCount = parseInt(statistics.videoCount, 10) || 0;
  const viewCount = parseInt(statistics.viewCount, 10) || 0;

  return {
    id: item.id,
    title: snippet.title || '',
    description: snippet.description || '',
    publishedAt: snippet.publishedAt,
    thumbnails: snippet.thumbnails,
    country: snippet.country,
    language: snippet.defaultLanguage,
    subscriberCount,
    subscriberCountHidden: statistics.hiddenSubscriberCount || false,
    estimatedSubscriberCount: statistics.hiddenSubscriberCount
      ? Math.round(viewCount / Math.max(videoCount, 1) * 10)
      : null,
    viewCount,
    videoCount,
    viewsPerVideo: videoCount > 0 ? Math.round(viewCount / videoCount) : 0,
    uploadsPlaylistId: contentDetails.relatedPlaylists?.uploads,
    keywords: brandingSettings.channel?.keywords?.split(' ') || [],
    url: `https://youtube.com/channel/${item.id}`
  };
}

// ============================================
// 搜索功能
// ============================================

/**
 * 搜索视频
 * @param {Object} options - 搜索选项
 * @returns {Promise<Object>} 搜索结果
 */
async function searchVideos(options = {}) {
  const {
    q,
    channelId,
    maxResults = 25,
    order = 'relevance',
    publishedAfter,
    publishedBefore,
    pageToken
  } = options;

  const params = {
    part: 'snippet',
    type: 'video',
    maxResults: Math.min(maxResults, CONFIG.batchSize),
    order
  };

  if (q) params.q = q;
  if (channelId) params.channelId = channelId;
  if (publishedAfter) params.publishedAfter = publishedAfter;
  if (publishedBefore) params.publishedBefore = publishedBefore;
  if (pageToken) params.pageToken = pageToken;

  const response = await apiRequestWithRetry('search', params, 'search');

  return {
    items: (response.items || []).map(item => ({
      videoId: item.id.videoId,
      title: item.snippet.title,
      description: item.snippet.description,
      publishedAt: item.snippet.publishedAt,
      channelId: item.snippet.channelId,
      channelTitle: item.snippet.channelTitle,
      thumbnails: item.snippet.thumbnails,
      url: `https://youtube.com/watch?v=${item.id.videoId}`
    })),
    nextPageToken: response.nextPageToken,
    totalResults: response.pageInfo?.totalResults || 0
  };
}

/**
 * 搜索频道
 * @param {Object} options - 搜索选项
 * @returns {Promise<Object>} 搜索结果
 */
async function searchChannels(options = {}) {
  const { q, maxResults = 25 } = options;

  const response = await apiRequestWithRetry('search', {
    part: 'snippet',
    type: 'channel',
    q,
    maxResults: Math.min(maxResults, CONFIG.batchSize)
  }, 'search');

  return {
    items: (response.items || []).map(item => ({
      channelId: item.id.channelId,
      title: item.snippet.title,
      description: item.snippet.description,
      publishedAt: item.snippet.publishedAt,
      thumbnails: item.snippet.thumbnails,
      url: `https://youtube.com/channel/${item.id.channelId}`
    })),
    nextPageToken: response.nextPageToken,
    totalResults: response.pageInfo?.totalResults || 0
  };
}

// ============================================
// 播放列表功能
// ============================================

/**
 * 获取播放列表中的视频
 * @param {string} playlistId - 播放列表ID
 * @param {number} maxResults - 最大结果数
 * @param {string} pageToken - 分页令牌
 * @returns {Promise<Object>} 视频列表
 */
async function getPlaylistItems(playlistId, maxResults = 50, pageToken) {
  const params = {
    part: 'snippet,contentDetails',
    playlistId,
    maxResults: Math.min(maxResults, CONFIG.batchSize)
  };

  if (pageToken) params.pageToken = pageToken;

  const response = await apiRequestWithRetry('playlistItems', params, 'playlistItems');

  return {
    items: (response.items || []).map(item => ({
      playlistItemId: item.id,
      videoId: item.contentDetails?.videoId,
      title: item.snippet.title,
      description: item.snippet.description,
      publishedAt: item.snippet.publishedAt,
      position: item.snippet.position,
      thumbnails: item.snippet.thumbnails
    })),
    nextPageToken: response.nextPageToken,
    totalResults: response.pageInfo?.totalResults || 0
  };
}

/**
 * 获取频道上传列表
 * @param {string} uploadsPlaylistId - 上传播放列表ID
 * @param {number} maxResults - 最大结果数
 * @returns {Promise<Object>} 视频列表
 */
async function getChannelUploads(uploadsPlaylistId, maxResults = 50) {
  if (!uploadsPlaylistId) {
    throw new Error('uploadsPlaylistId is required');
  }

  const results = [];
  let pageToken;
  let fetched = 0;

  while (fetched < maxResults) {
    const response = await getPlaylistItems(
      uploadsPlaylistId,
      Math.min(CONFIG.batchSize, maxResults - fetched),
      pageToken
    );

    results.push(...response.items);
    fetched += response.items.length;
    pageToken = response.nextPageToken;

    if (!pageToken || response.items.length === 0) break;
  }

  return {
    items: results,
    totalFetched: results.length
  };
}

// ============================================
// 数据导出功能
// ============================================

/**
 * 导出为 JSON
 * @param {Array} data - 数据数组
 * @param {Object} metadata - 元数据
 * @returns {Object} JSON 对象
 */
function exportToJSON(data, metadata = {}) {
  return {
    metadata: {
      exported_at: new Date().toISOString(),
      total_items: data.length,
      quota_used: quotaUsage.total,
      ...metadata
    },
    data
  };
}

/**
 * 导出为 CSV
 * @param {Array} data - 数据数组
 * @returns {string} CSV 字符串
 */
function exportToCSV(data) {
  if (!data || data.length === 0) return '';

  const headers = Object.keys(data[0]);
  const rows = data.map(item =>
    headers.map(header => {
      const value = item[header];
      if (value === null || value === undefined) return '';
      if (typeof value === 'object') return JSON.stringify(value);
      const str = String(value);
      // 处理包含逗号或引号的字段
      if (str.includes(',') || str.includes('"') || str.includes('\n')) {
        return `"${str.replace(/"/g, '""')}"`;
      }
      return str;
    }).join(',')
  );

  return [headers.join(','), ...rows].join('\n');
}

// ============================================
// 配额和工具函数
// ============================================

/**
 * 获取配额使用报告
 * @returns {Object} 配额使用详情
 */
function getQuotaReport() {
  return {
    used: quotaUsage,
    remaining: CONFIG.quotaLimit - quotaUsage.total,
    limit: CONFIG.quotaLimit,
    percentage: ((quotaUsage.total / CONFIG.quotaLimit) * 100).toFixed(2) + '%'
  };
}

/**
 * 重置配额计数
 */
function resetQuota() {
  quotaUsage = {
    search: 0,
    videos: 0,
    channels: 0,
    playlistItems: 0,
    total: 0
  };
}

/**
 * 验证视频ID格式
 * @param {string} videoId - 视频ID
 * @returns {boolean} 是否有效
 */
function isValidVideoId(videoId) {
  if (!videoId || typeof videoId !== 'string') return false;
  return /^[a-zA-Z0-9_-]{11}$/.test(videoId);
}

/**
 * 验证频道ID格式
 * @param {string} channelId - 频道ID
 * @returns {boolean} 是否有效
 */
function isValidChannelId(channelId) {
  if (!channelId || typeof channelId !== 'string') return false;
  return channelId.startsWith('UC') && channelId.length >= 20;
}

// ============================================
// 主入口函数
// ============================================

/**
 * 完整分析流程
 * @param {Object} options - 分析选项
 * @returns {Promise<Object>} 分析结果
 */
async function analyze(options = {}) {
  const { target, type = 'video' } = options;

  console.log('🚀 Starting YouTube Data Analysis...\n');

  // 重置配额
  resetQuota();

  try {
    if (type === 'video') {
      console.log(`📹 Fetching video: ${target}`);
      const video = await getVideo(target);
      console.log('✅ Video data retrieved\n');
      return exportToJSON([video], { type: 'video' });

    } else if (type === 'channel') {
      console.log(`📺 Fetching channel: ${target}`);
      const channel = await getChannel(target);
      console.log('✅ Channel data retrieved\n');
      return exportToJSON([channel], { type: 'channel' });

    } else if (type === 'search') {
      console.log(`🔍 Searching: ${target}`);
      const results = await searchVideos({ q: target, maxResults: 25 });
      console.log(`✅ Found ${results.items.length} videos\n`);
      return exportToJSON(results.items, {
        type: 'search',
        query: target,
        totalResults: results.totalResults
      });
    }

  } catch (error) {
    console.error('❌ Analysis failed:', error.message);
    throw error;
  }
}

// ============================================
// CLI 入口
// ============================================
async function main() {
  const args = process.argv.slice(2);
  const command = args[0];
  const target = args[1];

  if (!command || !target) {
    console.log('Usage: node youtube_data_expert.js <command> <target>');
    console.log('');
    console.log('Commands:');
    console.log('  video <videoId>     - Get video details');
    console.log('  channel <channelId> - Get channel details');
    console.log('  search <query>      - Search videos');
    console.log('');
    console.log('Environment:');
    console.log('  YOUTUBE_API_KEY     - Required');
    process.exit(1);
  }

  try {
    let result;

    switch (command) {
      case 'video':
        result = await getVideo(target);
        break;
      case 'channel':
        result = await getChannel(target);
        break;
      case 'search':
        result = await searchVideos({ q: target, maxResults: 10 });
        break;
      default:
        console.error(`Unknown command: ${command}`);
        process.exit(1);
    }

    console.log(JSON.stringify(result, null, 2));
    console.log('\n📊 Quota usage:', getQuotaReport().percentage);

  } catch (error) {
    console.error('Error:', error.message);
    process.exit(1);
  }
}

if (require.main === module) {
  main();
}

// ============================================
// 导出模块
// ============================================
module.exports = {
  // 核心 API 函数
  getVideo,
  getVideos,
  getChannel,
  getChannels,
  searchVideos,
  searchChannels,
  getPlaylistItems,
  getChannelUploads,

  // 数据导出
  exportToJSON,
  exportToCSV,

  // 配额管理
  getQuotaReport,
  resetQuota,

  // 验证工具
  isValidVideoId,
  isValidChannelId,

  // 主入口
  analyze,

  // 配置
  CONFIG
};