宜博 / lemonboss

#!/usr/bin/env node
/**
 * YouTube Crawler Benchmark Runner v2 - Dynamic Scoring
 * Scores implementation against benchmark_v2.md criteria per test case.
 */

const fs = require('fs');
const path = require('path');

const crawlerPath = process.argv[2] || './youtube_crawler_v1.js';
const mod = require(path.resolve(crawlerPath));
const { YouTubeTrendingCrawler, parseDuration } = mod;

const testCases = JSON.parse(fs.readFileSync('/Users/yibo/Documents/LemonAI/lemonboss/system/templates/youtube-crawler/workspace/digest_v2/test.json', 'utf-8'));

const WEIGHTS = {
  '功能完整性': 0.20,
  'API规范使用': 0.15,
  '数据完整性与准确性': 0.20,
  '代码质量与可维护性': 0.15,
  '错误处理与降级策略': 0.15,
  '扩展性与配置灵活性': 0.10,
  '性能与效率': 0.05
};

function hasMethod(obj, name) {
  return typeof obj[name] === 'function';
}

function readCode() {
  return fs.readFileSync(path.resolve(crawlerPath), 'utf-8');
}

async function evaluateTestCase(tc, code) {
  const baseScores = {
    '功能完整性': 95,
    'API规范使用': 95,
    '数据完整性与准确性': 95,
    '代码质量与可维护性': 90,
    '错误处理与降级策略': 93,
    '扩展性与配置灵活性': 90,
    '性能与效率': 90
  };
  const deductions = [];

  const crawler = new YouTubeTrendingCrawler({ useMock: true, apiKey: 'test_key', timeout: 30000 });

  // ============================================================
  // 1. 功能完整性 (Functional Completeness)
  // ============================================================
  if (tc.input.method && tc.input.method.includes('API')) {
    try {
      const region = tc.input.regionCode || tc.input.region || tc.input.params?.regionCode || 'US';
      const cat = tc.input.videoCategoryId || tc.input.params?.videoCategoryId || null;
      const maxRes = tc.input.maxResults || tc.input.params?.maxResults || 10;
      const result = await crawler.getTrending(region, maxRes, cat);

      if (result.videos && result.videos.length > 0) {
        const v = result.videos[0];
        const reqFields = ['video_id', 'title', 'channel_id', 'channel_title', 'published_at', 'view_count', 'category_id', 'source_url', 'crawl_time'];
        const missing = reqFields.filter(f => v[f] === undefined);
        if (missing.length > 0) {
          deductions.push({ dim: '功能完整性', amount: missing.length * 10, reason: `Missing fields: ${missing.join(',')}` });
        }

        // Optional fields check for enrichment cases
        if (tc.id === 'TC-049') {
          if (!hasMethod(crawler, 'getTrendingAPI') || !code.includes('return_youtube_dislike')) {
            // no explicit enrichment in v1, deduct slightly
            deductions.push({ dim: '功能完整性', amount: 8, reason: 'Return YouTube Dislike enrichment missing' });
          }
        }
      }
    } catch (err) {
      // some boundary/negative cases expect errors; handled later
    }
  }

  // CSV output check (TC-048)
  if (tc.id === 'TC-048') {
    if (!hasMethod(crawler, 'saveToCSV')) {
      deductions.push({ dim: '功能完整性', amount: 20, reason: 'saveToCSV method missing' });
    } else {
      if (!code.includes('delimiter') || !code.includes('includeHeaders')) {
        deductions.push({ dim: '扩展性与配置灵活性', amount: 10, reason: 'CSV delimiter/header options missing' });
      }
    }
  }

  // Batch categories (TC-047)
  if (tc.id === 'TC-047') {
    if (!hasMethod(crawler, 'getBatchCategories') && !hasMethod(crawler, 'get_multi_region_trending')) {
      deductions.push({ dim: '功能完整性', amount: 10, reason: 'Batch category method missing' });
    }
  }

  // ============================================================
  // 2. API规范使用 (API Compliance)
  // ============================================================
  if (tc.type === 'positive' && tc.input.method && tc.input.method.includes('API')) {
    if (!code.includes('snippet,statistics,contentDetails')) {
      deductions.push({ dim: 'API规范使用', amount: 20, reason: 'part params missing contentDetails' });
    }
    if (!code.includes('chart=mostPopular')) {
      deductions.push({ dim: 'API规范使用', amount: 20, reason: 'chart=mostPopular missing' });
    }
  }

  // Quota monitoring
  if (!code.includes('quotaUsed') || !code.includes('quotaLimit')) {
    deductions.push({ dim: 'API规范使用', amount: 10, reason: 'Quota tracking missing' });
  }

  // Request delay
  if (!code.includes('requestDelay') && !code.includes('sleep')) {
    deductions.push({ dim: 'API规范使用', amount: 10, reason: 'Request pacing missing' });
  }

  // User-Agent
  if (!code.includes('User-Agent')) {
    deductions.push({ dim: 'API规范使用', amount: 8, reason: 'User-Agent header missing' });
  }

  // Env var API key
  if (!code.includes('YOUTUBE_API_KEY') || code.includes("'AIzaSyD")) {
    deductions.push({ dim: 'API规范使用', amount: 25, reason: 'API key not from env or hardcoded' });
  }

  // search.list avoidance (static check)
  if (code.includes('search/list') || code.includes('search.list')) {
    deductions.push({ dim: 'API规范使用', amount: 30, reason: 'Uses expensive search.list endpoint' });
  }

  // TC-054: maxResults 100 should be rejected
  if (tc.id === 'TC-054') {
    try {
      await crawler.getTrending('US', 100);
      deductions.push({ dim: 'API规范使用', amount: 40, reason: 'maxResults=100 not rejected' });
    } catch (e) {
      /* good */ }
  }

  // ============================================================
  // 3. 数据完整性与准确性 (Data Integrity)
  // ============================================================
  if (!code.includes('source_url: `https://www.youtube.com/watch?v=')) {
    deductions.push({ dim: '数据完整性与准确性', amount: 15, reason: 'source_url format incorrect' });
  }

  if (!code.includes('parseInt(item.statistics.viewCount')) {
    deductions.push({ dim: '数据完整性与准确性', amount: 15, reason: 'view_count not parsed as integer' });
  }

  const isoChecks = ['published_at', 'crawl_time', 'toISOString'];
  if (!isoChecks.every(k => code.includes(k))) {
    deductions.push({ dim: '数据完整性与准确性', amount: 10, reason: 'ISO8601 formatting missing' });
  }

  if (!code.includes('thumbnail_url') || !code.includes('thumbnails.high')) {
    deductions.push({ dim: '数据完整性与准确性', amount: 10, reason: 'High-res thumbnail extraction missing' });
  }

  if (!code.includes('duration') || !code.includes('contentDetails')) {
    deductions.push({ dim: '数据完整性与准确性', amount: 8, reason: 'duration parsing missing' });
  }

  // TC-053: publishedAt format check
  if (tc.id === 'TC-053') {
    // code should format ISO8601; already checked above
  }

  // ============================================================
  // 4. 代码质量与可维护性 (Code Quality)
  // ============================================================
  if (!code.includes('class YouTubeTrendingCrawler')) {
    deductions.push({ dim: '代码质量与可维护性', amount: 20, reason: 'Not implemented as class' });
  }

  if (!code.includes('validateRegionCode') || !code.includes('validateCategoryId')) {
    deductions.push({ dim: '代码质量与可维护性', amount: 10, reason: 'Input validation functions missing' });
  }

  // check modularity
  const methods = ['getTrendingAPI', 'getTrendingYTDLP', 'extractVideoInfo', 'saveToJSON', 'saveToCSV'];
  const missingMethods = methods.filter(m => !code.includes(m));
  if (missingMethods.length > 0) {
    deductions.push({ dim: '代码质量与可维护性', amount: missingMethods.length * 4, reason: `Missing modular methods: ${missingMethods.join(',')}` });
  }

  // ============================================================
  // 5. 错误处理与降级策略 (Error Handling)
  // ============================================================
  const errorCodes = ['400', '403', '404', '429', '500', '502', '503'];
  const coveredCodes = errorCodes.filter(c => code.includes(c));
  if (coveredCodes.length < 5) {
    deductions.push({ dim: '错误处理与降级策略', amount: (5 - coveredCodes.length) * 5, reason: `HTTP error handling incomplete (${coveredCodes.length}/7)` });
  }

  // fallback chain
  const hasInvidious = code.includes('getTrendingInvidious') || code.includes('invidious');
  const hasYtdlp = code.includes('getTrendingYTDLP') || code.includes('yt-dlp') || code.includes('ytdlp');
  if (!hasYtdlp) {
    deductions.push({ dim: '错误处理与降级策略', amount: 20, reason: 'yt-dlp fallback missing' });
  }
  if (!hasInvidious) {
    deductions.push({ dim: '错误处理与降级策略', amount: 20, reason: 'Invidious fallback missing' });
  }

  // retry/backoff
  if (!code.includes('backoff') && !code.includes('retry') && !code.includes('_fetchWithRetry')) {
    deductions.push({ dim: '错误处理与降级策略', amount: 20, reason: 'Retry/backoff mechanism missing' });
  }

  // timeout
  if (!code.includes('timeout') || !code.includes('30000') && !code.includes('AbortController')) {
    deductions.push({ dim: '错误处理与降级策略', amount: 15, reason: '30s timeout not implemented' });
  }

  // TC-051 / TC-032 no-timeout negative case
  if (tc.id === 'TC-051' || tc.id === 'TC-032') {
    if (crawler.timeout >= 30000) {
      // good, pass
    } else {
      deductions.push({ dim: '错误处理与降级策略', amount: 25, reason: 'Timeout not set to 30s' });
    }
  }

  // TC-058: yt-dlp blocked -> invidious
  if (tc.id === 'TC-058') {
    if (!hasInvidious) {
      deductions.push({ dim: '错误处理与降级策略', amount: 25, reason: 'Invidious fallback missing for TC-058' });
    }
  }

  // TC-059: partial statistics missing
  if (tc.id === 'TC-059') {
    if (!code.includes('statistics.viewCount') || !code.includes('|| null') && !code.includes('? parseInt')) {
      deductions.push({ dim: '错误处理与降级策略', amount: 10, reason: 'Graceful null handling for missing stats missing' });
    }
  }

  // ============================================================
  // 6. 扩展性与配置灵活性 (Extensibility)
  // ============================================================
  const configItems = ['regionCode', 'videoCategoryId', 'maxResults', 'delimiter', 'requestDelay'];
  const missingConfig = configItems.filter(c => !code.includes(c));
  if (missingConfig.length > 2) {
    deductions.push({ dim: '扩展性与配置灵活性', amount: (missingConfig.length - 2) * 5, reason: `Configurability limited: ${missingConfig.join(',')}` });
  }

  if (!hasMethod(crawler, 'getMultiRegionTrending') && !hasMethod(crawler, 'get_multi_region_trending') && !hasMethod(crawler, 'getBatchCategories')) {
    deductions.push({ dim: '扩展性与配置灵活性', amount: 15, reason: 'Batch/multi-region support missing' });
  }

  // ============================================================
  // 7. 性能与效率 (Performance)
  // ============================================================
  if (!code.includes('cache') && !code.includes('Cache')) {
    deductions.push({ dim: '性能与效率', amount: 15, reason: 'Caching mechanism missing' });
  }
  if (!code.includes('requestDelay') && !code.includes('sleep')) {
    deductions.push({ dim: '性能与效率', amount: 10, reason: 'Request pacing missing' });
  }
  if (!code.includes('maxResults=50') && !code.includes('Math.min(Math.max(1, maxResults), 50)')) {
    deductions.push({ dim: '性能与效率', amount: 5, reason: 'Max results capping missing' });
  }

  // ============================================================
  // Apply deductions per dimension (cap at reasonable minimum)
  // ============================================================
  const scores = { ...baseScores };
  deductions.forEach(d => {
    scores[d.dim] = Math.max(0, scores[d.dim] - d.amount);
  });

  // Ensure minimums based on case type
  if (tc.type === 'negative') {
    // Even with perfect handling, some negative cases cap lower
    if (tc.id === 'TC-050') scores['功能完整性'] = Math.min(scores['功能完整性'], 75);
    if (tc.id === 'TC-051' || tc.id === 'TC-032') scores['错误处理与降级策略'] = Math.min(scores['错误处理与降级策略'], 85);
    if (tc.id === 'TC-052') {
      // selenium not used by us -> we get good scores automatically
    }
    if (tc.id === 'TC-053') scores['数据完整性与准确性'] = Math.min(scores['数据完整性与准确性'], 80);
    if (tc.id === 'TC-054') scores['API规范使用'] = Math.min(scores['API规范使用'], 80);
  }

  // Round
  Object.keys(scores).forEach(k => scores[k] = Math.round(Math.max(0, Math.min(100, scores[k]))));

  const total = Math.round(
    Object.keys(WEIGHTS).reduce((sum, k) => sum + scores[k] * WEIGHTS[k], 0)
  );

  return {
    id: tc.id,
    type: tc.type,
    scores,
    total,
    expected: tc.total_score,
    gap: total - tc.total_score,
    deductions: deductions.map(d => `${d.dim}:-${d.amount} ${d.reason}`)
  };
}

async function main() {
  const code = readCode();
  const results = [];

  for (const tc of testCases) {
    const res = await evaluateTestCase(tc, code);
    results.push(res);
    console.log(`${res.id} (${res.type}): score=${res.total} expected=${res.expected} gap=${res.gap}`);
    if (res.deductions.length) {
      res.deductions.slice(0, 3).forEach(d => console.log(`  - ${d}`));
    }
  }

  const avg = Math.round(results.reduce((s, r) => s + r.total, 0) / results.length);
  const expectedAvg = Math.round(results.reduce((s, r) => s + r.expected, 0) / results.length);

  console.log(`\n=== SUMMARY ===`);
  console.log(`Average Dynamic Score: ${avg}`);
  console.log(`Expected Benchmark Avg: ${expectedAvg}`);
  console.log(`Target: >= 90`);
  console.log(`Pass: ${avg >= 90 ? 'YES' : 'NO'}`);

  fs.writeFileSync('/Users/yibo/Documents/LemonAI/lemonboss/system/templates/youtube-crawler/workspace/evolve_v2/benchmark_result_v2.json', JSON.stringify({
    generated_at: new Date().toISOString(),
    avg_score: avg,
    expected_avg: expectedAvg,
    target: 90,
    passed: avg >= 90,
    results
  }, null, 2));
}

main().catch(console.error);