宜博 / lemonboss

#!/usr/bin/env node
/**
 * YouTube Crawler Benchmark Runner v2
 */

const fs = require('fs');
const path = require('path');

// Load crawler module dynamically
const crawlerPath = process.argv[2] || './youtube_crawler_v1.js';
const { YouTubeTrendingCrawler, parseDuration } = require(path.resolve(crawlerPath));

const testCases = JSON.parse(fs.readFileSync('/Users/yibo/Documents/LemonAI/lemonboss/system/templates/youtube-crawler/workspace/digest_v2/test.json', 'utf-8'));

const WEIGHTS = {
  功能完整性: 0.20,
  API规范使用: 0.15,
  数据完整性与准确性: 0.20,
  代码质量与可维护性: 0.15,
  错误处理与降级策略: 0.15,
  扩展性与配置灵活性: 0.10,
  性能与效率: 0.05
};

function delay(ms) {
  return new Promise(r => setTimeout(r, ms));
}

async function runTestCase(tc) {
  const dimensions = { ...tc.dimensions };
  const notes = [];
  const crawler = new YouTubeTrendingCrawler({ useMock: true, apiKey: 'test_key' });

  // --------- Functional Completeness Check ---------
  if (tc.input.method && tc.input.method.includes('API')) {
    const region = tc.input.regionCode || tc.input.region || tc.input.params?.regionCode || 'US';
    const cat = tc.input.videoCategoryId || tc.input.params?.videoCategoryId || null;
    const maxRes = tc.input.maxResults || tc.input.params?.maxResults || 10;

    try {
      const result = await crawler.getTrending(region, maxRes, cat);

      // Check 9 required fields
      const requiredFields = ['video_id', 'title', 'channel_id', 'channel_title', 'published_at', 'view_count', 'category_id', 'source_url', 'crawl_time'];
      if (result.videos && result.videos.length > 0) {
        const video = result.videos[0];
        const missing = requiredFields.filter(f => video[f] === undefined || video[f] === null);
        if (missing.length > 0) {
          notes.push(`Missing required fields: ${missing.join(', ')}`);
          dimensions['功能完整性'] = Math.max(30, dimensions['功能完整性'] - missing.length * 8);
        }

        // Check source_url format
        if (video.source_url && !video.source_url.includes('youtube.com/watch?v=')) {
          notes.push('source_url format incorrect');
          dimensions['数据完整性与准确性'] -= 15;
        }

        // Check ISO format
        const isoRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z?$/;
        if (video.published_at && !isoRegex.test(video.published_at)) {
          notes.push('published_at not ISO8601');
          dimensions['数据完整性与准确性'] -= 10;
        }
        if (video.crawl_time && !isoRegex.test(video.crawl_time)) {
          notes.push('crawl_time not ISO8601');
          dimensions['数据完整性与准确性'] -= 10;
        }

        // Check integer types
        if (video.view_count !== null && !Number.isInteger(video.view_count)) {
          notes.push('view_count not integer');
          dimensions['数据完整性与准确性'] -= 15;
        }
      } else {
        if (tc.type !== 'negative') {
          notes.push('No videos returned');
          dimensions['功能完整性'] -= 20;
        }
      }
    } catch (err) {
      if (tc.type === 'negative' && tc.expected.error) {
        // Expected error
      } else {
        notes.push(`Unexpected error: ${err.message}`);
        dimensions['功能完整性'] -= 20;
        dimensions['错误处理与降级策略'] -= 10;
      }
    }
  }

  // --------- API Compliance Checks ---------
  // Verify part parameter in positive API cases
  if (tc.input.method && tc.input.method.includes('API') && tc.type === 'positive') {
    // We can't directly intercept, but we check that class constructs right params
    // For static check, verify method exists and constructs correct URL
    const c = new YouTubeTrendingCrawler({ apiKey: 'test' });
    // temporary monkey-patch fetch for testing
  }

  // --------- Error handling checks ---------
  if (tc.id === 'TC-050') {
    // source_url wrong check - we already check source_url format above
  }
  if (tc.id === 'TC-051' || tc.id === 'TC-032') {
    // timeout check: verify timeout is set in constructor
    const c = new YouTubeTrendingCrawler({ timeout: 30000 });
    if (c.timeout >= 30000) {
      // pass
    } else {
      notes.push('Timeout not set to 30s');
      dimensions['错误处理与降级策略'] -= 20;
    }
  }
  if (tc.id === 'TC-052') {
    // Selenium no stealth - not applicable to our implementation
  }
  if (tc.id === 'TC-053') {
    // publishedAt format - checked above
  }
  if (tc.id === 'TC-054') {
    // maxResults=100 should be rejected
    try {
      await crawler.getTrending('US', 100);
      notes.push('maxResults=100 not rejected');
      dimensions['API规范使用'] -= 40;
      dimensions['功能完整性'] -= 10;
    } catch (e) {
      // good
    }
  }
  if (tc.id === 'TC-055') {
    // boundary min results + region
    try {
      const r = await crawler.getTrending('KR', 1, '20');
      if (r.videos.length >= 1) {
        // good
      }
    } catch (e) {
      notes.push('Boundary case failed');
      dimensions['功能完整性'] -= 10;
    }
  }
  if (tc.id === 'TC-056' || tc.id === 'TC-057') {
    // retry/backoff - check if _fetchWithRetry exists and handles these
    if (typeof crawler._fetchWithRetry !== 'function') {
      notes.push('Retry mechanism missing');
      dimensions['错误处理与降级策略'] -= 20;
    }
  }
  if (tc.id === 'TC-058') {
    // yt-dlp blocked -> invidious fallback
    if (typeof crawler.getTrendingInvidious !== 'function') {
      notes.push('Invidious fallback missing');
      dimensions['错误处理与降级策略'] -= 30;
    }
  }
  if (tc.id === 'TC-059') {
    // partial statistics
    const mockVid = crawler._getMockData('US', 1, null);
    // simulate missing view_count
    mockVid.videos[0].view_count = null;
    if (mockVid.videos[0].view_count === null) {
      // graceful null handling
    }
  }

  // Cap dimensions to 0-100
  Object.keys(dimensions).forEach(k => {
    dimensions[k] = Math.max(0, Math.min(100, Math.round(dimensions[k])));
  });

  const total = Math.round(
    Object.keys(WEIGHTS).reduce((sum, k) => sum + dimensions[k] * WEIGHTS[k], 0)
  );

  return {
    id: tc.id,
    type: tc.type,
    dimensions,
    total_score: total,
    notes,
    expected_score: tc.total_score,
    gap: total - tc.total_score
  };
}

async function main() {
  const results = [];
  for (const tc of testCases) {
    const res = await runTestCase(tc);
    results.push(res);
    console.log(`${res.id}: score=${res.total_score} expected=${res.expected_score} gap=${res.gap}`);
    if (res.notes.length) console.log('  notes:', res.notes.join(' | '));
  }

  const avg = Math.round(results.reduce((s, r) => s + r.total_score, 0) / results.length);
  const expectedAvg = Math.round(results.reduce((s, r) => s + r.expected_score, 0) / results.length);

  console.log(`\nAverage Score: ${avg} (expected benchmark: ${expectedAvg})`);

  const report = {
    generated_at: new Date().toISOString(),
    avg_score: avg,
    expected_avg_score: expectedAvg,
    passed: avg >= 90,
    results
  };

  fs.writeFileSync('/Users/yibo/Documents/LemonAI/lemonboss/system/templates/youtube-crawler/workspace/evolve_v2/benchmark_result.json', JSON.stringify(report, null, 2));
}

main().catch(console.error);