宜博 / lemonboss

#!/usr/bin/env node

/**
 * rebuild-benchmarks.js — 重建 Benchmark 评测
 *
 * 对所有 Skill 重新运行 Benchmark + ELO Tournament，
 * 按新的 (scene × platform) 分组计算排名。
 *
 * 用法:
 *   node scripts/rebuild-benchmarks.js               # 全量重建
 *   node scripts/rebuild-benchmarks.js --scene video_creation   # 只重建某场景
 *   node scripts/rebuild-benchmarks.js --dry-run      # 预览模式
 *   node scripts/rebuild-benchmarks.js --stats        # 只输出统计，不运行
 *
 * Cron 示例 (每天凌晨 3 点运行):
 *   0 3 * * * cd /path/to/lemonape && node scripts/rebuild-benchmarks.js >> .logs/benchmark-cron.log 2>&1
 */

const path = require('path');
const ROOT = path.resolve(__dirname, '..');

// 加载项目模块
const skillDb = require(path.join(ROOT, 'server/services/skill-db'));
const benchmarkDB = require(path.join(ROOT, 'server/services/benchmark-db'));
const benchmarkRunner = require(path.join(ROOT, 'server/services/benchmark-runner'));
const taxonomy = require(path.join(ROOT, 'server/config/taxonomy'));

// ─── 参数解析 ───
const args = process.argv.slice(2);
const dryRun = args.includes('--dry-run');
const statsOnly = args.includes('--stats');
const sceneIdx = args.indexOf('--scene');
const targetScene = sceneIdx !== -1 ? args[sceneIdx + 1] : null;

async function main() {
  console.log('═══════════════════════════════════════════');
  console.log('  🔬 Benchmark Rebuild');
  console.log('═══════════════════════════════════════════');
  console.log(`  Dry run:  ${dryRun}`);
  console.log(`  Scene:    ${targetScene || 'ALL'}`);
  console.log(`  Stats:    ${statsOnly}`);
  console.log('');

  // 初始化 benchmark DB
  benchmarkDB.initDB();

  // 加载所有 Skill
  let skills = skillDb.listSkills();
  console.log(`[rebuild] 📦 总计 ${skills.length} 个 Skills`);

  // 确保所有 Skill 都有三维标签
  let taggedCount = 0;
  for (const skill of skills) {
    taxonomy.migrateOldScene(skill);
    if (!skill.roles || skill.roles.length === 0 ||
        !skill.scenes || skill.scenes.length === 0 ||
        !skill.platforms || skill.platforms.length === 0) {
      const tags = taxonomy.autoTag(skill);
      if (!skill.roles || skill.roles.length === 0) skill.roles = tags.roles;
      if (!skill.scenes || skill.scenes.length === 0) skill.scenes = tags.scenes;
      if (!skill.platforms || skill.platforms.length === 0) skill.platforms = tags.platforms;
      if (!skill.scene && skill.scenes.length > 0) skill.scene = skill.scenes[0];
      taggedCount++;
    }
  }
  if (taggedCount > 0) {
    console.log(`[rebuild] 🏷️  自动打标 ${taggedCount} 个未标注 Skills`);
  }

  // 按场景过滤
  if (targetScene) {
    skills = skills.filter(s => (s.scenes || [s.scene]).includes(targetScene));
    console.log(`[rebuild] 🎯 筛选场景 "${targetScene}": ${skills.length} 个 Skills`);
  }

  if (skills.length === 0) {
    console.log('[rebuild] ⚠️  没有 Skills，退出。');
    return;
  }

  // 统计维度分布
  printTaxonomyStats(skills);

  if (statsOnly) {
    console.log('[rebuild] ℹ️  Stats-only 模式，跳过 benchmark。');
    return;
  }

  if (dryRun) {
    console.log('[rebuild] ℹ️  Dry-run 模式，以下为将执行的操作预览:');
    console.log(`[rebuild]    将对 ${skills.length} 个 Skills 运行 Benchmark`);
    printScenePlatformGroups(skills);
    return;
  }

  // ── 执行 Benchmark ──
  console.log('\n[rebuild] 🚀 开始 Benchmark...');

  const results = [];
  let benchCount = 0;

  for (const skill of skills) {
    try {
      const { scores, weightedScore } = benchmarkRunner.benchmarkSkill(skill);
      results.push({
        skillName: skill.skillName,
        layer: skill.layer,
        scenes: skill.scenes,
        platforms: skill.platforms,
        weightedScore,
        elo: skill.arena?.elo || 1200,
      });
      benchCount++;

      if (benchCount % 20 === 0) {
        console.log(`[rebuild]    进度: ${benchCount}/${skills.length}`);
      }
    } catch (e) {
      console.error(`[rebuild] ❌ ${skill.skillName}: ${e.message}`);
    }
  }

  console.log(`[rebuild] ✅ Benchmark 完成: ${benchCount}/${skills.length}`);

  // ── ELO Tournament (按 scene × platform 分组) ──
  console.log('\n[rebuild] ⚔️  运行 ELO Tournament...');
  const totalComparisons = runScenePlatformTournament(skills);
  console.log(`[rebuild] ✅ ELO 对战完成: ${totalComparisons} 次比较`);

  // ── 计算排名 ──
  computeRanks(skills);

  // ── 保存 ──
  console.log('[rebuild] 💾 保存结果...');
  for (const skill of skills) {
    const updates = { ...skill };
    delete updates.skillId;
    delete updates.skillName;
    skillDb.updateSkill(skill.skillName, updates);
  }

  // ── 打印排行榜 ──
  printTopSkills(results);

  console.log('\n═══════════════════════════════════════════');
  console.log(`  ✅ Benchmark Rebuild 完成`);
  console.log(`     Skills: ${benchCount}`);
  console.log(`     ELO Comparisons: ${totalComparisons}`);
  console.log('═══════════════════════════════════════════');
}

/**
 * 按 scene × platform 分组做 ELO Tournament
 */
function runScenePlatformTournament(skills) {
  const groups = {};

  for (const s of skills) {
    const sceneList = s.scenes || [s.scene || 'unknown'];
    const platList = s.platforms || ['general'];
    for (const scene of sceneList) {
      for (const plat of platList) {
        const key = `${scene}_${plat}`;
        if (!groups[key]) groups[key] = [];
        groups[key].push(s);
      }
    }
  }

  let totalComparisons = 0;

  for (const [groupKey, group] of Object.entries(groups)) {
    if (group.length < 2) continue;

    for (let i = 0; i < group.length; i++) {
      for (let j = i + 1; j < group.length; j++) {
        const a = group[i];
        const b = group[j];

        const scoreA = a.arena?.scores ? benchmarkRunner.computeWeightedScore(a.arena.scores, a.layer) : 50;
        const scoreB = b.arena?.scores ? benchmarkRunner.computeWeightedScore(b.arena.scores, b.layer) : 50;

        const eloA = a.arena?.elo || 1200;
        const eloB = b.arena?.elo || 1200;

        const kA = benchmarkRunner.getKFactor(a.arena?.benchmarkRuns || 0);
        const kB = benchmarkRunner.getKFactor(b.arena?.benchmarkRuns || 0);

        // Inline ELO calculation
        const expA = 1 / (1 + Math.pow(10, (eloB - eloA) / 400));
        const actualA = scoreA > scoreB ? 1 : scoreA === scoreB ? 0.5 : 0;
        const newA = Math.round(eloA + kA * (actualA - expA));
        const newB = Math.round(eloB + kB * ((1 - actualA) - (1 - expA)));

        if (a.arena) a.arena.elo = newA;
        if (b.arena) b.arena.elo = newB;

        const winner = scoreA > scoreB ? a.skillName
                     : scoreB > scoreA ? b.skillName
                     : 'tie';

        benchmarkDB.insertComparison({
          skillA: a.skillName,
          skillB: b.skillName,
          scene: groupKey.split('_')[0],
          layer: a.layer,
          winner,
          scoreA,
          scoreB,
        });

        totalComparisons++;
      }
    }
  }

  return totalComparisons;
}

/**
 * 计算排名
 */
function computeRanks(skills) {
  const groups = {};
  for (const s of skills) {
    const sceneList = s.scenes || [s.scene || 'unknown'];
    const platList = s.platforms || ['general'];
    for (const scene of sceneList) {
      for (const plat of platList) {
        const key = `${scene}_${plat}`;
        if (!groups[key]) groups[key] = [];
        groups[key].push(s);
      }
    }
  }

  for (const [, group] of Object.entries(groups)) {
    group.sort((a, b) => (b.arena?.elo || 1200) - (a.arena?.elo || 1200));
    group.forEach((s, i) => {
      if (s.arena) {
        s.arena.rank = i + 1;
        s.arena.highlighted = (i < 3);
      }
    });
  }

  const allSorted = [...skills].sort((a, b) => (b.arena?.elo || 1200) - (a.arena?.elo || 1200));
  allSorted.forEach((s, i) => {
    if (s.arena) s.arena.globalRank = i + 1;
  });
}

// ─── 输出函数 ───

function printTaxonomyStats(skills) {
  console.log('\n[rebuild] 📊 维度分布:');

  const roleCounts = {};
  const sceneCounts = {};
  const platCounts = {};

  for (const s of skills) {
    for (const r of (s.roles || [])) roleCounts[r] = (roleCounts[r] || 0) + 1;
    for (const sc of (s.scenes || [s.scene])) sceneCounts[sc] = (sceneCounts[sc] || 0) + 1;
    for (const p of (s.platforms || [])) platCounts[p] = (platCounts[p] || 0) + 1;
  }

  console.log('  Roles:', Object.entries(roleCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}(${v})`).join(', ') || '(无)');
  console.log('  Scenes:', Object.entries(sceneCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}(${v})`).join(', ') || '(无)');
  console.log('  Platforms:', Object.entries(platCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}(${v})`).join(', ') || '(无)');
}

function printScenePlatformGroups(skills) {
  const groups = {};
  for (const s of skills) {
    const sceneList = s.scenes || [s.scene || 'unknown'];
    const platList = s.platforms || ['general'];
    for (const scene of sceneList) {
      for (const plat of platList) {
        const key = `${scene} × ${plat}`;
        if (!groups[key]) groups[key] = 0;
        groups[key]++;
      }
    }
  }

  console.log('\n[rebuild] 📋 Scene × Platform 分组:');
  const sorted = Object.entries(groups).sort((a, b) => b[1] - a[1]);
  for (const [key, count] of sorted) {
    console.log(`  ${key}: ${count} skills`);
  }
}

function printTopSkills(results) {
  console.log('\n[rebuild] 🏆 Top 10 Skills (by weighted score):');
  const sorted = results.sort((a, b) => b.weightedScore - a.weightedScore).slice(0, 10);
  sorted.forEach((s, i) => {
    console.log(`  ${i + 1}. ${s.skillName.padEnd(30)} Score: ${s.weightedScore.toFixed(1).padStart(5)}  ELO: ${s.elo}`);
  });
}

main().catch(e => {
  console.error('[rebuild] 💥 未预期错误:', e);
  process.exit(1);
});