#!/usr/bin/env node /** * rebuild-benchmarks.js — 重建 Benchmark 评测 * * 对所有 Skill 重新运行 Benchmark + ELO Tournament, * 按新的 (scene × platform) 分组计算排名。 * * 用法: * node scripts/rebuild-benchmarks.js # 全量重建 * node scripts/rebuild-benchmarks.js --scene video_creation # 只重建某场景 * node scripts/rebuild-benchmarks.js --dry-run # 预览模式 * node scripts/rebuild-benchmarks.js --stats # 只输出统计,不运行 * * Cron 示例 (每天凌晨 3 点运行): * 0 3 * * * cd /path/to/lemonape && node scripts/rebuild-benchmarks.js >> .logs/benchmark-cron.log 2>&1 */ const path = require('path'); const ROOT = path.resolve(__dirname, '..'); // 加载项目模块 const skillDb = require(path.join(ROOT, 'server/services/skill-db')); const benchmarkDB = require(path.join(ROOT, 'server/services/benchmark-db')); const benchmarkRunner = require(path.join(ROOT, 'server/services/benchmark-runner')); const taxonomy = require(path.join(ROOT, 'server/config/taxonomy')); // ─── 参数解析 ─── const args = process.argv.slice(2); const dryRun = args.includes('--dry-run'); const statsOnly = args.includes('--stats'); const sceneIdx = args.indexOf('--scene'); const targetScene = sceneIdx !== -1 ? args[sceneIdx + 1] : null; async function main() { console.log('═══════════════════════════════════════════'); console.log(' 🔬 Benchmark Rebuild'); console.log('═══════════════════════════════════════════'); console.log(` Dry run: ${dryRun}`); console.log(` Scene: ${targetScene || 'ALL'}`); console.log(` Stats: ${statsOnly}`); console.log(''); // 初始化 benchmark DB benchmarkDB.initDB(); // 加载所有 Skill let skills = skillDb.listSkills(); console.log(`[rebuild] 📦 总计 ${skills.length} 个 Skills`); // 确保所有 Skill 都有三维标签 let taggedCount = 0; for (const skill of skills) { taxonomy.migrateOldScene(skill); if (!skill.roles || skill.roles.length === 0 || !skill.scenes || skill.scenes.length === 0 || !skill.platforms || skill.platforms.length === 0) { const tags = taxonomy.autoTag(skill); if (!skill.roles || skill.roles.length === 0) skill.roles = tags.roles; if (!skill.scenes || skill.scenes.length === 0) skill.scenes = tags.scenes; if (!skill.platforms || skill.platforms.length === 0) skill.platforms = tags.platforms; if (!skill.scene && skill.scenes.length > 0) skill.scene = skill.scenes[0]; taggedCount++; } } if (taggedCount > 0) { console.log(`[rebuild] 🏷️ 自动打标 ${taggedCount} 个未标注 Skills`); } // 按场景过滤 if (targetScene) { skills = skills.filter(s => (s.scenes || [s.scene]).includes(targetScene)); console.log(`[rebuild] 🎯 筛选场景 "${targetScene}": ${skills.length} 个 Skills`); } if (skills.length === 0) { console.log('[rebuild] ⚠️ 没有 Skills,退出。'); return; } // 统计维度分布 printTaxonomyStats(skills); if (statsOnly) { console.log('[rebuild] ℹ️ Stats-only 模式,跳过 benchmark。'); return; } if (dryRun) { console.log('[rebuild] ℹ️ Dry-run 模式,以下为将执行的操作预览:'); console.log(`[rebuild] 将对 ${skills.length} 个 Skills 运行 Benchmark`); printScenePlatformGroups(skills); return; } // ── 执行 Benchmark ── console.log('\n[rebuild] 🚀 开始 Benchmark...'); const results = []; let benchCount = 0; for (const skill of skills) { try { const { scores, weightedScore } = benchmarkRunner.benchmarkSkill(skill); results.push({ skillName: skill.skillName, layer: skill.layer, scenes: skill.scenes, platforms: skill.platforms, weightedScore, elo: skill.arena?.elo || 1200, }); benchCount++; if (benchCount % 20 === 0) { console.log(`[rebuild] 进度: ${benchCount}/${skills.length}`); } } catch (e) { console.error(`[rebuild] ❌ ${skill.skillName}: ${e.message}`); } } console.log(`[rebuild] ✅ Benchmark 完成: ${benchCount}/${skills.length}`); // ── ELO Tournament (按 scene × platform 分组) ── console.log('\n[rebuild] ⚔️ 运行 ELO Tournament...'); const totalComparisons = runScenePlatformTournament(skills); console.log(`[rebuild] ✅ ELO 对战完成: ${totalComparisons} 次比较`); // ── 计算排名 ── computeRanks(skills); // ── 保存 ── console.log('[rebuild] 💾 保存结果...'); for (const skill of skills) { const updates = { ...skill }; delete updates.skillId; delete updates.skillName; skillDb.updateSkill(skill.skillName, updates); } // ── 打印排行榜 ── printTopSkills(results); console.log('\n═══════════════════════════════════════════'); console.log(` ✅ Benchmark Rebuild 完成`); console.log(` Skills: ${benchCount}`); console.log(` ELO Comparisons: ${totalComparisons}`); console.log('═══════════════════════════════════════════'); } /** * 按 scene × platform 分组做 ELO Tournament */ function runScenePlatformTournament(skills) { const groups = {}; for (const s of skills) { const sceneList = s.scenes || [s.scene || 'unknown']; const platList = s.platforms || ['general']; for (const scene of sceneList) { for (const plat of platList) { const key = `${scene}_${plat}`; if (!groups[key]) groups[key] = []; groups[key].push(s); } } } let totalComparisons = 0; for (const [groupKey, group] of Object.entries(groups)) { if (group.length < 2) continue; for (let i = 0; i < group.length; i++) { for (let j = i + 1; j < group.length; j++) { const a = group[i]; const b = group[j]; const scoreA = a.arena?.scores ? benchmarkRunner.computeWeightedScore(a.arena.scores, a.layer) : 50; const scoreB = b.arena?.scores ? benchmarkRunner.computeWeightedScore(b.arena.scores, b.layer) : 50; const eloA = a.arena?.elo || 1200; const eloB = b.arena?.elo || 1200; const kA = benchmarkRunner.getKFactor(a.arena?.benchmarkRuns || 0); const kB = benchmarkRunner.getKFactor(b.arena?.benchmarkRuns || 0); // Inline ELO calculation const expA = 1 / (1 + Math.pow(10, (eloB - eloA) / 400)); const actualA = scoreA > scoreB ? 1 : scoreA === scoreB ? 0.5 : 0; const newA = Math.round(eloA + kA * (actualA - expA)); const newB = Math.round(eloB + kB * ((1 - actualA) - (1 - expA))); if (a.arena) a.arena.elo = newA; if (b.arena) b.arena.elo = newB; const winner = scoreA > scoreB ? a.skillName : scoreB > scoreA ? b.skillName : 'tie'; benchmarkDB.insertComparison({ skillA: a.skillName, skillB: b.skillName, scene: groupKey.split('_')[0], layer: a.layer, winner, scoreA, scoreB, }); totalComparisons++; } } } return totalComparisons; } /** * 计算排名 */ function computeRanks(skills) { const groups = {}; for (const s of skills) { const sceneList = s.scenes || [s.scene || 'unknown']; const platList = s.platforms || ['general']; for (const scene of sceneList) { for (const plat of platList) { const key = `${scene}_${plat}`; if (!groups[key]) groups[key] = []; groups[key].push(s); } } } for (const [, group] of Object.entries(groups)) { group.sort((a, b) => (b.arena?.elo || 1200) - (a.arena?.elo || 1200)); group.forEach((s, i) => { if (s.arena) { s.arena.rank = i + 1; s.arena.highlighted = (i < 3); } }); } const allSorted = [...skills].sort((a, b) => (b.arena?.elo || 1200) - (a.arena?.elo || 1200)); allSorted.forEach((s, i) => { if (s.arena) s.arena.globalRank = i + 1; }); } // ─── 输出函数 ─── function printTaxonomyStats(skills) { console.log('\n[rebuild] 📊 维度分布:'); const roleCounts = {}; const sceneCounts = {}; const platCounts = {}; for (const s of skills) { for (const r of (s.roles || [])) roleCounts[r] = (roleCounts[r] || 0) + 1; for (const sc of (s.scenes || [s.scene])) sceneCounts[sc] = (sceneCounts[sc] || 0) + 1; for (const p of (s.platforms || [])) platCounts[p] = (platCounts[p] || 0) + 1; } console.log(' Roles:', Object.entries(roleCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}(${v})`).join(', ') || '(无)'); console.log(' Scenes:', Object.entries(sceneCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}(${v})`).join(', ') || '(无)'); console.log(' Platforms:', Object.entries(platCounts).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${k}(${v})`).join(', ') || '(无)'); } function printScenePlatformGroups(skills) { const groups = {}; for (const s of skills) { const sceneList = s.scenes || [s.scene || 'unknown']; const platList = s.platforms || ['general']; for (const scene of sceneList) { for (const plat of platList) { const key = `${scene} × ${plat}`; if (!groups[key]) groups[key] = 0; groups[key]++; } } } console.log('\n[rebuild] 📋 Scene × Platform 分组:'); const sorted = Object.entries(groups).sort((a, b) => b[1] - a[1]); for (const [key, count] of sorted) { console.log(` ${key}: ${count} skills`); } } function printTopSkills(results) { console.log('\n[rebuild] 🏆 Top 10 Skills (by weighted score):'); const sorted = results.sort((a, b) => b.weightedScore - a.weightedScore).slice(0, 10); sorted.forEach((s, i) => { console.log(` ${i + 1}. ${s.skillName.padEnd(30)} Score: ${s.weightedScore.toFixed(1).padStart(5)} ELO: ${s.elo}`); }); } main().catch(e => { console.error('[rebuild] 💥 未预期错误:', e); process.exit(1); });