/** * Benchmark Executor — 用 Benchmark 探针评估基因组 * * 所有探针评估由 LLM Agent 完成,无字符串匹配降级。 */ const evolutionQueries = require('../queries').evolutionQueries; const { evaluateBenchmarkProbe } = require('../evaluator'); /** * 用 Benchmark 评估单个基因组 */ async function evaluateGenomeWithBenchmark(genomeId, benchmarkId) { const genome = await evolutionQueries.getGenome(genomeId); if (!genome) throw new Error(`Genome ${genomeId} not found`); const benchmarks = await evolutionQueries.getBenchmarks(genome.skill_name); const benchmark = benchmarks.find(b => b.id === benchmarkId); if (!benchmark) throw new Error(`Benchmark ${benchmarkId} not found`); const probes = typeof benchmark.probes === 'string' ? JSON.parse(benchmark.probes) : (benchmark.probes || []); if (probes.length === 0) return { score: null, passed: false, message: 'No probes' }; const probeResults = {}; let totalScore = 0; for (const probe of probes) { const type = probe.type || 'positive'; const dimMap = { trap: 'scar_tissue', positive: 'pattern_library', boundary: 'judgment' }; const dimKey = dimMap[type] || 'pattern_library'; const dimData = typeof genome[dimKey] === 'string' ? JSON.parse(genome[dimKey]) : (genome[dimKey] || {}); const result = await evaluateBenchmarkProbe(probe, dimData); probeResults[probe.name] = { probeName: probe.name, type, passed: !!result.passed, score: typeof result.score === 'number' ? result.score : (result.passed ? 1.0 : 0.0), reasoning: result.reasoning || '', }; totalScore += probeResults[probe.name].score; } const overall = probes.length > 0 ? totalScore / probes.length : 0; const passed = overall >= (benchmark.pass_threshold || 0.6); return { genomeId, benchmarkId, probeResults, overall, passed }; } /** * 用该 Skill 的所有 Benchmark 评估 Run 中所有存活基因组 */ async function evaluateRunWithBenchmarks(runId) { const genomes = await evolutionQueries.getGenomesByRun(runId, { aliveOnly: true }); if (genomes.length === 0) return []; // 只取该 Skill 的 Benchmark const skillName = genomes[0].skill_name; const benchmarks = await evolutionQueries.getBenchmarks(skillName); if (benchmarks.length === 0) { console.log(` ⚠️ ${skillName} 没有 Benchmark,跳过探针评估`); return []; } console.log(`🎯 Benchmark 评估: ${genomes.length} 基因组 × ${benchmarks.length} Benchmark\n`); const results = []; for (const genome of genomes) { for (const benchmark of benchmarks) { try { const result = await evaluateGenomeWithBenchmark(genome.id, benchmark.id); results.push(result); console.log(` ${genome.skill_name}[${genome.id.slice(0, 6)}] × ${benchmark.id}: ${result.passed ? '✅' : '❌'} ${(result.overall * 100).toFixed(0)}%`); } catch (err) { console.error(` ❌ ${genome.id.slice(0, 6)} × ${benchmark.id}: ${err.message}`); } } } return results; } module.exports = { evaluateGenomeWithBenchmark, evaluateRunWithBenchmarks, };