benchmark-executor.js
3.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
/**
* Benchmark Executor — 用 Benchmark 探针评估基因组
*
* 所有探针评估由 LLM Agent 完成，无字符串匹配降级。
*/
const evolutionQueries = require('../queries').evolutionQueries;
const { evaluateBenchmarkProbe } = require('../evaluator');
/**
* 用 Benchmark 评估单个基因组
*/
async function evaluateGenomeWithBenchmark(genomeId, benchmarkId) {
const genome = await evolutionQueries.getGenome(genomeId);
if (!genome) throw new Error(`Genome ${genomeId} not found`);
const benchmarks = await evolutionQueries.getBenchmarks(genome.skill_name);
const benchmark = benchmarks.find(b => b.id === benchmarkId);
if (!benchmark) throw new Error(`Benchmark ${benchmarkId} not found`);
const probes = typeof benchmark.probes === 'string'
? JSON.parse(benchmark.probes) : (benchmark.probes || []);
if (probes.length === 0) return { score: null, passed: false, message: 'No probes' };
const probeResults = {};
let totalScore = 0;
for (const probe of probes) {
const type = probe.type || 'positive';
const dimMap = { trap: 'scar_tissue', positive: 'pattern_library', boundary: 'judgment' };
const dimKey = dimMap[type] || 'pattern_library';
const dimData = typeof genome[dimKey] === 'string'
? JSON.parse(genome[dimKey]) : (genome[dimKey] || {});
const result = await evaluateBenchmarkProbe(probe, dimData);
probeResults[probe.name] = {
probeName: probe.name,
type,
passed: !!result.passed,
score: typeof result.score === 'number' ? result.score : (result.passed ? 1.0 : 0.0),
reasoning: result.reasoning || '',
};
totalScore += probeResults[probe.name].score;
}
const overall = probes.length > 0 ? totalScore / probes.length : 0;
const passed = overall >= (benchmark.pass_threshold || 0.6);
return { genomeId, benchmarkId, probeResults, overall, passed };
}
/**
* 用该 Skill 的所有 Benchmark 评估 Run 中所有存活基因组
*/
async function evaluateRunWithBenchmarks(runId) {
const genomes = await evolutionQueries.getGenomesByRun(runId, { aliveOnly: true });
if (genomes.length === 0) return [];
// 只取该 Skill 的 Benchmark
const skillName = genomes[0].skill_name;
const benchmarks = await evolutionQueries.getBenchmarks(skillName);
if (benchmarks.length === 0) {
console.log(` ⚠️ ${skillName} 没有 Benchmark，跳过探针评估`);
return [];
}
console.log(`🎯 Benchmark 评估: ${genomes.length} 基因组 × ${benchmarks.length} Benchmark\n`);
const results = [];
for (const genome of genomes) {
for (const benchmark of benchmarks) {
try {
const result = await evaluateGenomeWithBenchmark(genome.id, benchmark.id);
results.push(result);
console.log(` ${genome.skill_name}[${genome.id.slice(0, 6)}] × ${benchmark.id}: ${result.passed ? '✅' : '❌'} ${(result.overall * 100).toFixed(0)}%`);
} catch (err) {
console.error(` ❌ ${genome.id.slice(0, 6)} × ${benchmark.id}: ${err.message}`);
}
}
}
return results;
}
module.exports = {
evaluateGenomeWithBenchmark,
evaluateRunWithBenchmarks,
};