evolver.js
3.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
/**
* Benchmark Auto-Evolver — 红皇后效应
*
* 种群变强时自动提高 Benchmark 难度:
* 通过率 > 90% → difficulty +1
* 通过率 < 20% → difficulty -1
* 30%-80% → 保持（健康压力区间）
*
* Per-skill: 只调整该 Run 关联 Skill 的 Benchmark。
*/
const evolutionQueries = require('../queries').evolutionQueries;
/**
* 评估并调整单个 Benchmark 的难度
*/
async function evolveBenchmark(benchmarkId, runId) {
const evaluations = await evolutionQueries.getEvaluationsByRun(runId, {});
const benchEvals = evaluations.filter(e => e.benchmark_id === benchmarkId);
if (benchEvals.length < 3) {
return { action: 'skip', reason: '评估数据不足 (<3)' };
}
const passCount = benchEvals.filter(e => e.overall_score >= 0.7).length;
const passRate = passCount / benchEvals.length;
const run = await evolutionQueries.getRun(runId);
const benchmarks = await evolutionQueries.getBenchmarks(run?.skill_name);
const benchmark = benchmarks.find(b => b.id === benchmarkId);
if (!benchmark) return { action: 'skip', reason: 'Benchmark not found' };
const currentDifficulty = benchmark.difficulty || 3;
const currentThreshold = benchmark.pass_threshold || 0.7;
let newDifficulty = currentDifficulty;
let newThreshold = currentThreshold;
let action = 'maintain';
if (passRate > 0.9) {
newDifficulty = Math.min(5, currentDifficulty + 1);
newThreshold = Math.min(0.95, currentThreshold + 0.05);
action = 'increase';
} else if (passRate < 0.2) {
newDifficulty = Math.max(1, currentDifficulty - 1);
newThreshold = Math.max(0.3, currentThreshold - 0.05);
action = 'decrease';
}
if (action !== 'maintain') {
await evolutionQueries.upsertBenchmark({
...benchmark,
difficulty: newDifficulty,
passThreshold: newThreshold,
});
}
return {
benchmarkId, action,
passRate: parseFloat(passRate.toFixed(2)),
evaluationCount: benchEvals.length,
difficulty: { from: currentDifficulty, to: newDifficulty },
threshold: { from: currentThreshold, to: newThreshold },
};
}
/**
* 自进化该 Run 关联 Skill 的所有 Benchmark
*/
async function evolveAllBenchmarks(runId) {
const run = await evolutionQueries.getRun(runId);
const skillName = run?.skill_name;
const benchmarks = await evolutionQueries.getBenchmarks(skillName);
console.log(`🔄 自进化 ${benchmarks.length} 个 Benchmark [${skillName || '?'}]\n`);
const results = [];
for (const benchmark of benchmarks) {
const result = await evolveBenchmark(benchmark.id, runId);
results.push(result);
if (result.action === 'increase') {
console.log(` 📈 ${benchmark.id}: 难度 ${result.difficulty.from} → ${result.difficulty.to} (通过率 ${(result.passRate * 100).toFixed(0)}%)`);
} else if (result.action === 'decrease') {
console.log(` 📉 ${benchmark.id}: 难度 ${result.difficulty.from} → ${result.difficulty.to} (通过率 ${(result.passRate * 100).toFixed(0)}%)`);
} else if (result.action === 'skip') {
console.log(` ⏭️ ${benchmark.id}: ${result.reason}`);
}
}
const adjusted = results.filter(r => r.action !== 'maintain' && r.action !== 'skip');
console.log(`\n✅ Benchmark 自进化完成: ${adjusted.length}/${benchmarks.length} 调整`);
return results;
}
module.exports = { evolveBenchmark, evolveAllBenchmarks };