generator.js
2.79 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
/**
* Benchmark Generator — 从进化目标自动生成 Benchmark 探针
*
* 所有生成由 LLM Agent 完成，无规则降级。
*/
const evolutionQueries = require('../queries').evolutionQueries;
const { callEvaluator } = require('../evaluator');
/**
* 用 LLM 为指定 Skill 自动生成 Benchmark 探针
*/
async function generateBenchmark(skillName, goal, options = {}) {
const { difficulty = 2, probeCount = 2 } = options;
return callEvaluator(
`你是 Agent Skill 评估专家。为一个 Skill 设计 Benchmark 测试探针。
每个探针测试一个认知维度：
1. trap — 陷阱规避：容易犯的错误场景
2. positive — 正向能力：处理场景的能力
3. boundary — 边界识别：知道自己的能力边界
6 个认知维度：mentalModels, patternLibrary, judgment, toolFluency, taste, scarTissue
难度等级: ${difficulty}/5`,
`Skill: ${skillName}
目标: ${goal}
每个维度生成 ${probeCount} 个探针
返回 JSON:
{
"description": "对 ${skillName} 的评估基准",
"probes": [
{ "type": "trap", "dimension": "scarTissue", "scenario": "...", "expected": "...", "weight": 1.0 },
{ "type": "positive", "dimension": "patternLibrary", "scenario": "...", "expected": "...", "weight": 1.0 }
]
}`,
);
}
/**
* 生成并持久化 Benchmark
*/
async function generateAndSaveBenchmark(skillName, goal, options = {}) {
const { difficulty = 2, probeCount = 2 } = options;
console.log(`🎯 为 ${skillName} 生成 Benchmark...`);
const result = await generateBenchmark(skillName, goal, { difficulty, probeCount });
const benchmarkId = `bench-${skillName}-auto-${Date.now()}`;
const benchmark = await evolutionQueries.upsertBenchmark({
id: benchmarkId,
skillName,
description: result.description || `${skillName} 自动生成评估基准`,
difficulty,
probes: result.probes || [],
passThreshold: 0.6,
});
console.log(` ✅ 生成 ${(result.probes || []).length} 个探针 (难度 ${difficulty})`);
return benchmark;
}
/**
* 为 run 中的 Skill 生成 Benchmark
*/
async function generateRunBenchmarks(runId, goal, options = {}) {
const genomes = await evolutionQueries.getGenomesByRun(runId, { aliveOnly: true });
const skillNames = [...new Set(genomes.map(g => g.skill_name))];
console.log(`🎯 为 ${skillNames.length} 个 Skill 生成 Benchmark\n`);
const results = [];
for (const skillName of skillNames) {
try {
const bench = await generateAndSaveBenchmark(skillName, goal, options);
results.push(bench);
} catch (err) {
console.error(` ❌ ${skillName}: ${err.message}`);
}
}
console.log(`\n✅ Benchmark 生成完成: ${results.length}/${skillNames.length}`);
return results;
}
module.exports = {
generateBenchmark,
generateAndSaveBenchmark,
generateRunBenchmarks,
};