arena.js
6.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
/**
* Arena — 进化竞技场
*
* 核心选择压力来源。
* 把基因组编译为 prompt → 用 prompt 在文本场景中推理 → Judge 评分。
* 不需要文件系统,不需要沙盒目录,纯 LLM 推理。
*/
const evolutionQueries = require('../queries').evolutionQueries;
const { evaluateScenario } = require('../evaluator');
// ════════════════════════════════════════════════════════
// Genome → Prompt 编译
// ════════════════════════════════════════════════════════
/**
* 将 6 维基因组编译为自然语言 prompt
*
* 这个 prompt 代表基因组的"表型"——它如何在环境中表现。
*/
function compileGenomeToPrompt(genome) {
const dims = {};
for (const key of ['mental_models', 'pattern_library', 'judgment', 'tool_fluency', 'taste', 'scar_tissue']) {
const val = genome[key];
dims[key] = typeof val === 'string' ? JSON.parse(val) : (val || {});
}
const sections = [];
// 心智模型 → 第一性原理
const mm = dims.mental_models;
if (mm.ironLaws?.length) sections.push(`## 铁律\n${mm.ironLaws.map(l => `- ${typeof l === 'string' ? l : l.insight || JSON.stringify(l)}`).join('\n')}`);
if (mm.causalChains?.length) sections.push(`## 因果推理\n${mm.causalChains.map(c => `- ${typeof c === 'string' ? c : c.model || JSON.stringify(c)}`).join('\n')}`);
// 模式库 → 识别模式
const pl = dims.pattern_library;
if (pl.problemPatterns?.length) sections.push(`## 问题识别模式\n${pl.problemPatterns.map(p => `- **${p.id || p.signature || ''}**: ${p.signature || p.solution || JSON.stringify(p)}`).join('\n')}`);
if (pl.solutionPatterns?.length) sections.push(`## 解决方案模式\n${pl.solutionPatterns.map(p => `- **${p.id || p.name || ''}**: ${p.applicableWhen || JSON.stringify(p)}`).join('\n')}`);
// 判断力 → 决策规则
const jd = dims.judgment;
if (jd.decisionRules?.length) sections.push(`## 决策规则\n${jd.decisionRules.map(r => `- ${typeof r === 'string' ? r : r.decision || JSON.stringify(r)}`).join('\n')}`);
if (jd.exclusions?.length) sections.push(`## 不适用场景\n${jd.exclusions.map(e => `- ${typeof e === 'string' ? e : JSON.stringify(e)}`).join('\n')}`);
// 伤疤记忆 → 反模式
const st = dims.scar_tissue;
if (st.antiPatterns?.length) sections.push(`## 反模式(必须避开)\n${st.antiPatterns.map(a => `- ❌ ${a.pattern || JSON.stringify(a)}`).join('\n')}`);
if (st.failures?.length) sections.push(`## 历史教训\n${st.failures.map(f => `- ${f.lesson || f.preventionRule || JSON.stringify(f)}`).join('\n')}`);
// 品味 → 质量标准
const ta = dims.taste;
if (ta.qualitySignals?.length) sections.push(`## 质量标准\n${ta.qualitySignals.map(s => `- ${s.signal || (typeof s === 'string' ? s : JSON.stringify(s))}`).join('\n')}`);
// 工具流利度 → 工具使用
const tf = dims.tool_fluency;
if (tf.idioms?.length) sections.push(`## 工具使用惯用法\n${tf.idioms.map(i => `- ${i.step || ''}: ${i.tools?.join(', ') || JSON.stringify(i)}`).join('\n')}`);
return `你是一个 ${genome.skill_name} 专家 Agent。以下是你的能力配置:\n\n${sections.join('\n\n')}\n\n请基于以上能力完成后续任务。`;
}
// ════════════════════════════════════════════════════════
// 竞技场评估
// ════════════════════════════════════════════════════════
/**
* 在 Arena 中评估单个基因组
*
* @param {object} genome - 基因组记录
* @param {object[]} scenarios - Benchmark 场景数组
* @returns {{ overall, scores }}
*/
async function evaluateGenomeInArena(genome, scenarios) {
const prompt = compileGenomeToPrompt(genome);
const scores = [];
for (const scenario of scenarios) {
try {
const result = await evaluateScenario(
prompt,
scenario.scenario || scenario.description || '',
scenario.evaluationCriteria || {},
);
scores.push({
scenarioId: scenario.id,
overall: result.overall || 0,
dimensionScores: result.dimensionScores || {},
reasoning: result.reasoning || '',
});
} catch (err) {
console.error(` ⚠️ Arena 场景 ${scenario.id} 评估失败: ${err.message}`);
scores.push({ scenarioId: scenario.id, overall: 0, error: err.message });
}
}
const avgOverall = scores.length > 0
? scores.reduce((sum, s) => sum + (s.overall || 0), 0) / scores.length
: 0;
return { genomeId: genome.id, overall: avgOverall, scores };
}
/**
* 评估 Run 中所有存活基因组
*/
async function evaluateRunInArena(runId) {
const genomes = await evolutionQueries.getGenomesByRun(runId, { aliveOnly: true });
if (genomes.length === 0) return { ranked: [] };
const skillName = genomes[0].skill_name;
const benchmarks = await evolutionQueries.getBenchmarks(skillName);
// 优先使用新的 scenario 字段,兼容旧的 description-only 格式
const scenarios = benchmarks.map(b => {
const criteria = typeof b.evaluation_criteria === 'string'
? JSON.parse(b.evaluation_criteria) : (b.evaluation_criteria || {});
return {
id: b.id,
scenario: b.scenario || b.description || '',
evaluationCriteria: criteria,
};
});
if (scenarios.length === 0) {
console.log(` ⚠️ ${skillName} 没有 Benchmark 场景,跳过 Arena`);
return { ranked: [] };
}
console.log(`🏟️ Arena: ${genomes.length} 基因组 × ${scenarios.length} 场景\n`);
const results = [];
for (const genome of genomes) {
const result = await evaluateGenomeInArena(genome, scenarios);
results.push(result);
// 更新 overall_fitness
await evolutionQueries.updateGenomeFitness(genome.id, {}, result.overall);
console.log(` ${genome.skill_name}[${genome.id.slice(0, 6)}]: ${(result.overall * 100).toFixed(1)}%`);
}
// 按得分排序
results.sort((a, b) => b.overall - a.overall);
return { ranked: results };
}
module.exports = {
compileGenomeToPrompt,
evaluateGenomeInArena,
evaluateRunInArena,
};