/** * EvolutionLoop — Skill 进化 RL 闭环引擎 * * 实现完整的 Perceive → Reason/Plan → Execute → Evaluate → Reflect 循环, * 将 Skill 质量从初始状态迭代到 Goal Score (默认 ≥ 90)。 * * 哲学方法论映射: * Perceive → _perceive() — 评估当前 Skill 状态 * Reason & Plan → _plan() — 制定进化策略 (参考 Memory) * Execute → _execute() — 调用 Hunt/Digest/Evolve agents * Feedback → _evaluate() — 调用 Benchmark agent 评分 * Reflect & Adapt→ _reflect() — 分析失分 / 提取教训 * Memory → EvolutionMemory — 存储历史经验 * Goal Check → score >= goalScore */ const fs = require('fs'); const path = require('path'); const { EvolutionMemory } = require('../src/core/evolution-memory'); // 默认配置 const DEFAULT_MAX_ITERATIONS = 5; const DEFAULT_GOAL_SCORE = 90; const STAGNATION_ROUNDS = 3; const STAGNATION_THRESHOLD = 2; class EvolutionLoop { /** * @param {object} opts * @param {string} opts.skillName - Skill 名称 * @param {string} opts.description - Skill 描述 * @param {string} opts.skillDir - Skill 目录绝对路径 * @param {string} [opts.instructions] - 用户额外要求 * @param {object} opts.manager - AgentManager 实例 * @param {object} [opts.jobManager] - JobManager 实例 * @param {object} [opts.job] - Job 对象 * @param {object} [opts.logger] - ConversationLogger 实例 * @param {object} [opts.taskMgr] - SkillTaskManager 模块 * @param {number} [opts.maxIterations] - 最大迭代次数 * @param {number} [opts.goalScore] - 目标分数 * @param {function} [opts.onData] - 流式数据回调 */ constructor(opts) { this.skillName = opts.skillName; this.description = opts.description; this.skillDir = opts.skillDir; this.instructions = opts.instructions || ''; this.manager = opts.manager; this.jobManager = opts.jobManager || null; this.job = opts.job || null; this.logger = opts.logger || null; this.taskMgr = opts.taskMgr || null; this.maxIterations = opts.maxIterations || DEFAULT_MAX_ITERATIONS; this.goalScore = opts.goalScore || DEFAULT_GOAL_SCORE; this.onData = opts.onData || (() => {}); this.memory = new EvolutionMemory(this.skillDir); // 运行时状态 this._currentIteration = 0; this._lastScore = null; this._bestScore = null; this._bestIteration = -1; } /** * 执行完整的 RL 进化循环 * * @returns {Promise} */ async run() { this._log(`🔄 EvolutionLoop 启动: ${this.skillName} (Goal: ≥${this.goalScore}, Max: ${this.maxIterations}轮)`); for (let i = 0; i < this.maxIterations; i++) { this._currentIteration = i; this._log(`\n═══ 迭代 ${i + 1}/${this.maxIterations} ═══`); this._updateJob('evolving', `🔄 迭代 ${i + 1}/${this.maxIterations}...`); // ── 1. PERCEIVE: 感知当前状态 ── this._updateJob('perceiving', `👁️ 迭代 ${i + 1}: 感知环境...`); const state = await this._perceive(); this._log(`[Perceive] 状态: hasSkillMd=${state.hasSkillMd}, hasKnowhow=${state.hasKnowhow}, previousScore=${state.previousScore}`); // ── 2. REASON & PLAN: 制定策略 ── this._updateJob('planning', `🧠 迭代 ${i + 1}: 制定策略...`); const strategy = this._plan(state, i); this._log(`[Plan] 策略: ${strategy.approach}, 步骤: [${strategy.steps.join(', ')}]`); // ── 3. EXECUTE: 执行进化 ── this._updateJob('executing', `⚡ 迭代 ${i + 1}: 执行 [${strategy.steps.join('→')}]...`); const execResult = await this._execute(strategy); // ── 4. FEEDBACK: 评分 ── this._updateJob('evaluating', `📊 迭代 ${i + 1}: Benchmark 评分...`); const score = await this._evaluate(); this._log(`[Feedback] 分数: ${score.totalScore}/${this.goalScore} (passed: ${score.passed})`); // 更新最佳分数 if (this._bestScore === null || score.totalScore > this._bestScore) { this._bestScore = score.totalScore; this._bestIteration = i; } this._lastScore = score.totalScore; // ── 5. REFLECT & ADAPT: 反思 ── this._updateJob('reflecting', `🔍 迭代 ${i + 1}: 反思分析...`); const reflection = this._reflect(score, strategy); // ── 6. MEMORY: 记录 ── this.memory.record({ iteration: i, strategy: { approach: strategy.approach, steps: strategy.steps, focus: strategy.focus }, score: score.totalScore, weakDimensions: reflection.weakDimensions, strongDimensions: reflection.strongDimensions, adjustments: reflection.adjustments, lessonsLearned: reflection.lessonsLearned, }); // ── GOAL CHECK ── if (reflection.goalAchieved) { this._log(`✅ Goal Achieved! Score: ${score.totalScore} ≥ ${this.goalScore} (迭代 ${i + 1} 轮)`); return { status: 'goal_achieved', finalScore: score.totalScore, iterations: i + 1, bestScore: this._bestScore, scoreDetails: score, }; } // 停滞检测 if (this.memory.isStagnant(STAGNATION_ROUNDS, STAGNATION_THRESHOLD)) { this._log(`⚠️ 停滞检测: 连续 ${STAGNATION_ROUNDS} 轮分数变化 < ${STAGNATION_THRESHOLD},停止迭代`); return { status: 'stagnant', finalScore: score.totalScore, iterations: i + 1, bestScore: this._bestScore, scoreDetails: score, }; } this._log(`[Reflect] 未达标 (${score.totalScore} < ${this.goalScore}), 继续迭代...`); } // 最大迭代次数用尽 this._log(`⚠️ 达到最大迭代次数 ${this.maxIterations}, 最佳分数: ${this._bestScore}`); return { status: 'max_iterations', finalScore: this._lastScore, iterations: this.maxIterations, bestScore: this._bestScore, scoreDetails: null, }; } // ══════════════════════════════════════════════ // 1. PERCEIVE — 感知当前状态 // ══════════════════════════════════════════════ async _perceive() { const state = { hasSkillMd: fs.existsSync(path.join(this.skillDir, 'SKILL.md')), hasKnowhow: false, hasBenchmark: false, previousScore: this._lastScore, totalAttempts: this.memory.getHistory().totalAttempts, bestHistoricalScore: this.memory.getHistory().bestScore, skillMdSummary: '', knowhowSummary: '', }; // 检查 knowhow (可能在 hunt/ 或 hunter/ 目录下) const knowhowPaths = [ path.join(this.skillDir, 'hunt', 'knowhow.md'), path.join(this.skillDir, 'hunter', 'knowhow.md'), ]; for (const p of knowhowPaths) { if (fs.existsSync(p)) { state.hasKnowhow = true; state.knowhowSummary = this._readFirst(p, 500); break; } } // 检查 benchmark const benchmarkPath = path.join(this.skillDir, 'digest', 'benchmark.md'); if (fs.existsSync(benchmarkPath)) { state.hasBenchmark = true; } // 读取 SKILL.md 摘要 if (state.hasSkillMd) { state.skillMdSummary = this._readFirst(path.join(this.skillDir, 'SKILL.md'), 500); } return state; } // ══════════════════════════════════════════════ // 2. REASON & PLAN — 策略制定 // ══════════════════════════════════════════════ _plan(state, iteration) { const lessons = this.memory.getLessons(); const recentWeaks = this.memory.getRecentWeakDimensions(3); // 第一轮 + 没有基础内容: 完整流程 if (iteration === 0 && !state.hasSkillMd) { return { approach: 'full', steps: ['hunt', 'digest', 'skill_create'], focus: [], avoid: lessons, reasoning: '首次创建,需要完整 Hunt→Digest→SkillCreate 流程', }; } // 第一轮 + 有 SKILL.md 但没有 benchmark: 需要先 Digest if (iteration === 0 && state.hasSkillMd && !state.hasBenchmark) { return { approach: 'build_benchmark', steps: state.hasKnowhow ? ['digest', 'evolve'] : ['hunt', 'digest', 'evolve'], focus: [], avoid: lessons, reasoning: '已有 SKILL.md 但缺少评估基准,需先生成 benchmark', }; } // 有分数且 ≥ 70: 局部优化 (仅 Evolve) if (state.previousScore && state.previousScore >= 70) { return { approach: 'refine', steps: ['evolve'], focus: recentWeaks, avoid: lessons, reasoning: `分数 ${state.previousScore} ≥ 70,仅对弱项维度做局部优化`, }; } // 有分数但 < 70: 深度重构 if (state.previousScore && state.previousScore < 70) { return { approach: 'restructure', steps: ['hunt', 'digest', 'evolve'], focus: recentWeaks, avoid: lessons, reasoning: `分数 ${state.previousScore} < 70,需要深度重构 (Re-Hunt + Re-Digest + Evolve)`, }; } // 非第一轮、有基础内容: 默认 evolve return { approach: 'refine', steps: ['evolve'], focus: recentWeaks, avoid: lessons, reasoning: '默认局部优化策略', }; } // ══════════════════════════════════════════════ // 3. EXECUTE — 按策略执行 // ══════════════════════════════════════════════ async _execute(strategy) { const agentMap = { 'hunt': { agent: 'l3-hunt', label: '🔍 Hunt' }, 'digest': { agent: 'l2-digest', label: '📝 Digest' }, 'skill_create': { agent: 'skill-creator', label: '🔨 SkillCreate' }, 'evolve': { agent: 'innovator-evolver', label: '🔄 Evolve' }, }; const results = {}; for (const step of strategy.steps) { const mapping = agentMap[step]; if (!mapping) { this._log(`[Execute] 跳过未知步骤: ${step}`); continue; } this._log(`[Execute] ${mapping.label} — ${mapping.agent}`); if (this.taskMgr) { try { this.taskMgr.updateStep(this.skillDir, step, 'running'); } catch {} } const prompt = this._buildStepPrompt(step, strategy); try { const result = await this.manager.callAgent(mapping.agent, prompt, { onData: (chunk) => { if (chunk?.trim()) { this.onData(mapping.agent, chunk); if (this.logger) this.logger.logStreamChunk(mapping.agent, chunk); } }, }); results[step] = { success: true, output: result?.output || '' }; if (this.taskMgr) { try { this.taskMgr.updateStep(this.skillDir, step, 'done'); } catch {} } this._log(`[Execute] ✅ ${step} 完成`); } catch (err) { results[step] = { success: false, error: err.message }; if (this.taskMgr) { try { this.taskMgr.updateStep(this.skillDir, step, 'failed', err.message); } catch {} } this._log(`[Execute] ❌ ${step} 失败: ${err.message}`); throw err; // 关键步骤失败,中断 Loop } } return results; } /** * 构建单步 Agent Prompt (注入历史教训 + 聚焦指令) */ _buildStepPrompt(step, strategy) { const instructionBlock = this.instructions ? `\n\n## 用户额外要求\n${this.instructions}\n` : ''; // 教训 + 聚焦信息 let guidanceBlock = ''; if (strategy.focus && strategy.focus.length > 0) { guidanceBlock += `\n\n## 聚焦改进维度\n以下维度在之前的评估中得分较低,请重点关注:\n${strategy.focus.map(d => `- ${d}`).join('\n')}\n`; } if (strategy.avoid && strategy.avoid.length > 0) { guidanceBlock += `\n\n## 历史教训 (避免重复)\n${strategy.avoid.map(l => `- ${l}`).join('\n')}\n`; } const workDir = path.dirname(this.skillDir); switch (step) { case 'hunt': return [ `请为 "${this.skillName}" 猎取领域知识。`, `- 描述: ${this.description}`, `- 目标: 在 ${this.skillName}/hunt/ 下生成 knowhow.md`, `- 工作目录: ${workDir}`, instructionBlock, guidanceBlock, ].join('\n'); case 'digest': return [ `请消化 ${this.skillName}/hunt/ 下的知识,生成 benchmark。`, `- 描述: ${this.description}`, `- 目标: 在 ${this.skillName}/digest/ 下生成 benchmark.md`, `- 工作目录: ${workDir}`, instructionBlock, guidanceBlock, ].join('\n'); case 'skill_create': return [ `请为 "${this.skillName}" 创建 SKILL.md。`, `- 描述: ${this.description}`, `- 知识库: ${this.skillName}/hunt/knowhow.md`, `- 评估基准: ${this.skillName}/digest/benchmark.md`, `- SKILL.md 输出位置: ${this.skillName}/SKILL.md`, `- 工作目录: ${workDir}`, instructionBlock, guidanceBlock, ].join('\n'); case 'evolve': return [ `请对 Skill "${this.skillName}" 进行进化优化:`, `- skillName: ${this.skillName}`, `- 当前 SKILL.md: ${this.skillName}/SKILL.md`, `- 知识库: ${this.skillName}/hunt/knowhow.md 或 ${this.skillName}/hunter/knowhow.md`, `- 评估基准: ${this.skillName}/digest/benchmark.md`, `- 通过阈值: ${this.goalScore} 分`, `- 最大迭代: 3 次 (这是内循环,外循环会再次调用你)`, `- 工作目录: ${workDir}`, instructionBlock, guidanceBlock, `请执行 evolve 循环: 执行 → 评分 → 分析弱项 → 改写 SKILL.md → 重复。`, `结果保存到 ${this.skillName}/evolve/ 目录。`, `最终更新 ${this.skillName}/SKILL.md。`, ].join('\n'); default: return `请执行 ${step} 步骤。\n${this.description}${instructionBlock}${guidanceBlock}`; } } // ══════════════════════════════════════════════ // 4. FEEDBACK — Benchmark 评分 // ══════════════════════════════════════════════ async _evaluate() { // 读取当前 SKILL.md const skillMdPath = path.join(this.skillDir, 'SKILL.md'); if (!fs.existsSync(skillMdPath)) { this._log('[Evaluate] SKILL.md 不存在,返回 0 分'); return { totalScore: 0, passed: false, dimensions: [], weakDimensions: [], overallFeedback: 'SKILL.md 不存在' }; } const skillMd = fs.readFileSync(skillMdPath, 'utf-8'); // 读取 benchmark 定义 const benchmarkPath = path.join(this.skillDir, 'digest', 'benchmark.md'); if (!fs.existsSync(benchmarkPath)) { this._log('[Evaluate] benchmark.md 不存在,返回 0 分'); return { totalScore: 0, passed: false, dimensions: [], weakDimensions: [], overallFeedback: 'benchmark.md 不存在' }; } const benchmarkDef = fs.readFileSync(benchmarkPath, 'utf-8'); // 调用 benchmark agent 做评估 const evalPrompt = [ `请评估以下 Skill 的质量。`, ``, `## 模式: 评估 Skill 输出`, ``, `- skillName: ${this.skillName}`, ``, `### benchmarkDef (评分标准):`, ``, benchmarkDef, ``, `### taskOutput (待评估的 SKILL.md 内容):`, ``, skillMd, ``, `### testTask:`, `评估 SKILL.md 的方法论完整性、可执行性和质量标准覆盖度。`, ``, `请严格按 benchmark 的 Detailed Rubric 逐维度评分,输出 JSON 格式结果。`, ].join('\n'); try { const result = await this.manager.callAgent('benchmark', evalPrompt, { onData: (chunk) => { if (chunk?.trim()) { this.onData('benchmark', chunk); if (this.logger) this.logger.logStreamChunk('benchmark', chunk); } }, }); return this._parseScoreFromOutput(result?.output || ''); } catch (err) { this._log(`[Evaluate] benchmark agent 调用失败: ${err.message}`); // 降级: 从 evolve 输出中正则提取 return this._fallbackScore(); } } /** * 解析 benchmark agent 输出的结构化分数 */ _parseScoreFromOutput(output) { // 优先: 解析 JSON 块 const jsonMatch = output.match(/```json\n([\s\S]*?)\n```/); if (jsonMatch) { try { const parsed = JSON.parse(jsonMatch[1]); return { totalScore: parsed.totalScore || 0, passed: parsed.passed || false, dimensions: parsed.dimensions || [], weakDimensions: parsed.weakDimensions || [], strongDimensions: parsed.strongDimensions || [], overallFeedback: parsed.overallFeedback || '', }; } catch {} } // 降级 1: 直接 JSON 解析 (可能无 code fence) try { const plainJsonMatch = output.match(/\{[\s\S]*"totalScore"[\s\S]*\}/); if (plainJsonMatch) { const parsed = JSON.parse(plainJsonMatch[0]); return { totalScore: parsed.totalScore || 0, passed: parsed.passed || false, dimensions: parsed.dimensions || [], weakDimensions: parsed.weakDimensions || [], strongDimensions: parsed.strongDimensions || [], overallFeedback: parsed.overallFeedback || '', }; } } catch {} // 降级 2: 正则提取 const scorePatterns = [ /totalScore["\s:]+(\d+)/, /总分[::\s]+(\d+)/, /评分[::\s]+(\d+)/, /最终分数[::\s]+(\d+)/, /score["\s:]+(\d+)/, ]; for (const pattern of scorePatterns) { const match = output.match(pattern); if (match) { const score = parseInt(match[1]); return { totalScore: score, passed: score >= this.goalScore, dimensions: [], weakDimensions: [], strongDimensions: [], overallFeedback: output.substring(0, 500), }; } } // 最后降级: 无法解析 this._log('[Evaluate] 无法从输出中解析分数,使用降级评分'); return this._fallbackScore(); } /** * 降级评分 (无法调用 benchmark 时) * 基于文件存在性做简单估分 */ _fallbackScore() { let score = 30; // 基础分 if (fs.existsSync(path.join(this.skillDir, 'SKILL.md'))) { const content = fs.readFileSync(path.join(this.skillDir, 'SKILL.md'), 'utf-8'); if (content.length > 500) score += 15; if (content.length > 2000) score += 10; if (content.includes('## 执行流程') || content.includes('## Methodology')) score += 10; if (content.includes('## 输出格式') || content.includes('## Output')) score += 5; if (content.includes('---\nname:')) score += 5; // YAML frontmatter } return { totalScore: score, passed: score >= this.goalScore, dimensions: [], weakDimensions: [], strongDimensions: [], overallFeedback: '降级评分 (benchmark agent 不可用)', }; } // ══════════════════════════════════════════════ // 5. REFLECT & ADAPT — 反思 // ══════════════════════════════════════════════ _reflect(score, strategy) { const reflection = { goalAchieved: score.totalScore >= this.goalScore, score: score.totalScore, weakDimensions: score.weakDimensions || [], strongDimensions: score.strongDimensions || [], adjustments: [], lessonsLearned: [], }; if (!reflection.goalAchieved) { // 分析弱项 for (const dim of (score.dimensions || [])) { if (dim.score && dim.score < 70) { reflection.adjustments.push({ dimension: dim.name, currentScore: dim.score, feedback: dim.feedback, action: `聚焦改进 ${dim.name}: ${dim.feedback || '需要加强'}`, }); } } // 提取策略级教训 if (strategy.approach === 'refine' && score.totalScore < 50) { reflection.lessonsLearned.push( `refine 策略对极低分 (${score.totalScore} < 50) 无效,应使用 restructure 策略` ); } if (strategy.approach === 'full' && score.totalScore < 40) { reflection.lessonsLearned.push( `首次 full 创建仅得 ${score.totalScore} 分,可能需要更详细的用户指令或更深度的 Hunt` ); } // 得高分但未达标的教训 if (score.totalScore >= 80 && score.totalScore < this.goalScore) { const weakNames = (score.weakDimensions || []).join(', '); if (weakNames) { reflection.lessonsLearned.push( `接近达标 (${score.totalScore}),瓶颈在: ${weakNames},需要精准优化这几个维度` ); } } } return reflection; } // ══════════════════════════════════════════════ // 辅助方法 // ══════════════════════════════════════════════ _readFirst(filePath, maxLen = 500) { try { const content = fs.readFileSync(filePath, 'utf-8'); return content.substring(0, maxLen); } catch { return ''; } } _log(msg) { console.log(`[EvolutionLoop:${this.skillName}] ${msg}`); } _updateJob(status, progress) { if (this.jobManager && this.job) { try { this.jobManager.updateJob(this.job.jobId, { status, progress }); } catch {} } } } module.exports = { EvolutionLoop, DEFAULT_MAX_ITERATIONS, DEFAULT_GOAL_SCORE };