import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import fs from 'fs'; const { Orchestrator } = require('../../core/orchestrator'); const { Prompts } = require('../../core/prompts'); const { ResultTracker } = require('../../core/result-tracker'); // 禁止 waitForGlobalLogManager 阻塞 vi.mock('../../core/logging', () => ({ getGlobalLogManager: () => ({ initialized: false, getOrCreateSession: () => null, }), waitForGlobalLogManager: () => Promise.resolve(), })); describe('Orchestrator', () => { let orchestrator; let mockManager; let readdirSyncSpy; const testConfig = { agents: { master: { type: 'executor', prompt: 'agents/master.md' }, coder: { type: 'executor', prompt: 'agents/coder.md' }, verifier: { type: 'validator', prompt: 'agents/verifier.md', dependsOn: ['coder'] }, tester: { type: 'validator', prompt: 'agents/tester.md', dependsOn: ['coder'] } } }; beforeEach(() => { orchestrator = new Orchestrator(testConfig, '/tmp/test'); mockManager = { callAgent: vi.fn() }; orchestrator.manager = mockManager; // 使用 spyOn 确保 CJS require 的 fs 也被 mock readdirSyncSpy = vi.spyOn(fs, 'readdirSync').mockReturnValue([]); }); afterEach(() => { vi.restoreAllMocks(); }); describe('parseCalls', () => { it('should delegate to Parser', () => { const output = '[CALL: coder]\ntask\n[/CALL]'; const calls = orchestrator.parseCalls(output); expect(calls).toHaveLength(1); expect(calls[0].agent).toBe('coder'); }); }); describe('parseResult', () => { it('should delegate to Parser', () => { const output = '[RESULT: PASS]\nok\n[/RESULT]'; const result = orchestrator.parseResult(output); expect(result.status).toBe('PASS'); }); }); describe('executeWithDependencies', () => { it('should execute agents in dependency order', async () => { const calls = [ { agent: 'coder', task: 'write code' }, { agent: 'verifier', task: 'verify' } ]; mockManager.callAgent .mockResolvedValueOnce({ output: '[RESULT: DONE]\ncode written\n[/RESULT]' }) .mockResolvedValueOnce({ output: '[RESULT: PASS]\nverified\n[/RESULT]' }); const results = await orchestrator.executeWithDependencies(calls, 1); expect(results).toHaveLength(2); expect(results[0].agent).toBe('coder'); expect(results[0].status).toBe('DONE'); expect(results[1].agent).toBe('verifier'); expect(results[1].status).toBe('PASS'); expect(mockManager.callAgent.mock.calls[0][0]).toBe('coder'); }); it('should execute independent agents in parallel', async () => { const calls = [ { agent: 'coder', task: 'write code' }, { agent: 'verifier', task: 'verify' }, { agent: 'tester', task: 'test' } ]; const callOrder = []; mockManager.callAgent.mockImplementation(async (name) => { callOrder.push(name); await new Promise(r => setTimeout(r, 10)); if (name === 'coder') return { output: '[RESULT: DONE]\ndone\n[/RESULT]' }; return { output: '[RESULT: PASS]\npass\n[/RESULT]' }; }); const results = await orchestrator.executeWithDependencies(calls, 1); expect(results).toHaveLength(3); expect(callOrder[0]).toBe('coder'); expect(callOrder.slice(1).sort()).toEqual(['tester', 'verifier']); }); it('should auto-retry agent with format correction on missing RESULT marker', async () => { const calls = [{ agent: 'verifier', task: 'verify code' }]; // 第一次返回无 RESULT 标记 mockManager.callAgent .mockResolvedValueOnce({ output: '代码看起来不错，通过验证。' }) // 第二次返回正确格式 .mockResolvedValueOnce({ output: '[RESULT: PASS]\n验证通过\n[/RESULT]' }); const results = await orchestrator.executeWithDependencies(calls, 1); expect(results).toHaveLength(1); expect(results[0].status).toBe('PASS'); expect(mockManager.callAgent).toHaveBeenCalledTimes(2); // 第二次调用的 task 应包含格式纠正提示 expect(mockManager.callAgent.mock.calls[1][1]).toContain('系统格式纠正'); }); it('should not make extra call after format retries exhausted (1.1 fix)', async () => { const calls = [{ agent: 'coder', task: 'write code' }]; // 所有调用都返回无 RESULT 标记 mockManager.callAgent.mockResolvedValue({ output: '没有任何标记的输出' }); const results = await orchestrator.executeWithDependencies(calls, 1); expect(results).toHaveLength(1); expect(results[0].status).toBe('ERROR'); // 应该调用 MAX_FORMAT_RETRIES + 1 = 3 次（首次 + 2次重试），不是 4 次 expect(mockManager.callAgent).toHaveBeenCalledTimes(3); }); it('should degrade single agent failure without crashing the group (1.2 fix)', async () => { const calls = [ { agent: 'coder', task: 'write code' }, { agent: 'verifier', task: 'verify' }, { agent: 'tester', task: 'test' } ]; const callOrder = []; mockManager.callAgent.mockImplementation(async (name) => { callOrder.push(name); await new Promise(r => setTimeout(r, 10)); if (name === 'coder') return { output: '[RESULT: DONE]\ndone\n[/RESULT]' }; if (name === 'verifier') throw new Error('Network timeout'); return { output: '[RESULT: PASS]\npass\n[/RESULT]' }; }); const results = await orchestrator.executeWithDependencies(calls, 1); // 所有 Agent 结果都应被收集（包括异常的降级为 ERROR） expect(results).toHaveLength(3); expect(results.find(r => r.agent === 'coder').status).toBe('DONE'); expect(results.find(r => r.agent === 'verifier').status).toBe('ERROR'); expect(results.find(r => r.agent === 'verifier').message).toContain('Network timeout'); expect(results.find(r => r.agent === 'tester').status).toBe('PASS'); }); }); describe('ResultTracker', () => { it('should track cumulative results', () => { const tracker = new ResultTracker(testConfig.agents); tracker.merge([{ agent: 'coder', status: 'DONE', message: 'ok' }]); expect(tracker.getAll()).toHaveLength(1); expect(tracker.isComplete()).toBe(false); tracker.merge([{ agent: 'verifier', status: 'PASS', message: 'ok' }]); expect(tracker.getAll()).toHaveLength(2); expect(tracker.isComplete()).toBe(false); // tester 还没通过 tracker.merge([{ agent: 'tester', status: 'PASS', message: 'ok' }]); expect(tracker.isComplete()).toBe(true); }); it('should report passed/unpassed validators', () => { const tracker = new ResultTracker(testConfig.agents); tracker.merge([{ agent: 'verifier', status: 'PASS', message: 'ok' }]); expect(tracker.passedValidators).toEqual(['verifier']); expect(tracker.unpassedValidators).toEqual(['tester']); }); it('should overwrite same agent with latest result', () => { const tracker = new ResultTracker(testConfig.agents); tracker.merge([{ agent: 'verifier', status: 'FAIL', message: 'bad' }]); tracker.merge([{ agent: 'verifier', status: 'PASS', message: 'fixed' }]); expect(tracker.getAll()).toHaveLength(1); expect(tracker.getAll()[0].status).toBe('PASS'); }); it('should warn on status degradation (3.4 state machine)', () => { const tracker = new ResultTracker(testConfig.agents); const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { }); // executor: DONE → ERROR = 退化 tracker.merge([{ agent: 'coder', status: 'DONE', message: 'ok' }]); tracker.merge([{ agent: 'coder', status: 'ERROR', message: 'oops' }]); expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('状态退化')); // 应该仍然允许覆盖 expect(tracker.getAll().find(r => r.agent === 'coder').status).toBe('ERROR'); warnSpy.mockRestore(); }); it('should allow normal state transitions without warning', () => { const tracker = new ResultTracker(testConfig.agents); const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => { }); // FAIL → PASS = 正常修复链路，不应 warn tracker.merge([{ agent: 'verifier', status: 'FAIL', message: 'bad' }]); tracker.merge([{ agent: 'verifier', status: 'PASS', message: 'fixed' }]); // 排除不匹配的 warn const tracker_warns = warnSpy.mock.calls.filter(c => c[0]?.includes?.('状态退化')); expect(tracker_warns).toHaveLength(0); warnSpy.mockRestore(); }); it('should record history trail (3.4 history)', () => { const tracker = new ResultTracker(testConfig.agents); tracker.merge([{ agent: 'coder', status: 'DONE', message: 'ok' }], 1); tracker.merge([{ agent: 'verifier', status: 'FAIL', message: 'bad' }], 1); tracker.merge([{ agent: 'verifier', status: 'PASS', message: 'fixed' }], 2); const allHistory = tracker.getHistory(); expect(allHistory).toHaveLength(3); const verifierHistory = tracker.getHistory('verifier'); expect(verifierHistory).toHaveLength(2); expect(verifierHistory[0].oldStatus).toBeNull(); expect(verifierHistory[0].newStatus).toBe('FAIL'); expect(verifierHistory[1].oldStatus).toBe('FAIL'); expect(verifierHistory[1].newStatus).toBe('PASS'); expect(verifierHistory[1].loop).toBe(2); }); }); describe('Prompts', () => { it('followup should include validation status', () => { const prompt = Prompts.followup({ currentResults: [ { agent: 'coder', status: 'DONE', message: 'created file' }, { agent: 'verifier', status: 'FAIL', message: 'test failed' }, ], passedValidators: [], unpassedValidators: ['verifier', 'tester'], }); expect(prompt).toContain('coder'); expect(prompt).toContain('DONE'); expect(prompt).toContain('verifier'); expect(prompt).toContain('FAIL'); expect(prompt).toContain('test failed'); // 当前实现使用 "Sub Agent 执行结果" 标题 expect(prompt).toContain('Sub Agent 执行结果'); }); it('followup should group results by agent', () => { const prompt = Prompts.followup({ currentResults: [ { agent: 'coder', status: 'DONE', message: 'ok' }, { agent: 'verifier', status: 'PASS', message: 'pass' }, { agent: 'tester', status: 'FAIL', message: 'fail' }, ], passedValidators: ['verifier'], unpassedValidators: ['tester'], }); // 每个 agent 都有对应的结果块 expect(prompt).toContain('--- coder ---'); expect(prompt).toContain('--- verifier ---'); expect(prompt).toContain('--- tester ---'); }); it('followup should include reflection instructions', () => { const prompt = Prompts.followup({ currentResults: [{ agent: 'coder', status: 'DONE', message: 'ok' }], passedValidators: [], unpassedValidators: ['verifier'], }); expect(prompt).toContain('Reflection'); expect(prompt).toContain('CALL'); expect(prompt).toContain('RESULT'); }); }); describe('EventEmitter (3.3 progress events)', () => { it('should emit progress during execution', async () => { const events = []; orchestrator.on('progress', (data) => events.push(data)); // 简单三轮：coder -> verifier -> tester + Master Reflection mockManager.callAgent .mockResolvedValueOnce({ output: '[CALL: coder]\nwrite\n[/CALL]' }) .mockResolvedValueOnce({ output: '[RESULT: DONE]\nok\n[/RESULT]' }) .mockResolvedValueOnce({ output: '[CALL: verifier]\nverify\n[/CALL]' }) .mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }) .mockResolvedValueOnce({ output: '[CALL: tester]\ntest\n[/CALL]' }) .mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }) // Master Reflection 轮（无 CALL，触发完成退出） .mockResolvedValueOnce({ output: '✅ 任务完成\n[RESULT: DONE]完成[/RESULT]' }); const result = await orchestrator.execute('task'); expect(events.length).toBeGreaterThanOrEqual(2); expect(events[0].loop).toBe(1); expect(events[0].results).toEqual([{ agent: 'coder', status: 'DONE' }]); }); it('should return completed result on success', async () => { mockManager.callAgent .mockResolvedValueOnce({ output: '[CALL: coder]\nwrite\n[/CALL]' }) .mockResolvedValueOnce({ output: '[RESULT: DONE]\nok\n[/RESULT]' }) .mockResolvedValueOnce({ output: '[CALL: verifier]\nverify\n[/CALL]' }) .mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }) .mockResolvedValueOnce({ output: '[CALL: tester]\ntest\n[/CALL]' }) .mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }) // Master Reflection .mockResolvedValueOnce({ output: '✅ 任务完成\n[RESULT: DONE]完成[/RESULT]' }); const result = await orchestrator.execute('task'); expect(result.completed).toBe(true); expect(result.results.length).toBe(3); }); }); describe('detectMasterOverreach', () => { it('should detect new files without baseline', () => { vi.mocked(fs.readdirSync).mockReturnValue(['file1.js', '.hidden', 'file2.py']); const files = orchestrator.detectMasterOverreach(); expect(files).toEqual(['file1.js', 'file2.py']); }); it('should skip pre-existing files with baseline (2.2 fix)', () => { vi.mocked(fs.readdirSync).mockReturnValue(['existing.js', 'new-file.py', '.hidden']); const baseline = new Set(['existing.js', '.hidden']); const files = orchestrator.detectMasterOverreach(baseline); // 只报告新增的非隐藏文件 expect(files).toEqual(['new-file.py']); }); it('should return empty when no new files with baseline', () => { vi.mocked(fs.readdirSync).mockReturnValue(['existing.js']); const baseline = new Set(['existing.js']); const files = orchestrator.detectMasterOverreach(baseline); expect(files).toEqual([]); }); }); describe('execute', () => { it('should complete when no agent calls and all validators previously passed', async () => { // 第一轮：master 调用 coder mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: coder]\nwrite code\n[/CALL]' }); // coder 完成 mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: DONE]\ncreated file\n[/RESULT]' }); // 第二轮：master 调用 verifier mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: verifier]\nverify\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }); // 第三轮：master 调 tester mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: tester]\ntest\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nall tests pass\n[/RESULT]' }); // Master Reflection 轮 mockManager.callAgent.mockResolvedValueOnce({ output: '✅ 任务完成\n[RESULT: DONE]完成[/RESULT]' }); const result = await orchestrator.execute('simple task'); expect(result.completed).toBe(true); }); it('should detect master overreach when files exist without CALL', async () => { // Mock：workDir 有文件（Master 越权创建的） vi.mocked(fs.readdirSync).mockReturnValue(['calculator.py', 'test_calculator.py']); // 第一轮：master 不输出 CALL（越权直接写了代码） mockManager.callAgent.mockResolvedValueOnce({ output: '我已经完成了计算器的实现' }); // 第二轮：系统强制要求调用 validator mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: verifier]\nverify calculator\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }); // 还需要 tester 通过 mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: tester]\ntest calculator\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nall pass\n[/RESULT]' }); // Master Reflection 轮 mockManager.callAgent.mockResolvedValueOnce({ output: '✅ 任务完成\n[RESULT: DONE]完成[/RESULT]' }); const result = await orchestrator.execute('implement calculator'); expect(result.completed).toBe(true); }); it('should iterate until validation passes', async () => { // 第一轮：master 调用 coder mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: coder]\nwrite add function\n[/CALL]' }); // coder 完成 mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: DONE]\ncreated add.js\n[/RESULT]' }); // 第二轮：master 调用 verifier mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: verifier]\nverify\n[/CALL]' }); // verifier 失败 mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: FAIL]\nadd(1,2) returns null\n[/RESULT]' }); // 第三轮：master 调用 coder 修复 mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: coder]\nfix add function\n[/CALL]' }); // coder 修复完成 mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: DONE]\nfixed add.js\n[/RESULT]' }); // 第四轮：master 调用 verifier mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: verifier]\nverify again\n[/CALL]' }); // verifier 通过 mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nall tests pass\n[/RESULT]' }); // 第五轮：master 调用 tester mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: tester]\nrun tests\n[/CALL]' }); // tester 通过 mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nall tests pass\n[/RESULT]' }); // Master Reflection 轮 mockManager.callAgent.mockResolvedValueOnce({ output: '✅ 任务完成\n[RESULT: DONE]完成[/RESULT]' }); const result = await orchestrator.execute('implement add function'); expect(result.completed).toBe(true); expect(result.iterations).toBe(6); // 5 original + 1 Master Reflection }); it('should stop at max repairs', async () => { // 每轮都是：master CALL coder → coder DONE → master CALL verifier → verifier FAIL let callCount = 0; mockManager.callAgent.mockImplementation(async (name) => { callCount++; if (name === 'master') { // 奇数轮调 coder，偶数轮调 verifier if (callCount % 4 === 1) return { output: '[CALL: coder]\nwrite\n[/CALL]' }; return { output: '[CALL: verifier]\nverify\n[/CALL]' }; } if (name === 'coder') return { output: '[RESULT: DONE]\nok\n[/RESULT]' }; // verifier 始终 FAIL return { output: '[RESULT: FAIL]\nbad\n[/RESULT]' }; }); const result = await orchestrator.execute('impossible task', { maxIterations: 2 }); expect(result.completed).toBe(false); expect(result.repairs).toBe(2); }); it('should save checkpoint on exception (1.3 fix)', async () => { const checkpoints = []; const onCheckpoint = vi.fn(async (cp) => checkpoints.push(cp)); // Master 调用抛出不可恢复的异常（在 allSettled 外面） mockManager.callAgent.mockRejectedValueOnce(new Error('Fatal auth error')); await expect( orchestrator.execute('task that crashes', { onCheckpoint }) ).rejects.toThrow('Fatal auth error'); // 应该在异常时保存了 checkpoint expect(onCheckpoint).toHaveBeenCalled(); const lastCheckpoint = checkpoints[checkpoints.length - 1]; expect(lastCheckpoint).toBeDefined(); expect(lastCheckpoint.error).toBe('Fatal auth error'); expect(lastCheckpoint.requirement).toBe('task that crashes'); }); it('should include requirement in checkpoint payload (1.3 fix)', async () => { const checkpoints = []; const onCheckpoint = vi.fn(async (cp) => checkpoints.push(cp)); // 第一轮执行正常，触发 checkpoint mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: coder]\nwrite\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: DONE]\nok\n[/RESULT]' }); // 第二轮继续（验证未全通过） mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: verifier]\nverify\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }); // 继续 mockManager.callAgent.mockResolvedValueOnce({ output: '[CALL: tester]\ntest\n[/CALL]' }); mockManager.callAgent.mockResolvedValueOnce({ output: '[RESULT: PASS]\nok\n[/RESULT]' }); // Master Reflection 轮 mockManager.callAgent.mockResolvedValueOnce({ output: '✅ 任务完成\n[RESULT: DONE]完成[/RESULT]' }); await orchestrator.execute('my requirement', { onCheckpoint }); // 所有 checkpoint 都应包含 requirement for (const cp of checkpoints) { expect(cp.requirement).toBe('my requirement'); } }); }); });