diff --git a/scripts/test-harness.js b/scripts/test-harness.js new file mode 100644 index 0000000..4f93906 --- /dev/null +++ b/scripts/test-harness.js @@ -0,0 +1,376 @@ +#!/usr/bin/env node +/** + * test-harness.js — Harness Engineering 四阶段流转手工测试脚本 + * + * 测试覆盖: + * ✅ Test 1: Meta-Skill 文件存在性检查 + * ✅ Test 2: Base Agent Template 完整性检查 + * ✅ Test 3: Task Transition API — Hot Swap 换脑验证 + * ✅ Test 4: Task Transition API — Context Flush 记忆断舍离 + * ✅ Test 5: P0 事前阻断阀 (Human-in-the-loop) + * ✅ Test 6: _lessons.md 注入 System Prompt + * ✅ Test 7: billing.log 计费日志 + * ✅ Test 8: OKR GoalsService 新增状态兼容 + * ✅ Test 9: CLI `task transition` 命令注册 + * + * 用法: + * node scripts/test-harness.js # 运行全部离线测试 (不需要服务器) + * node scripts/test-harness.js --online # 运行含 API 调用的在线测试 (需要服务器在 3000 端口) + */ + +const fs = require('fs'); +const path = require('path'); +const http = require('http'); + +const ROOT = path.resolve(__dirname, '..'); +const DATA = path.join(ROOT, 'data'); +const API_BASE = 'http://localhost:3000/api'; + +let passed = 0; +let failed = 0; +const results = []; + +function assert(condition, testName, detail = '') { + if (condition) { + passed++; + results.push({ status: '✅', testName }); + console.log(` ✅ ${testName}`); + } else { + failed++; + results.push({ status: '❌', testName, detail }); + console.log(` ❌ ${testName}${detail ? ` — ${detail}` : ''}`); + } +} + +function fileExists(relPath) { + return fs.existsSync(path.join(ROOT, relPath)); +} + +function fileContains(relPath, needle) { + if (!fileExists(relPath)) return false; + return fs.readFileSync(path.join(ROOT, relPath), 'utf-8').includes(needle); +} + +function httpPost(urlPath, body) { + return new Promise((resolve, reject) => { + const postData = JSON.stringify(body); + const url = new URL(API_BASE + urlPath); + const req = http.request({ + hostname: url.hostname, + port: url.port, + path: url.pathname, + method: 'POST', + headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(postData) }, + }, (res) => { + let data = ''; + res.on('data', c => data += c); + res.on('end', () => { + try { resolve({ status: res.statusCode, body: JSON.parse(data) }); } + catch { resolve({ status: res.statusCode, body: data }); } + }); + }); + req.on('error', reject); + req.write(postData); + req.end(); + }); +} + +// ═══════════════════════════════════════ +// Test 1: Meta-Skill 文件存在性 +// ═══════════════════════════════════════ +function test1_metaSkillFiles() { + console.log('\n📦 Test 1: Meta-Skill 文件存在性'); + const phases = ['meta-research', 'meta-plan', 'meta-execute', 'meta-verify']; + for (const phase of phases) { + const skillPath = `data/system/skills/${phase}/SKILL.md`; + assert(fileExists(skillPath), `${phase}/SKILL.md 存在`, `缺失: ${skillPath}`); + } + // 内容校验 + assert( + fileContains('data/system/skills/meta-research/SKILL.md', '调研'), + 'meta-research 包含"调研"关键词' + ); + assert( + fileContains('data/system/skills/meta-execute/SKILL.md', '服从纪律'), + 'meta-execute 包含"服从纪律"约束' + ); + assert( + fileContains('data/system/skills/meta-verify/SKILL.md', '_lessons.md'), + 'meta-verify 引用 _lessons.md 防坑机制' + ); +} + +// ═══════════════════════════════════════ +// Test 2: Base Agent Template 完整性 +// ═══════════════════════════════════════ +function test2_baseTemplate() { + console.log('\n📦 Test 2: Base Agent Template 完整性'); + assert(fileExists('data/system/templates/base/agent.md'), 'base/agent.md 存在'); + assert(fileExists('data/system/templates/base/config.json'), 'base/config.json 存在'); + + // config.json 校验工具策略 + const config = JSON.parse(fs.readFileSync(path.join(ROOT, 'data/system/templates/base/config.json'), 'utf-8')); + assert(config.cli?.toolPolicy?.mode === 'restricted', 'toolPolicy.mode = restricted'); + assert( + Array.isArray(config.cli?.toolPolicy?.allowedTools) && config.cli.toolPolicy.allowedTools.includes('bash'), + 'allowedTools 包含 bash' + ); + + // agent.md 校验四阶段规范 + assert( + fileContains('data/system/templates/base/agent.md', 'Research') && + fileContains('data/system/templates/base/agent.md', 'Plan') && + fileContains('data/system/templates/base/agent.md', 'Execute') && + fileContains('data/system/templates/base/agent.md', 'Verify'), + 'agent.md 包含 RPEV 四阶段关键词' + ); + assert( + fileContains('data/system/templates/base/agent.md', 'task transition'), + 'agent.md 引用 task transition CLI 命令' + ); + + // _registry.json 校验 + assert(fileExists('data/system/templates/_registry.json'), '_registry.json 存在'); + const registry = JSON.parse(fs.readFileSync(path.join(ROOT, 'data/system/templates/_registry.json'), 'utf-8')); + assert(registry.some(t => t.id === 'base'), '_registry.json 包含 base 模板'); +} + +// ═══════════════════════════════════════ +// Test 3: Hot Swap 换脑 — 模拟文件替换 +// ═══════════════════════════════════════ +function test3_hotSwap() { + console.log('\n🧠 Test 3: Hot Swap 模拟换脑'); + + // 创建临时测试 Agent 目录 + const testAgentDir = path.join(DATA, 'agents', '_test_harness_agent'); + const skillDir = path.join(testAgentDir, '.claude', 'skills', 'main_skill'); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), '# Original Skill\nI am the original.', 'utf-8'); + fs.writeFileSync(path.join(testAgentDir, 'config.json'), JSON.stringify({ id: '_test_harness_agent', type: 'skill', status: 'idle' }), 'utf-8'); + fs.writeFileSync(path.join(testAgentDir, 'identity.md'), '**名称:** 测试小兵\n**Emoji:** 🧪', 'utf-8'); + + // 模拟 Hot Swap: 复制 meta-plan 到 main_skill + const sourceSkill = path.join(ROOT, 'data/system/skills/meta-plan/SKILL.md'); + if (fs.existsSync(sourceSkill)) { + fs.copyFileSync(sourceSkill, path.join(skillDir, 'SKILL.md')); + const content = fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf-8'); + assert(content.includes('meta-plan'), 'Hot Swap 后 SKILL.md 内容变为 meta-plan'); + assert(!content.includes('Original Skill'), 'Hot Swap 后原始内容已被覆盖'); + } else { + assert(false, 'Hot Swap 测试', '源技能文件 meta-plan/SKILL.md 不存在'); + } + + // 清理 + fs.rmSync(testAgentDir, { recursive: true, force: true }); +} + +// ═══════════════════════════════════════ +// Test 4: Context Flush 历史清空模拟 +// ═══════════════════════════════════════ +function test4_contextFlush() { + console.log('\n🛑 Test 4: Context Flush 记忆断舍离模拟'); + + const testAgentDir = path.join(DATA, 'agents', '_test_flush_agent'); + const convDir = path.join(testAgentDir, 'conversations', 'test-conv-001'); + fs.mkdirSync(convDir, { recursive: true }); + + // 模拟 history.json 有大量历史 + const fakeHistory = Array.from({ length: 50 }, (_, i) => ({ + role: i % 2 === 0 ? 'user' : 'assistant', + content: `Message ${i}: ${'x'.repeat(200)}`, + timestamp: new Date().toISOString() + })); + fs.writeFileSync(path.join(convDir, 'history.json'), JSON.stringify(fakeHistory), 'utf-8'); + + assert(JSON.parse(fs.readFileSync(path.join(convDir, 'history.json'), 'utf-8')).length === 50, + '预置 50 条历史消息'); + + // 模拟 Context Flush: 清空 history + fs.writeFileSync(path.join(convDir, 'history.json'), '[]', 'utf-8'); + const afterFlush = JSON.parse(fs.readFileSync(path.join(convDir, 'history.json'), 'utf-8')); + assert(afterFlush.length === 0, 'Context Flush 后 history.json 被清空为空数组'); + + // 模拟注入新起步信息 + afterFlush.push({ + role: 'user', + content: '【系统指令:强制状态切换】你已进入 PLAN 阶段。', + messageId: 'msg-flush-test', + timestamp: new Date().toISOString() + }); + fs.writeFileSync(path.join(convDir, 'history.json'), JSON.stringify(afterFlush), 'utf-8'); + const restored = JSON.parse(fs.readFileSync(path.join(convDir, 'history.json'), 'utf-8')); + assert(restored.length === 1 && restored[0].content.includes('PLAN'), '注入新阶段起步指令成功'); + + // 清理 + fs.rmSync(testAgentDir, { recursive: true, force: true }); +} + +// ═══════════════════════════════════════ +// Test 5: _lessons.md 注入 System Prompt +// ═══════════════════════════════════════ +function test5_lessonsInjection() { + console.log('\n⚠️ Test 5: _lessons.md 防坑注入'); + + // 模拟 buildSystemPrompt 中 _lessons.md 的读取逻辑 + const testAgentDir = path.join(DATA, 'agents', '_test_lessons_agent'); + const wsDir = path.join(testAgentDir, 'workspace'); + fs.mkdirSync(wsDir, { recursive: true }); + fs.writeFileSync(path.join(testAgentDir, 'config.json'), JSON.stringify({ id: '_test_lessons_agent', type: 'skill' }), 'utf-8'); + + // 无 _lessons.md 时 + const lessonsPath = path.join(wsDir, '_lessons.md'); + assert(!fs.existsSync(lessonsPath), '初始状态无 _lessons.md'); + + // 写入教训 + fs.writeFileSync(lessonsPath, `# 试错记录 +- 2026-03-26: Python2 语法在 node 22 环境下不兼容, 必须用 python3 +- 2026-03-26: 不要用 fs.existsSync 检查远程 URL, 应该用 http.get +`, 'utf-8'); + + const content = fs.readFileSync(lessonsPath, 'utf-8'); + assert(content.includes('python3'), '_lessons.md 包含真实教训'); + assert(content.includes('fs.existsSync'), '_lessons.md 包含第二条防坑记录'); + + // 清理 + fs.rmSync(testAgentDir, { recursive: true, force: true }); +} + +// ═══════════════════════════════════════ +// Test 6: billing.log 计费格式 +// ═══════════════════════════════════════ +function test6_billingLog() { + console.log('\n💰 Test 6: billing.log 计费日志'); + + const logsDir = path.join(DATA, 'logs'); + if (!fs.existsSync(logsDir)) fs.mkdirSync(logsDir, { recursive: true }); + const billingPath = path.join(logsDir, 'billing.log'); + + // 模拟写入一条计费记录 + const testEntry = `${new Date().toISOString()} | Agent: _test_billing | Conv: test-conv | Cost: $0.003200 | Duration: 4521ms\n`; + fs.appendFileSync(billingPath, testEntry, 'utf-8'); + + const content = fs.readFileSync(billingPath, 'utf-8'); + assert(content.includes('_test_billing'), 'billing.log 包含测试 Agent ID'); + assert(content.includes('Cost: $'), 'billing.log 包含美元成本字段'); + assert(content.includes('Duration:'), 'billing.log 包含耗时字段'); + + // 清理测试行(不删文件,只清测试数据) + const cleanedContent = content.split('\n').filter(l => !l.includes('_test_billing')).join('\n'); + fs.writeFileSync(billingPath, cleanedContent, 'utf-8'); +} + +// ═══════════════════════════════════════ +// Test 7: GoalsService 新增状态兼容 +// ═══════════════════════════════════════ +function test7_goalsServiceStates() { + console.log('\n🎯 Test 7: GoalsService 新增状态兼容'); + + const { GoalsService } = require(path.join(ROOT, 'server/services/goals.js')); + const svc = new GoalsService(); // 使用全局 goals dir + + // 验证源码中有新状态 + const goalsSource = fs.readFileSync(path.join(ROOT, 'server/services/goals.js'), 'utf-8'); + assert(goalsSource.includes('researching'), 'GoalsService 源码包含 researching 状态'); + assert(goalsSource.includes('planning'), 'GoalsService 源码包含 planning 状态'); + assert(goalsSource.includes('executing'), 'GoalsService 源码包含 executing 状态'); + assert(goalsSource.includes('verifying'), 'GoalsService 源码包含 verifying 状态'); + assert(goalsSource.includes('pending_approval'), 'GoalsService 源码包含 pending_approval 状态'); +} + +// ═══════════════════════════════════════ +// Test 8: CLI task transition 命令注册 +// ═══════════════════════════════════════ +function test8_cliRegistration() { + console.log('\n🔧 Test 8: CLI task transition 命令注册'); + + const lemonSrc = fs.readFileSync(path.join(ROOT, 'bin/lemon.js'), 'utf-8'); + assert(lemonSrc.includes("'task transition'"), "bin/lemon.js 注册了 'task transition' 命令"); + assert(lemonSrc.includes("phase: { type: 'string'"), 'task transition 接受 phase 参数'); + assert(lemonSrc.includes("artifact: { type: 'string'"), 'task transition 接受 artifact 参数'); + assert(lemonSrc.includes("agentId: { type: 'string'"), 'task transition 接受 agentId 参数'); +} + +// ═══════════════════════════════════════ +// Test 9: agent-chat.js Transition 路由注册 +// ═══════════════════════════════════════ +function test9_transitionRoute() { + console.log('\n🌐 Test 9: Transition API 路由注册'); + + const routeSrc = fs.readFileSync(path.join(ROOT, 'server/routes/agent-chat.js'), 'utf-8'); + assert(routeSrc.includes("/:agentId/task/transition"), '存在 /:agentId/task/transition 路由'); + assert(routeSrc.includes('Hot Swap'), '路由包含 Hot Swap 换脑逻辑'); + assert(routeSrc.includes('Context Flush'), '路由包含 Context Flush 逻辑'); + assert(routeSrc.includes('pending_approval'), '路由包含 P0 审批拦截逻辑'); + assert(routeSrc.includes('billing.log'), '路由包含 billing.log 计费写入'); + assert(routeSrc.includes('_lessons.md'), '路由包含 _lessons.md 注入'); +} + +// ═══════════════════════════════════════ +// Test 10 (Online): 真实 API 调用测试 +// ═══════════════════════════════════════ +async function test10_onlineAPI() { + console.log('\n🌍 Test 10: 在线 API 调用测试 (需要服务器)'); + + try { + // 10a: 调用 transition API (对一个不存在的 Agent 应返回 404) + const res1 = await httpPost('/agent/_nonexistent_agent/task/transition', { phase: 'plan' }); + assert(res1.status === 404, 'Transition API 对不存在的 Agent 返回 404'); + + // 10b: 如果 master Agent 存在, 尝试对它做一次 research -> plan 的切换 + const res2 = await httpPost('/agent/master/task/transition', { phase: 'plan', artifact: '' }); + assert(res2.status === 200 && res2.body?.success === true, 'Master Agent transition(plan) 返回 success'); + + // 10c: 检查 master 的 main_skill 是否被换成了 meta-plan + const masterSkillPath = path.join(ROOT, 'data/system/agents/master/.claude/skills/main_skill/SKILL.md'); + if (fs.existsSync(masterSkillPath)) { + const content = fs.readFileSync(masterSkillPath, 'utf-8'); + assert(content.includes('meta-plan'), 'Master 的 main_skill 已被换为 meta-plan'); + } else { + assert(false, 'Master 的 main_skill/SKILL.md 不存在(可能目录结构不同)'); + } + } catch (err) { + assert(false, '在线 API 测试', `服务器连接失败: ${err.message}. 请先启动服务器 (node server/index.js)`); + } +} + +// ═══════════════════════════════════════ +// Runner +// ═══════════════════════════════════════ +async function main() { + console.log('═══════════════════════════════════════'); + console.log(' 🏗️ Harness Engineering 测试套件'); + console.log('═══════════════════════════════════════'); + + const isOnline = process.argv.includes('--online'); + + test1_metaSkillFiles(); + test2_baseTemplate(); + test3_hotSwap(); + test4_contextFlush(); + test5_lessonsInjection(); + test6_billingLog(); + test7_goalsServiceStates(); + test8_cliRegistration(); + test9_transitionRoute(); + + if (isOnline) { + await test10_onlineAPI(); + } else { + console.log('\n⏭️ 跳过在线测试 (使用 --online 参数启用)'); + } + + // 结果汇总 + console.log('\n═══════════════════════════════════════'); + console.log(` 📊 测试结果: ${passed} 通过, ${failed} 失败, 共 ${passed + failed} 项`); + if (failed > 0) { + console.log('\n 失败项:'); + results.filter(r => r.status === '❌').forEach(r => { + console.log(` ❌ ${r.testName}: ${r.detail}`); + }); + } + console.log('═══════════════════════════════════════\n'); + + process.exit(failed > 0 ? 1 : 0); +} + +main();