宜博 / lemonboss

Commit 483c132dcbe6a05f38853865fa65756d9f36f9af

Authored by yibo
1 parent 1037ab12

test: add comprehensive Harness Engineering test suite (41 assertions) 

scripts/test-harness.js covers:
- Meta-Skill file existence & content
- Base Agent template completeness
- Hot Swap simulation
- Context Flush simulation
- _lessons.md injection
- billing.log format
- GoalsService new statuses
- CLI registration
- Transition API route verification
- Optional --online mode for live API tests
Inline Side-by-side
Showing 1 changed file with 376 additions and 0 deletions
scripts/test-harness.js 0 → 100644
  View file @483c132
  1 +#!/usr/bin/env node
  2 +/**
  3 + * test-harness.js — Harness Engineering 四阶段流转手工测试脚本
  4 + *
  5 + * 测试覆盖:
  6 + * ✅ Test 1: Meta-Skill 文件存在性检查
  7 + * ✅ Test 2: Base Agent Template 完整性检查
  8 + * ✅ Test 3: Task Transition API — Hot Swap 换脑验证
  9 + * ✅ Test 4: Task Transition API — Context Flush 记忆断舍离
  10 + * ✅ Test 5: P0 事前阻断阀 (Human-in-the-loop)
  11 + * ✅ Test 6: _lessons.md 注入 System Prompt
  12 + * ✅ Test 7: billing.log 计费日志
  13 + * ✅ Test 8: OKR GoalsService 新增状态兼容
  14 + * ✅ Test 9: CLI `task transition` 命令注册
  15 + *
  16 + * 用法:
  17 + * node scripts/test-harness.js # 运行全部离线测试 (不需要服务器)
  18 + * node scripts/test-harness.js --online # 运行含 API 调用的在线测试 (需要服务器在 3000 端口)
  19 + */
  20 +
  21 +const fs = require('fs');
  22 +const path = require('path');
  23 +const http = require('http');
  24 +
  25 +const ROOT = path.resolve(__dirname, '..');
  26 +const DATA = path.join(ROOT, 'data');
  27 +const API_BASE = 'http://localhost:3000/api';
  28 +
  29 +let passed = 0;
  30 +let failed = 0;
  31 +const results = [];
  32 +
  33 +function assert(condition, testName, detail = '') {
  34 + if (condition) {
  35 + passed++;
  36 + results.push({ status: '✅', testName });
  37 + console.log(` ✅ ${testName}`);
  38 + } else {
  39 + failed++;
  40 + results.push({ status: '❌', testName, detail });
  41 + console.log(` ❌ ${testName}${detail ? ` — ${detail}` : ''}`);
  42 + }
  43 +}
  44 +
  45 +function fileExists(relPath) {
  46 + return fs.existsSync(path.join(ROOT, relPath));
  47 +}
  48 +
  49 +function fileContains(relPath, needle) {
  50 + if (!fileExists(relPath)) return false;
  51 + return fs.readFileSync(path.join(ROOT, relPath), 'utf-8').includes(needle);
  52 +}
  53 +
  54 +function httpPost(urlPath, body) {
  55 + return new Promise((resolve, reject) => {
  56 + const postData = JSON.stringify(body);
  57 + const url = new URL(API_BASE + urlPath);
  58 + const req = http.request({
  59 + hostname: url.hostname,
  60 + port: url.port,
  61 + path: url.pathname,
  62 + method: 'POST',
  63 + headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(postData) },
  64 + }, (res) => {
  65 + let data = '';
  66 + res.on('data', c => data += c);
  67 + res.on('end', () => {
  68 + try { resolve({ status: res.statusCode, body: JSON.parse(data) }); }
  69 + catch { resolve({ status: res.statusCode, body: data }); }
  70 + });
  71 + });
  72 + req.on('error', reject);
  73 + req.write(postData);
  74 + req.end();
  75 + });
  76 +}
  77 +
  78 +// ═══════════════════════════════════════
  79 +// Test 1: Meta-Skill 文件存在性
  80 +// ═══════════════════════════════════════
  81 +function test1_metaSkillFiles() {
  82 + console.log('\n📦 Test 1: Meta-Skill 文件存在性');
  83 + const phases = ['meta-research', 'meta-plan', 'meta-execute', 'meta-verify'];
  84 + for (const phase of phases) {
  85 + const skillPath = `data/system/skills/${phase}/SKILL.md`;
  86 + assert(fileExists(skillPath), `${phase}/SKILL.md 存在`, `缺失: ${skillPath}`);
  87 + }
  88 + // 内容校验
  89 + assert(
  90 + fileContains('data/system/skills/meta-research/SKILL.md', '调研'),
  91 + 'meta-research 包含"调研"关键词'
  92 + );
  93 + assert(
  94 + fileContains('data/system/skills/meta-execute/SKILL.md', '服从纪律'),
  95 + 'meta-execute 包含"服从纪律"约束'
  96 + );
  97 + assert(
  98 + fileContains('data/system/skills/meta-verify/SKILL.md', '_lessons.md'),
  99 + 'meta-verify 引用 _lessons.md 防坑机制'
  100 + );
  101 +}
  102 +
  103 +// ═══════════════════════════════════════
  104 +// Test 2: Base Agent Template 完整性
  105 +// ═══════════════════════════════════════
  106 +function test2_baseTemplate() {
  107 + console.log('\n📦 Test 2: Base Agent Template 完整性');
  108 + assert(fileExists('data/system/templates/base/agent.md'), 'base/agent.md 存在');
  109 + assert(fileExists('data/system/templates/base/config.json'), 'base/config.json 存在');
  110 +
  111 + // config.json 校验工具策略
  112 + const config = JSON.parse(fs.readFileSync(path.join(ROOT, 'data/system/templates/base/config.json'), 'utf-8'));
  113 + assert(config.cli?.toolPolicy?.mode === 'restricted', 'toolPolicy.mode = restricted');
  114 + assert(
  115 + Array.isArray(config.cli?.toolPolicy?.allowedTools) && config.cli.toolPolicy.allowedTools.includes('bash'),
  116 + 'allowedTools 包含 bash'
  117 + );
  118 +
  119 + // agent.md 校验四阶段规范
  120 + assert(
  121 + fileContains('data/system/templates/base/agent.md', 'Research') &&
  122 + fileContains('data/system/templates/base/agent.md', 'Plan') &&
  123 + fileContains('data/system/templates/base/agent.md', 'Execute') &&
  124 + fileContains('data/system/templates/base/agent.md', 'Verify'),
  125 + 'agent.md 包含 RPEV 四阶段关键词'
  126 + );
  127 + assert(
  128 + fileContains('data/system/templates/base/agent.md', 'task transition'),
  129 + 'agent.md 引用 task transition CLI 命令'
  130 + );
  131 +
  132 + // _registry.json 校验
  133 + assert(fileExists('data/system/templates/_registry.json'), '_registry.json 存在');
  134 + const registry = JSON.parse(fs.readFileSync(path.join(ROOT, 'data/system/templates/_registry.json'), 'utf-8'));
  135 + assert(registry.some(t => t.id === 'base'), '_registry.json 包含 base 模板');
  136 +}
  137 +
  138 +// ═══════════════════════════════════════
  139 +// Test 3: Hot Swap 换脑 — 模拟文件替换
  140 +// ═══════════════════════════════════════
  141 +function test3_hotSwap() {
  142 + console.log('\n🧠 Test 3: Hot Swap 模拟换脑');
  143 +
  144 + // 创建临时测试 Agent 目录
  145 + const testAgentDir = path.join(DATA, 'agents', '_test_harness_agent');
  146 + const skillDir = path.join(testAgentDir, '.claude', 'skills', 'main_skill');
  147 + fs.mkdirSync(skillDir, { recursive: true });
  148 + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), '# Original Skill\nI am the original.', 'utf-8');
  149 + fs.writeFileSync(path.join(testAgentDir, 'config.json'), JSON.stringify({ id: '_test_harness_agent', type: 'skill', status: 'idle' }), 'utf-8');
  150 + fs.writeFileSync(path.join(testAgentDir, 'identity.md'), '**名称：** 测试小兵\n**Emoji：** 🧪', 'utf-8');
  151 +
  152 + // 模拟 Hot Swap: 复制 meta-plan 到 main_skill
  153 + const sourceSkill = path.join(ROOT, 'data/system/skills/meta-plan/SKILL.md');
  154 + if (fs.existsSync(sourceSkill)) {
  155 + fs.copyFileSync(sourceSkill, path.join(skillDir, 'SKILL.md'));
  156 + const content = fs.readFileSync(path.join(skillDir, 'SKILL.md'), 'utf-8');
  157 + assert(content.includes('meta-plan'), 'Hot Swap 后 SKILL.md 内容变为 meta-plan');
  158 + assert(!content.includes('Original Skill'), 'Hot Swap 后原始内容已被覆盖');
  159 + } else {
  160 + assert(false, 'Hot Swap 测试', '源技能文件 meta-plan/SKILL.md 不存在');
  161 + }
  162 +
  163 + // 清理
  164 + fs.rmSync(testAgentDir, { recursive: true, force: true });
  165 +}
  166 +
  167 +// ═══════════════════════════════════════
  168 +// Test 4: Context Flush 历史清空模拟
  169 +// ═══════════════════════════════════════
  170 +function test4_contextFlush() {
  171 + console.log('\n🛑 Test 4: Context Flush 记忆断舍离模拟');
  172 +
  173 + const testAgentDir = path.join(DATA, 'agents', '_test_flush_agent');
  174 + const convDir = path.join(testAgentDir, 'conversations', 'test-conv-001');
  175 + fs.mkdirSync(convDir, { recursive: true });
  176 +
  177 + // 模拟 history.json 有大量历史
  178 + const fakeHistory = Array.from({ length: 50 }, (_, i) => ({
  179 + role: i % 2 === 0 ? 'user' : 'assistant',
  180 + content: `Message ${i}: ${'x'.repeat(200)}`,
  181 + timestamp: new Date().toISOString()
  182 + }));
  183 + fs.writeFileSync(path.join(convDir, 'history.json'), JSON.stringify(fakeHistory), 'utf-8');
  184 +
  185 + assert(JSON.parse(fs.readFileSync(path.join(convDir, 'history.json'), 'utf-8')).length === 50,
  186 + '预置 50 条历史消息');
  187 +
  188 + // 模拟 Context Flush: 清空 history
  189 + fs.writeFileSync(path.join(convDir, 'history.json'), '[]', 'utf-8');
  190 + const afterFlush = JSON.parse(fs.readFileSync(path.join(convDir, 'history.json'), 'utf-8'));
  191 + assert(afterFlush.length === 0, 'Context Flush 后 history.json 被清空为空数组');
  192 +
  193 + // 模拟注入新起步信息
  194 + afterFlush.push({
  195 + role: 'user',
  196 + content: '【系统指令：强制状态切换】你已进入 PLAN 阶段。',
  197 + messageId: 'msg-flush-test',
  198 + timestamp: new Date().toISOString()
  199 + });
  200 + fs.writeFileSync(path.join(convDir, 'history.json'), JSON.stringify(afterFlush), 'utf-8');
  201 + const restored = JSON.parse(fs.readFileSync(path.join(convDir, 'history.json'), 'utf-8'));
  202 + assert(restored.length === 1 && restored[0].content.includes('PLAN'), '注入新阶段起步指令成功');
  203 +
  204 + // 清理
  205 + fs.rmSync(testAgentDir, { recursive: true, force: true });
  206 +}
  207 +
  208 +// ═══════════════════════════════════════
  209 +// Test 5: _lessons.md 注入 System Prompt
  210 +// ═══════════════════════════════════════
  211 +function test5_lessonsInjection() {
  212 + console.log('\n⚠️ Test 5: _lessons.md 防坑注入');
  213 +
  214 + // 模拟 buildSystemPrompt 中 _lessons.md 的读取逻辑
  215 + const testAgentDir = path.join(DATA, 'agents', '_test_lessons_agent');
  216 + const wsDir = path.join(testAgentDir, 'workspace');
  217 + fs.mkdirSync(wsDir, { recursive: true });
  218 + fs.writeFileSync(path.join(testAgentDir, 'config.json'), JSON.stringify({ id: '_test_lessons_agent', type: 'skill' }), 'utf-8');
  219 +
  220 + // 无 _lessons.md 时
  221 + const lessonsPath = path.join(wsDir, '_lessons.md');
  222 + assert(!fs.existsSync(lessonsPath), '初始状态无 _lessons.md');
  223 +
  224 + // 写入教训
  225 + fs.writeFileSync(lessonsPath, `# 试错记录
  226 +- 2026-03-26: Python2 语法在 node 22 环境下不兼容, 必须用 python3
  227 +- 2026-03-26: 不要用 fs.existsSync 检查远程 URL, 应该用 http.get
  228 +`, 'utf-8');
  229 +
  230 + const content = fs.readFileSync(lessonsPath, 'utf-8');
  231 + assert(content.includes('python3'), '_lessons.md 包含真实教训');
  232 + assert(content.includes('fs.existsSync'), '_lessons.md 包含第二条防坑记录');
  233 +
  234 + // 清理
  235 + fs.rmSync(testAgentDir, { recursive: true, force: true });
  236 +}
  237 +
  238 +// ═══════════════════════════════════════
  239 +// Test 6: billing.log 计费格式
  240 +// ═══════════════════════════════════════
  241 +function test6_billingLog() {
  242 + console.log('\n💰 Test 6: billing.log 计费日志');
  243 +
  244 + const logsDir = path.join(DATA, 'logs');
  245 + if (!fs.existsSync(logsDir)) fs.mkdirSync(logsDir, { recursive: true });
  246 + const billingPath = path.join(logsDir, 'billing.log');
  247 +
  248 + // 模拟写入一条计费记录
  249 + const testEntry = `${new Date().toISOString()} | Agent: _test_billing | Conv: test-conv | Cost: $0.003200 | Duration: 4521ms\n`;
  250 + fs.appendFileSync(billingPath, testEntry, 'utf-8');
  251 +
  252 + const content = fs.readFileSync(billingPath, 'utf-8');
  253 + assert(content.includes('_test_billing'), 'billing.log 包含测试 Agent ID');
  254 + assert(content.includes('Cost: $'), 'billing.log 包含美元成本字段');
  255 + assert(content.includes('Duration:'), 'billing.log 包含耗时字段');
  256 +
  257 + // 清理测试行（不删文件，只清测试数据）
  258 + const cleanedContent = content.split('\n').filter(l => !l.includes('_test_billing')).join('\n');
  259 + fs.writeFileSync(billingPath, cleanedContent, 'utf-8');
  260 +}
  261 +
  262 +// ═══════════════════════════════════════
  263 +// Test 7: GoalsService 新增状态兼容
  264 +// ═══════════════════════════════════════
  265 +function test7_goalsServiceStates() {
  266 + console.log('\n🎯 Test 7: GoalsService 新增状态兼容');
  267 +
  268 + const { GoalsService } = require(path.join(ROOT, 'server/services/goals.js'));
  269 + const svc = new GoalsService(); // 使用全局 goals dir
  270 +
  271 + // 验证源码中有新状态
  272 + const goalsSource = fs.readFileSync(path.join(ROOT, 'server/services/goals.js'), 'utf-8');
  273 + assert(goalsSource.includes('researching'), 'GoalsService 源码包含 researching 状态');
  274 + assert(goalsSource.includes('planning'), 'GoalsService 源码包含 planning 状态');
  275 + assert(goalsSource.includes('executing'), 'GoalsService 源码包含 executing 状态');
  276 + assert(goalsSource.includes('verifying'), 'GoalsService 源码包含 verifying 状态');
  277 + assert(goalsSource.includes('pending_approval'), 'GoalsService 源码包含 pending_approval 状态');
  278 +}
  279 +
  280 +// ═══════════════════════════════════════
  281 +// Test 8: CLI task transition 命令注册
  282 +// ═══════════════════════════════════════
  283 +function test8_cliRegistration() {
  284 + console.log('\n🔧 Test 8: CLI task transition 命令注册');
  285 +
  286 + const lemonSrc = fs.readFileSync(path.join(ROOT, 'bin/lemon.js'), 'utf-8');
  287 + assert(lemonSrc.includes("'task transition'"), "bin/lemon.js 注册了 'task transition' 命令");
  288 + assert(lemonSrc.includes("phase: { type: 'string'"), 'task transition 接受 phase 参数');
  289 + assert(lemonSrc.includes("artifact: { type: 'string'"), 'task transition 接受 artifact 参数');
  290 + assert(lemonSrc.includes("agentId: { type: 'string'"), 'task transition 接受 agentId 参数');
  291 +}
  292 +
  293 +// ═══════════════════════════════════════
  294 +// Test 9: agent-chat.js Transition 路由注册
  295 +// ═══════════════════════════════════════
  296 +function test9_transitionRoute() {
  297 + console.log('\n🌐 Test 9: Transition API 路由注册');
  298 +
  299 + const routeSrc = fs.readFileSync(path.join(ROOT, 'server/routes/agent-chat.js'), 'utf-8');
  300 + assert(routeSrc.includes("/:agentId/task/transition"), '存在 /:agentId/task/transition 路由');
  301 + assert(routeSrc.includes('Hot Swap'), '路由包含 Hot Swap 换脑逻辑');
  302 + assert(routeSrc.includes('Context Flush'), '路由包含 Context Flush 逻辑');
  303 + assert(routeSrc.includes('pending_approval'), '路由包含 P0 审批拦截逻辑');
  304 + assert(routeSrc.includes('billing.log'), '路由包含 billing.log 计费写入');
  305 + assert(routeSrc.includes('_lessons.md'), '路由包含 _lessons.md 注入');
  306 +}
  307 +
  308 +// ═══════════════════════════════════════
  309 +// Test 10 (Online): 真实 API 调用测试
  310 +// ═══════════════════════════════════════
  311 +async function test10_onlineAPI() {
  312 + console.log('\n🌍 Test 10: 在线 API 调用测试 (需要服务器)');
  313 +
  314 + try {
  315 + // 10a: 调用 transition API (对一个不存在的 Agent 应返回 404)
  316 + const res1 = await httpPost('/agent/_nonexistent_agent/task/transition', { phase: 'plan' });
  317 + assert(res1.status === 404, 'Transition API 对不存在的 Agent 返回 404');
  318 +
  319 + // 10b: 如果 master Agent 存在, 尝试对它做一次 research -> plan 的切换
  320 + const res2 = await httpPost('/agent/master/task/transition', { phase: 'plan', artifact: '' });
  321 + assert(res2.status === 200 && res2.body?.success === true, 'Master Agent transition(plan) 返回 success');
  322 +
  323 + // 10c: 检查 master 的 main_skill 是否被换成了 meta-plan
  324 + const masterSkillPath = path.join(ROOT, 'data/system/agents/master/.claude/skills/main_skill/SKILL.md');
  325 + if (fs.existsSync(masterSkillPath)) {
  326 + const content = fs.readFileSync(masterSkillPath, 'utf-8');
  327 + assert(content.includes('meta-plan'), 'Master 的 main_skill 已被换为 meta-plan');
  328 + } else {
  329 + assert(false, 'Master 的 main_skill/SKILL.md 不存在（可能目录结构不同）');
  330 + }
  331 + } catch (err) {
  332 + assert(false, '在线 API 测试', `服务器连接失败: ${err.message}. 请先启动服务器 (node server/index.js)`);
  333 + }
  334 +}
  335 +
  336 +// ═══════════════════════════════════════
  337 +// Runner
  338 +// ═══════════════════════════════════════
  339 +async function main() {
  340 + console.log('═══════════════════════════════════════');
  341 + console.log(' 🏗️ Harness Engineering 测试套件');
  342 + console.log('═══════════════════════════════════════');
  343 +
  344 + const isOnline = process.argv.includes('--online');
  345 +
  346 + test1_metaSkillFiles();
  347 + test2_baseTemplate();
  348 + test3_hotSwap();
  349 + test4_contextFlush();
  350 + test5_lessonsInjection();
  351 + test6_billingLog();
  352 + test7_goalsServiceStates();
  353 + test8_cliRegistration();
  354 + test9_transitionRoute();
  355 +
  356 + if (isOnline) {
  357 + await test10_onlineAPI();
  358 + } else {
  359 + console.log('\n⏭️ 跳过在线测试 (使用 --online 参数启用)');
  360 + }
  361 +
  362 + // 结果汇总
  363 + console.log('\n═══════════════════════════════════════');
  364 + console.log(` 📊 测试结果: ${passed} 通过, ${failed} 失败, 共 ${passed + failed} 项`);
  365 + if (failed > 0) {
  366 + console.log('\n 失败项:');
  367 + results.filter(r => r.status === '❌').forEach(r => {
  368 + console.log(` ❌ ${r.testName}: ${r.detail}`);
  369 + });
  370 + }
  371 + console.log('═══════════════════════════════════════\n');
  372 +
  373 + process.exit(failed > 0 ? 1 : 0);
  374 +}
  375 +
  376 +main();