#!/usr/bin/env node /** * scripts/phase3-verify.js * * P3 L1 + L2 联动验收 (mock 模式 · 4 类分支): * ① contract_mismatch → shipped · sdd_rev=2 · l2_rounds=1 · sdd.updated SSE · changelog sdd_rev 条 * ② impl_bug → shipped · sdd_rev=1 · l2_rounds=0 · 仅 reset Unit * ③ spec_ambiguous → l2_blocked · human_intervention_needed SSE * ④ infra_flake → shipped · l2_rounds=0 · rerun 无 reset * ⑤ Observability → bugs.ndjson · timeline.ndjson · stream.log SSE replay * * 用法: node scripts/phase3-verify.js */ process.env.LEMONBOSS_CLAUDE_MODE = 'mock'; const fs = require('fs'); const path = require('path'); const serverNodeModules = path.resolve(__dirname, '../server/node_modules'); require('module').globalPaths.push(serverNodeModules); const express = (() => { try { return require('express'); } catch { return require(require.resolve('express', { paths: [path.resolve(__dirname, '../server')] })); } })(); const pathResolver = require('../server/services/devloop/path-resolver'); const sseBroker = require('../server/services/devloop/sse-broker'); const bugTracker = require('../server/services/devloop/bug-tracker'); const lc = require('../server/src/core/devloop/loop-controller'); const SCENARIO_BASE = path.resolve(__dirname, '../fixtures/devloop-mock-scenarios'); const results = []; function step(name, fn) { return (async () => { try { const r = await fn(); results.push({ name, ok: true, detail: r?.detail || '' }); return r; } catch (e) { results.push({ name, ok: false, reason: e.message }); throw e; } })(); } function makeFakeRes(events) { return { setHeader() {}, flushHeaders() {}, end() {}, on() {}, write(line) { const m = line.match(/^event: (.+)$/m); if (m) events.push(m[1]); }, }; } async function runScenario(scenario) { pathResolver._resetForTesting(); sseBroker._resetForTesting(); const cid = 'c-phase3-' + scenario + '-' + Date.now().toString(36); const dir = path.join(SCENARIO_BASE, scenario); const proj = await lc.createProject({ company_id: cid, mode: 'sandbox', idea: 'phase3 ' + scenario }); await lc.runDocsPhase(proj.pid, 'TODO demo', { mockFixtureDir: dir }); const events = []; sseBroker.subscribe(proj.pid, makeFakeRes(events), {}); const result = await lc.runProject(proj.pid, { mockFixtureDir: dir }); sseBroker.close(proj.pid); return { pid: proj.pid, cid, result, events }; } function cleanup(cid) { try { fs.rmSync(path.join(pathResolver.DATA_DIR, 'companies', cid), { recursive: true, force: true }); } catch { /* ignore */ } } async function main() { const cidsToClean = []; // ① contract_mismatch let cm; try { await step('① contract_mismatch → shipped (sdd_rev=2, l2_rounds=1)', async () => { cm = await runScenario('contract_mismatch'); cidsToClean.push(cm.cid); if (cm.result.final_state !== 'shipped') throw new Error('not shipped: ' + cm.result.final_state); const u = JSON.parse(fs.readFileSync(pathResolver.resolveUnitsJson(cm.pid), 'utf8')); if (u.l2_rounds !== 1) throw new Error('l2_rounds=' + u.l2_rounds); if (u.sdd_rev !== '2') throw new Error('sdd_rev=' + u.sdd_rev); return { detail: `pid=${cm.pid} sdd_rev=2 l2_rounds=1` }; }); await step(' sdd.updated SSE event', () => { if (!cm.events.includes('sdd.updated')) throw new Error('no sdd.updated in: ' + cm.events.join(',')); return { detail: 'sdd.updated emitted' }; }); await step(' changelog has sdd_rev=2 entry', () => { const cl = pathResolver.resolveChangelog(cm.pid); if (!fs.existsSync(cl)) throw new Error('changelog missing'); const lines = fs.readFileSync(cl, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); if (!lines.some(l => l.kind === 'sdd_rev' && l.rev === 2)) throw new Error('no sdd_rev=2 in changelog'); return { detail: `${lines.length} changelog entries` }; }); await step(' bug opened → fixed', () => { const bugs = bugTracker.list(cm.pid, {}); if (bugs.length !== 1) throw new Error('want 1 bug, got ' + bugs.length); if (bugs[0].status !== 'fixed') throw new Error('bug.status=' + bugs[0].status); return { detail: `${bugs[0].id} (${bugs[0].status})` }; }); } catch { /* errors recorded by step */ } // ② impl_bug let ib; try { await step('② impl_bug → shipped · SDD unchanged · l2_rounds=0', async () => { ib = await runScenario('impl_bug'); cidsToClean.push(ib.cid); if (ib.result.final_state !== 'shipped') throw new Error('state=' + ib.result.final_state); const u = JSON.parse(fs.readFileSync(pathResolver.resolveUnitsJson(ib.pid), 'utf8')); if (u.l2_rounds !== 0) throw new Error('l2_rounds=' + u.l2_rounds + ' (impl_bug should not burn)'); if (u.sdd_rev !== '1') throw new Error('sdd_rev=' + u.sdd_rev + ' (should remain 1)'); return { detail: `sdd_rev=1 l2_rounds=0` }; }); } catch {} // ③ spec_ambiguous let sa; try { await step('③ spec_ambiguous → l2_blocked + human_intervention_needed SSE', async () => { sa = await runScenario('spec_ambiguous'); cidsToClean.push(sa.cid); if (sa.result.final_state !== 'l2_blocked') throw new Error('state=' + sa.result.final_state); if (!sa.events.includes('human_intervention_needed')) throw new Error('no human_intervention_needed SSE'); return { detail: 'l2_blocked · SSE fired' }; }); } catch {} // ④ infra_flake let inf; try { await step('④ infra_flake → shipped · l2_rounds=0 · rerun_integrator', async () => { inf = await runScenario('infra_flake'); cidsToClean.push(inf.cid); if (inf.result.final_state !== 'shipped') throw new Error('state=' + inf.result.final_state); const u = JSON.parse(fs.readFileSync(pathResolver.resolveUnitsJson(inf.pid), 'utf8')); if (u.l2_rounds !== 0) throw new Error('l2_rounds=' + u.l2_rounds); const actions = inf.result.history.filter(h => h.phase === 'L2.dispatch').map(h => h.action); if (!actions.some(a => /rerun_integrator/.test(a))) throw new Error('no rerun_integrator action'); return { detail: 'shipped without budget burn' }; }); } catch {} // ⑤ Observability cross-check try { await step('⑤ Observability — timeline / bugs / changelog / stream.log', () => { // 用 contract_mismatch 项目验 if (!cm) throw new Error('cm scenario did not run'); const tl = pathResolver.resolveTimelineLog(cm.pid); const bg = pathResolver.resolveBugsLog(cm.pid); const cl = pathResolver.resolveChangelog(cm.pid); const sl = pathResolver.resolveStreamLog(cm.pid); for (const [name, p] of [['timeline', tl], ['bugs', bg], ['changelog', cl], ['stream.log', sl]]) { if (!fs.existsSync(p)) throw new Error(`${name} missing`); } const tlKinds = fs.readFileSync(tl, 'utf8').trim().split('\n').filter(Boolean) .map(l => { try { return JSON.parse(l).kind; } catch { return ''; }}); for (const must of ['unit.state', 'integration.result', 'triage.result', 'sdd.updated']) { // unit.state is via sseBroker.emit not timeline; but we should at least have integration.result + sdd.updated as timeline kinds — actually those go via sseBroker.emit too. Check timeline has agent.call entries. } // 至少: project.created + agent.call (PM + Architect + L1 coder/tester + integrator + triager + architect-patch) + project.shipped const expectedTimelineKinds = ['project.created', 'agent.call', 'project.shipped']; const missing = expectedTimelineKinds.filter(k => !tlKinds.includes(k)); if (missing.length) throw new Error('timeline missing kinds: ' + missing.join(',')); return { detail: `timeline=${tlKinds.length}ev · bugs+changelog+stream.log present` }; }); await step('⑥ SSE replay via Last-Event-ID', async () => { if (!cm) throw new Error('cm scenario did not run'); const replayed = []; sseBroker.subscribe(cm.pid, makeFakeRes(replayed), { replayFromOffset: 3 }); sseBroker.close(cm.pid); // 至少应回放出 sdd.updated / project.shipped 之类 if (replayed.length < 2) throw new Error('replay too few events: ' + replayed.length); return { detail: `replayed ${replayed.length} events from id=3` }; }); } catch {} for (const c of cidsToClean) cleanup(c); let pass = 0, fail = 0; console.log('\n┌─ DevLoop Phase 3 — L1+L2 联动 Acceptance ────────'); for (const r of results) { const tag = r.ok ? '✓' : '✗'; console.log(`│ ${tag} ${r.name.padEnd(62)} ${r.ok ? r.detail : '— ' + r.reason}`); r.ok ? pass++ : fail++; } console.log(`└─ ${pass}/${pass + fail} steps passed${fail ? ' · ' + fail + ' failed' : ''}\n`); process.exit(fail ? 1 : 0); } main().catch(e => { console.error('\n[phase3-verify] fatal:', e); for (const r of results) console.log(' ', r.ok ? '✓' : '✗', r.name, r.ok ? r.detail : '— ' + r.reason); process.exit(1); });