From fadb6fa18ab0c903f1fd32be5eef87460aa11f15 Mon Sep 17 00:00:00 2001 From: yibo Date: Sat, 22 Aug 2026 16:52:52 +0800 Subject: [PATCH] feat(validation): 提案 B/C/D 一起落地——证据可见性 + 复用检查工具 + 引擎证据闸门 --- scripts/visual-acceptance-check.mjs | 197 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/visual-acceptance-check.smoke.mjs | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ server/routes/graph-run.js | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ server/services/goal-contract.js | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------ server/tests/unit/goal-contract.test.js | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ server/tests/unit/graph-run-route-io.test.js | 41 +++++++++++++++++++++++++++++++++++++++++ system/agents/benchmark-validator/agent.md | 20 ++++++++++++++++++++ web/src/components/graph-console/GraphRunPane.vue | 4 ++++ web/src/components/graph-console/NodeDetailPanel.vue | 43 +++++++++++++++++++++++++++++++++++++++++++ web/src/views/GraphConsole.vue | 1 + 10 files changed, 578 insertions(+), 6 deletions(-) create mode 100644 scripts/visual-acceptance-check.mjs create mode 100644 scripts/visual-acceptance-check.smoke.mjs diff --git a/scripts/visual-acceptance-check.mjs b/scripts/visual-acceptance-check.mjs new file mode 100644 index 0000000..2e64fb2 --- /dev/null +++ b/scripts/visual-acceptance-check.mjs @@ -0,0 +1,197 @@ +#!/usr/bin/env node +/** + * visual-acceptance-check.mjs — 可复用的真实浏览器验收检查（35 号文档提案 C） + * + * 起因：真实回归任务里，goal-benchmark.json 的 evidence_hint 常常点名"打开浏览器 / + * 依次设置视口宽度 / 开发者工具 Console+Network 面板"这类检查，但系统里没有一份 + * 标准化的工具——每次都靠 agent 自己临时想（这次撞见的两种极端：一次真的写了一份 + * Playwright 脚本认真测，一次只写了正则解析文本）。这份脚本把最常见的几类检查 + * 封装成一个通用工具，agent 通过 bash 直接调用，不用每次重新发明。 + * + * 用法： + * node scripts/visual-acceptance-check.mjs overflow --file index.html --widths 375,768,1440 + * node scripts/visual-acceptance-check.mjs style --file index.html --selector .hero-title --prop font-size + * node scripts/visual-acceptance-check.mjs contrast --fg "#1d1d1f" --bg "#ffffff" + * node scripts/visual-acceptance-check.mjs console --file index.html + * + * 每个子命令把结果打到 stdout（一行 JSON），可以直接摘录进 attempt-N.scores.json 的 + * evidence. 字段（比如 `node ... | node -e "console.log(JSON.parse(require('fs').readFileSync(0)).ratio)"`， + * 或者更常见的做法：把这行 JSON 原样贴进 evidence 文本）。 + * + * 退出码：0 = 检查本身跑完了（不代表"通过"，通不通过看输出里的字段）； + * 1 = 参数错 / 文件不存在 / 浏览器跑不起来，检查没能真正执行。 + * 这个区分很重要：agent 应该只在退出码 0 时把输出当"真实证据"贴进 evidence， + * 退出码 1 时说明检查本身没跑成，不能拿它的输出（如果有）冒充证据。 + */ + +import fs from 'fs'; +import path from 'path'; +import { pathToFileURL } from 'url'; + +function parseArgs(argv) { + const out = { _: [] }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith('--')) { + const key = a.slice(2); + const next = argv[i + 1]; + if (next === undefined || next.startsWith('--')) { out[key] = true; } + else { out[key] = next; i += 1; } + } else out._.push(a); + } + return out; +} + +function fail(msg) { + process.stderr.write(`ERROR ${msg}\n`); + process.exit(1); +} + +function requireFile(p) { + const abs = path.resolve(p); + if (!fs.existsSync(abs)) fail(`文件不存在: ${abs}`); + return abs; +} + +function parseWidths(s) { + const widths = String(s || '').split(',').map((x) => Number(x.trim())).filter((n) => Number.isFinite(n) && n > 0); + if (!widths.length) fail('--widths 需要至少一个正整数，用逗号分隔，如 375,768,1440'); + return widths; +} + +async function loadChromium() { + try { + const { chromium } = await import('playwright'); + return chromium; + } catch (e) { + fail(`加载 playwright 失败（${e.message}）——确认在仓库根目录下跑这份脚本，node_modules/playwright 才找得到`); + } +} + +// ── overflow：多档视口宽度下有没有横向溢出 ────────────────────────── +async function cmdOverflow(args) { + const file = requireFile(args.file || fail('--file 必填')); + const widths = parseWidths(args.widths); + const height = Number(args.height) || 900; + const chromium = await loadChromium(); + const browser = await chromium.launch(); + try { + const page = await browser.newPage(); + const results = []; + for (const width of widths) { + await page.setViewportSize({ width, height }); + await page.goto(pathToFileURL(file).href, { waitUntil: 'load' }); + const m = await page.evaluate(() => ({ + scrollWidth: document.documentElement.scrollWidth, + clientWidth: document.documentElement.clientWidth, + })); + results.push({ width, ...m, overflow: m.scrollWidth > m.clientWidth }); + } + print({ file, widths: results, overflow_any: results.some((r) => r.overflow) }); + } finally { + await browser.close(); + } +} + +// ── style：某个选择器在指定视口下的某个 computedStyle 属性 ────────── +async function cmdStyle(args) { + const file = requireFile(args.file || fail('--file 必填')); + const selector = args.selector || fail('--selector 必填'); + const prop = args.prop || fail('--prop 必填'); + const width = Number(args.width) || 1440; + const height = Number(args.height) || 900; + const chromium = await loadChromium(); + const browser = await chromium.launch(); + try { + const page = await browser.newPage(); + await page.setViewportSize({ width, height }); + await page.goto(pathToFileURL(file).href, { waitUntil: 'load' }); + const value = await page.evaluate(({ selector, prop }) => { + const el = document.querySelector(selector); + if (!el) return null; + return getComputedStyle(el)[prop] ?? null; + }, { selector, prop }); + print({ file, selector, prop, width, height, value, found: value !== null }); + } finally { + await browser.close(); + } +} + +// ── console：加载页面时的 console 错误 + 失败的网络请求 ────────────── +async function cmdConsole(args) { + const file = requireFile(args.file || fail('--file 必填')); + const width = Number(args.width) || 1440; + const height = Number(args.height) || 900; + const chromium = await loadChromium(); + const browser = await chromium.launch(); + try { + const page = await browser.newPage(); + await page.setViewportSize({ width, height }); + const consoleErrors = []; + const failedRequests = []; + page.on('console', (msg) => { if (msg.type() === 'error') consoleErrors.push(msg.text()); }); + page.on('requestfailed', (req) => failedRequests.push({ url: req.url(), failure: req.failure()?.errorText || null })); + await page.goto(pathToFileURL(file).href, { waitUntil: 'load' }); + print({ + file, console_errors: consoleErrors, failed_requests: failedRequests, + clean: consoleErrors.length === 0 && failedRequests.length === 0, + }); + } finally { + await browser.close(); + } +} + +// ── contrast：WCAG 相对亮度对比度，纯计算，不需要浏览器 ───────────── +function srgbToLinear(c) { + const v = c / 255; + return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4; +} +function relativeLuminance(hex) { + const h = hex.replace('#', ''); + const r = parseInt(h.slice(0, 2), 16); + const g = parseInt(h.slice(2, 4), 16); + const b = parseInt(h.slice(4, 6), 16); + if ([r, g, b].some((n) => Number.isNaN(n))) return null; + return 0.2126 * srgbToLinear(r) + 0.7152 * srgbToLinear(g) + 0.0722 * srgbToLinear(b); +} +function cmdContrast(args) { + const fg = args.fg || fail('--fg 必填（如 "#1d1d1f"）'); + const bg = args.bg || fail('--bg 必填（如 "#ffffff"）'); + const lFg = relativeLuminance(fg); + const lBg = relativeLuminance(bg); + if (lFg === null || lBg === null) fail(`颜色解析失败，需要 #rrggbb 形式：fg=${fg} bg=${bg}`); + const [lighter, darker] = lFg > lBg ? [lFg, lBg] : [lBg, lFg]; + const ratio = (lighter + 0.05) / (darker + 0.05); + print({ + fg, bg, ratio: Number(ratio.toFixed(2)), + wcag_aa_normal_text: ratio >= 4.5, // WCAG 2.1 AA：正文 ≥4.5:1 + wcag_aa_large_text: ratio >= 3, // WCAG 2.1 AA：大字号（≥18pt 或 14pt 加粗）≥3:1 + }); +} + +function print(obj) { + process.stdout.write(`${JSON.stringify(obj)}\n`); +} + +async function main() { + const [cmd, ...rest] = process.argv.slice(2); + const args = parseArgs(rest); + switch (cmd) { + case 'overflow': return cmdOverflow(args); + case 'style': return cmdStyle(args); + case 'console': return cmdConsole(args); + case 'contrast': return cmdContrast(args); + default: + process.stderr.write([ + '用法: node scripts/visual-acceptance-check.mjs [选项]', + ' overflow --file --widths [--height ]', + ' style --file --selector --prop [--width ] [--height ]', + ' console --file [--width ] [--height ]', + ' contrast --fg --bg ', + '', + ].join('\n')); + process.exit(cmd ? 1 : 0); + } +} + +main().catch((e) => fail(e.stack || e.message)); diff --git a/scripts/visual-acceptance-check.smoke.mjs b/scripts/visual-acceptance-check.smoke.mjs new file mode 100644 index 0000000..a500fa4 --- /dev/null +++ b/scripts/visual-acceptance-check.smoke.mjs @@ -0,0 +1,76 @@ +#!/usr/bin/env node +/** + * 冒烟检查（不进 vitest 套件）：visual-acceptance-check.mjs 四个子命令各跑一次， + * 用真实 Chromium + 一份现造的最小 HTML fixture，断言输出形状与已知答案。 + * 起真浏览器要几秒，故意不挂进 vitest 的快速单测循环——手动跑： + * node scripts/visual-acceptance-check.smoke.mjs + */ +import assert from 'assert'; +import { execFileSync } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const CLI = fileURLToPath(new URL('./visual-acceptance-check.mjs', import.meta.url)); +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'visual-check-smoke-')); +const htmlPath = path.join(tmpDir, 'index.html'); + +// 故意做成「小宽度会溢出、大宽度不会」，且带一个必定失败的图片请求， +// 已知答案才好断言，不是随手糊一份页面 +fs.writeFileSync(htmlPath, ` + + +
Hi
+
+ + +`); + +function run(args) { + const out = execFileSync(process.execPath, [CLI, ...args], { encoding: 'utf8' }); + return JSON.parse(out.trim().split('\n').pop()); +} + +// ① overflow：375 应该溢出（.wide 是 500px），1440 不该溢出 +{ + const r = run(['overflow', '--file', htmlPath, '--widths', '375,1440']); + const at375 = r.widths.find((w) => w.width === 375); + const at1440 = r.widths.find((w) => w.width === 1440); + assert.strictEqual(at375.overflow, true, '375px 视口下 500px 宽的 div 应该判定为溢出'); + assert.strictEqual(at1440.overflow, false, '1440px 视口下不该溢出'); + assert.strictEqual(r.overflow_any, true); + console.log('✓ overflow'); +} + +// ② style：真读到 .title 的 font-size 计算值 +{ + const r = run(['style', '--file', htmlPath, '--selector', '.title', '--prop', 'font-size']); + assert.strictEqual(r.value, '40px', `期望 40px，实得 ${r.value}`); + assert.strictEqual(r.found, true); + console.log('✓ style'); +} + +// ③ console：真捕获到 console.error 与失败的图片请求 +{ + const r = run(['console', '--file', htmlPath]); + assert.ok(r.console_errors.includes('boom'), 'console_errors 应包含页面里那条 console.error'); + assert.ok(r.failed_requests.length >= 1, '应该捕获到 does-not-exist.png 的失败请求'); + assert.strictEqual(r.clean, false); + console.log('✓ console'); +} + +// ④ contrast：#1d1d1f / #ffffff 是真实网站常用的高对比度组合，已知比值约 16.8:1 +{ + const r = run(['contrast', '--fg', '#1d1d1f', '--bg', '#ffffff']); + assert.ok(r.ratio > 16 && r.ratio < 17.5, `期望约 16.8:1，实得 ${r.ratio}`); + assert.strictEqual(r.wcag_aa_normal_text, true); + console.log('✓ contrast'); +} + +fs.rmSync(tmpDir, { recursive: true, force: true }); +console.log('全部通过'); diff --git a/server/routes/graph-run.js b/server/routes/graph-run.js index 93d148f..e846114 100644 --- a/server/routes/graph-run.js +++ b/server/routes/graph-run.js @@ -340,6 +340,53 @@ router.post('/:id/nodes/:nodeId/retry', h((req, res) => { * loopRecords = 回退账本，画布上 `N/3` 徽标的数据来源（§四.B.2） * budget = root run 的 budget_json 解析结果，页头预算条用（§四.B.4）；非 root run 也回这个值 */ +/** + * 35 号文档提案 B：控制台"这份验收有没有证据支撑"要读的数据。 + * + * 只读 `goal-contract.js` 已经落盘的判决索引 + 判决全文（`verdicts/index.ndjson` + + * `verdicts/{id}.json`），不重新计算——判决在 `runAcceptance`/`goal-contract.js run` + * 那一刻就已经算好了，这里只是把 `sections.benchmark.dimensions[]`（含提案 D 新加的 + * `evidence_required`）摘出来给前端渲染。读不到 / 解析不出就跳过那一条，不拖垮整个 + * 响应——这是纯展示数据，读盘失败不该让 GET /:id 报错。 + */ +function collectBenchmarkVerdicts(rootRun) { + if (!rootRun) return []; + let gc; + try { + gc = require('../services/goal-contract'); + } catch { + return []; + } + const runDir = gc.resolveRunDir(rootRun.company_id, rootRun.root_run_id || rootRun.id); + let index; + try { + index = gc.listVerdicts(runDir, {}); + } catch { + return []; + } + const out = []; + for (const entry of index) { + let full; + try { + full = gc.readVerdict(runDir, entry.verdict_id); + } catch { + full = null; + } + const dims = full && full.sections && full.sections.benchmark && full.sections.benchmark.dimensions; + if (!Array.isArray(dims)) continue; + out.push({ + verdictId: entry.verdict_id, + nodeRunId: entry.node_run_id || null, + ok: entry.ok, + dimensions: dims.map((d) => ({ + name: d.name, ok: d.ok, score: d.score, + evidenceRequired: Boolean(d.evidence_required), + })), + }); + } + return out; +} + router.get('/:id', h((req, res) => { const state = getRunState(req.params.id); if (!state) return res.status(404).json({ error: `graph run 不存在: ${req.params.id}` }); @@ -373,6 +420,9 @@ router.get('/:id', h((req, res) => { // （见 graph-engine.js「W2 · 每个 root run 一个项目根」那段），两者不是同一份数据。 // readProjectRootRecord 按 root_run_id 查，任一层 run 传进去结果相同（同一棵树共享一个项目根）。 projectRoot: engine.readProjectRootRecord(state.run), + // 35 号文档提案 B：每个 benchmark 判决按维度标了有没有证据支撑，前端拿去在 + // 节点详情上画一个小标签——不改变任何判定逻辑，纯展示。 + benchmarkVerdicts: collectBenchmarkVerdicts(rootRun), loopRecords: listLoopRecords(state.run.id), childRuns: listChildRuns(state.run.id), budget, diff --git a/server/services/goal-contract.js b/server/services/goal-contract.js index 95a3afb..d0ea806 100644 --- a/server/services/goal-contract.js +++ b/server/services/goal-contract.js @@ -570,10 +570,50 @@ const SHORT_CIRCUIT_HINTS = { }; /** + * 35 号文档提案 D：evidence_hint 里点名"打开浏览器/视口/开发者工具/截图"这类维度， + * 光有一个分数不够——34 号文档真实撞见过 validator 给这类维度打满分、却没有任何 + * 迹象证明它真的做过对应检查。这里只做**关键词检测 + 证据字段存在性检查**， + * 不校验证据内容真假（那需要重新起一次浏览器去验证，超出「引擎判分时能做什么」 + * 的范围，留给下次；见 35 号文档 §二 提案 D 的范围说明）。 + */ +const EVIDENCE_REQUIRED_RE = /浏览器|viewport|视口|开发者工具|devtools|console|network|截图|screenshot/i; +function requiresEvidence(dimension) { + return EVIDENCE_REQUIRED_RE.test(String((dimension && dimension.evidence_hint) || '')); +} + +/** + * `--target` 通常是 `attempt-.scores.json`（只有分数，见 benchmark-validator/agent.md + * Step 3），证据写在**同目录的姊妹文件** `attempt-.json`（判词文件，Step 5）的 + * `dimensions[].evidence` 里——这是已有的、agent.md 早就在教的约定，不是本次新加的格式。 + * 这里只是把两份文件的证据接起来给 runBenchmark 用，不要求 agent 多写一份。 + */ +function readSiblingEvidence(targetAbs) { + if (!targetAbs || !targetAbs.endsWith('.scores.json')) return {}; + const siblingPath = `${targetAbs.slice(0, -'.scores.json'.length)}.json`; + try { + const doc = JSON.parse(fs.readFileSync(siblingPath, 'utf8')); + const dims = Array.isArray(doc.dimensions) ? doc.dimensions : []; + const out = {}; + for (const d of dims) { + if (d && typeof d.name === 'string' && typeof d.evidence === 'string' && d.evidence.trim()) { + out[d.name] = d.evidence; + } + } + return out; + } catch { + return {}; // 判词文件还没写 / 不是合法 JSON：不该拖垮打分本身，靠下面的证据缺失走正常失败分支 + } +} + +/** * benchmark：被测产物提供各维度分数，执行器只做加权与阈值比较（纯声明式，不起 LLM）。 - * 被测产物形态：{"scores": {"": 87, ...}} 或直接 {"": 87, ...} + * 被测产物形态：{"scores": {"": 87, ...}} 或直接 {"": 87, ...}。 + * `evidence_hint` 命中浏览器类关键词的维度必须有证据支撑——证据来源两处都认： + * `--target` 文件里自带的 `evidence.`（谁都可以这么写），或者同目录姊妹判词文件 + * `attempt-.json` 里 `dimensions[].evidence`（`benchmark-validator` 已有的既定写法， + * 见上面 `readSiblingEvidence`）；`--target` 里的显式声明优先。 */ -function runBenchmark(sec, target) { +function runBenchmark(sec, target, targetAbs) { // 短路分支也必须进 failures：verdict.failures 是下游（validator agent / validator-triager） // 唯一的扁平归因入口，只留 reason 不留 failures 的话「读回」这一环就空了（§十 四环缺陷）。 if (!target.ok) { @@ -584,6 +624,8 @@ function runBenchmark(sec, target) { }; } const scores = (target.json && typeof target.json.scores === 'object' && target.json.scores) || target.json || {}; + const explicitEvidence = (target.json && typeof target.json.evidence === 'object' && target.json.evidence) || {}; + const evidence = { ...readSiblingEvidence(targetAbs), ...explicitEvidence }; const dims = []; let weighted = 0; let ok = true; @@ -592,7 +634,10 @@ function runBenchmark(sec, target) { const score = scores[d.name]; const has = typeof score === 'number'; const threshold = typeof d.pass_threshold === 'number' ? d.pass_threshold : null; - const passed = has && (threshold === null || score >= threshold); + const needsEvidence = requiresEvidence(d); + const hasEvidence = !needsEvidence + || (typeof evidence[d.name] === 'string' && evidence[d.name].trim() !== ''); + const passed = has && hasEvidence && (threshold === null || score >= threshold); if (!passed) ok = false; if (has) weighted += score * d.weight; dims.push({ @@ -601,7 +646,8 @@ function runBenchmark(sec, target) { score: has ? score : null, pass_threshold: threshold, ok: passed, - reason: has ? undefined : 'missing score in target', + evidence_required: needsEvidence || undefined, + reason: !has ? 'missing score in target' : (!hasEvidence ? 'evidence_required' : undefined), }); } @@ -619,7 +665,9 @@ function runBenchmark(sec, target) { case: d.name, expected: d.pass_threshold, got: d.score, - hint: d.reason || 'dimension score below pass_threshold', + hint: d.reason === 'evidence_required' + ? `evidence_hint 要求真实检查（浏览器/视口/开发者工具类），但被测产物没有在 evidence.${d.name} 留下证据——只给了一个分数` + : (d.reason || 'dimension score below pass_threshold'), })), }; } @@ -818,7 +866,7 @@ async function executeContract(contract, targetPath, opts = {}) { : { ok: true, json: null }; const sections = { benchmark: null, test_data: null, test_script: null }; - if (contract.benchmark) sections.benchmark = runBenchmark(contract.benchmark, target); + if (contract.benchmark) sections.benchmark = runBenchmark(contract.benchmark, target, targetAbs); if (contract.test_data) sections.test_data = runTestData(contract.test_data, target); if (contract.test_script) { // 脚本拿到的 ACCEPTANCE_TARGET 必须是绝对路径：子进程的 cwd 是 runDir/artifacts 下的某处， diff --git a/server/tests/unit/goal-contract.test.js b/server/tests/unit/goal-contract.test.js index d96549e..2957f00 100644 --- a/server/tests/unit/goal-contract.test.js +++ b/server/tests/unit/goal-contract.test.js @@ -253,6 +253,98 @@ describe('执行器：benchmark', () => { }); }); +// 35 号文档提案 D：evidence_hint 点名"浏览器/视口/开发者工具"类检查的维度， +// 没有 evidence. 就不能凭一个分数通过——真实撞见过 validator 给这类维度 +// 打满分却没做过对应检查（见 34 号文档）。 +describe('执行器：benchmark 的证据闸门（35 号文档提案 D）', () => { + const visualBenchmark = { + dimensions: [ + { + name: 'hero_first_impression', weight: 0.5, pass_threshold: 80, + evidence_hint: '打开浏览器截图首屏，量标题与标语的计算字号比例', + }, + { name: 'code_quality', weight: 0.5, pass_threshold: 70 }, // 不要求证据的普通维度 + ], + overall_pass_threshold: 75, + }; + const contract = { version: '1', goal_id: 'GOAL-visual', benchmark: visualBenchmark }; + + it('要求证据的维度只给分数、没给 evidence：即使分数达标也判不过', async () => { + const runDir = newRunDir(); + const target = writeTarget(runDir, 'rn_a', 1, { + scores: { hero_first_impression: 100, code_quality: 90 }, + }); + const v = await gc.executeContract(contract, target, { runDir }); + expect(v.ok).toBe(false); + const f = v.failures.find((x) => x.case === 'hero_first_impression'); + expect(f).toBeTruthy(); + expect(f.hint).toMatch(/evidence_hint 要求真实检查/); + }); + + it('给了 evidence. 之后，达标分数正常通过', async () => { + const runDir = newRunDir(); + const target = writeTarget(runDir, 'rn_a', 1, { + scores: { hero_first_impression: 100, code_quality: 90 }, + evidence: { hero_first_impression: 'Playwright 实测：标题48px/标语16px，比例3:1' }, + }); + const v = await gc.executeContract(contract, target, { runDir }); + expect(v.ok).toBe(true); + expect(v.failures).toEqual([]); + }); + + it('不含"浏览器/视口/开发者工具"类关键词的普通维度，不受这道闸门约束（不用给 evidence）', async () => { + const runDir = newRunDir(); + // 只给 code_quality 一个维度的分数，hero_first_impression 缺分（走既有的"缺分"分支， + // 不是这道新闸门），确认新闸门没有误伤跟证据无关的维度 + const target = writeTarget(runDir, 'rn_a', 1, { scores: { code_quality: 90 } }); + const v = await gc.executeContract(contract, target, { runDir }); + const codeQuality = v.sections.benchmark.dimensions.find((d) => d.name === 'code_quality'); + expect(codeQuality.ok).toBe(true); + expect(codeQuality.evidence_required).toBeUndefined(); + }); + + it('evidence 是空字符串 / 非字符串时按"没给"处理，不当真', async () => { + const runDir = newRunDir(); + const target = writeTarget(runDir, 'rn_a', 1, { + scores: { hero_first_impression: 100, code_quality: 90 }, + evidence: { hero_first_impression: ' ' }, + }); + const v = await gc.executeContract(contract, target, { runDir }); + expect(v.ok).toBe(false); + expect(v.failures.some((f) => f.case === 'hero_first_impression')).toBe(true); + }); + + it('证据也认姊妹判词文件 attempt-.json 的 dimensions[].evidence——benchmark-validator/agent.md 早就在教这个写法', async () => { + const runDir = newRunDir(); + // target 是 attempt-1.scores.json（只有分数，Step 3 的写法） + const target = writeTarget(runDir, 'rn_a', '1.scores', { + scores: { hero_first_impression: 100, code_quality: 90 }, + }); + // 同目录的姊妹文件 attempt-1.json（判词文件，Step 5 的写法，带逐维度 evidence） + writeTarget(runDir, 'rn_a', 1, { + schema: 'work-validator/v1', validator: 'benchmark-validator', verdict: 'passed', + dimensions: [ + { name: 'hero_first_impression', score: 100, ok: true, evidence: 'Playwright 实测标题48px/标语16px，比例3:1' }, + { name: 'code_quality', score: 90, ok: true, evidence: 'eslint 零告警' }, + ], + }); + const v = await gc.executeContract(contract, target, { runDir }); + expect(v.ok).toBe(true); + expect(v.failures).toEqual([]); + }); + + it('姊妹判词文件不存在 / 不是合法 JSON 时不拖垮打分，按"没有证据"正常走失败分支', async () => { + const runDir = newRunDir(); + const target = writeTarget(runDir, 'rn_a', '1.scores', { + scores: { hero_first_impression: 100, code_quality: 90 }, + }); + // 故意不写 attempt-1.json（姊妹判词文件缺失） + const v = await gc.executeContract(contract, target, { runDir }); + expect(v.ok).toBe(false); + expect(v.failures.some((f) => f.case === 'hero_first_impression')).toBe(true); + }); +}); + // ──────────────────────────────────────────────────────────────── describe('执行器：test_data', () => { it('通过（subset 模式忽略额外键）', async () => { diff --git a/server/tests/unit/graph-run-route-io.test.js b/server/tests/unit/graph-run-route-io.test.js index ca16df0..871d994 100644 --- a/server/tests/unit/graph-run-route-io.test.js +++ b/server/tests/unit/graph-run-route-io.test.js @@ -168,4 +168,45 @@ describe('GET /api/graph-runs/:id — R-19 输入输出三字段', () => { // 幂等：再查一次不会重新分配一个不同的项目根 expect(engine.ensureProjectRoot({ run, companyId, nodeRunId: 'rn-fake-2' }).root_path).toBe(rec.root_path); }); + + // 35 号文档提案 B：控制台「这份验收有没有证据支撑」要读的数据——只读 + // goal-contract.js 已经落盘的判决，不重新计算。 + it('benchmarkVerdicts：没有任何判决落盘时是空数组（不是整块降级）', async () => { + const r = await fetch(`${base}/api/graph-runs/${runId}`); + const d = await r.json(); + expect(d.benchmarkVerdicts).toEqual([]); + }); + + it('benchmarkVerdicts：落过判决之后，逐维度的 evidence_required/ok 原样透传给前端', async () => { + const gc = require('../../services/goal-contract'); + const run = wdb.getRun(runId); + const runDir = gc.resolveRunDir(companyId, run.root_run_id || run.id); + + const verdict = { + verdict_id: 'vd-test-1', ok: false, fail_reason: 'acceptance_not_met', + goal_id: 'G-1', run_id: run.id, node_run_id: 'rn-benchmark-1', attempt: 0, + contract_hash: 'x', target: 'x', created_at: new Date().toISOString(), + sections: { + benchmark: { + ok: false, + dimensions: [ + { name: 'hero_first_impression', score: 100, ok: false, evidence_required: true }, + { name: 'code_quality', score: 90, ok: true }, + ], + }, + }, + }; + gc.persistVerdict(runDir, verdict); + + const r = await fetch(`${base}/api/graph-runs/${runId}`); + const d = await r.json(); + expect(d.benchmarkVerdicts).toHaveLength(1); + const v = d.benchmarkVerdicts[0]; + expect(v.nodeRunId).toBe('rn-benchmark-1'); + expect(v.ok).toBe(false); + expect(v.dimensions).toEqual([ + { name: 'hero_first_impression', ok: false, score: 100, evidenceRequired: true }, + { name: 'code_quality', ok: true, score: 90, evidenceRequired: false }, + ]); + }); }); diff --git a/system/agents/benchmark-validator/agent.md b/system/agents/benchmark-validator/agent.md index 9be738e..67bb03e 100644 --- a/system/agents/benchmark-validator/agent.md +++ b/system/agents/benchmark-validator/agent.md @@ -76,9 +76,29 @@ cat $R/artifacts/goal-benchmark.json # 维度、权重、及格线、rubric 对每一个维度： 1. 按 `evidence_hint` 去被测产物里找**具体证据**（文件 + 位置 + 原文）。 + **`evidence_hint` 里出现"打开浏览器 / 视口 / 开发者工具 / Console / Network / + 截图"这类字样时，别自己临时想办法**（正则解析源码文本不算，那是猜）—— + 仓库里有一份现成的真实浏览器检查工具： + + ```bash + node scripts/visual-acceptance-check.mjs overflow --file --widths 375,768,1440 + node scripts/visual-acceptance-check.mjs style --file --selector --prop font-size + node scripts/visual-acceptance-check.mjs console --file + node scripts/visual-acceptance-check.mjs contrast --fg "#111111" --bg "#ffffff" + ``` + + 每条命令跑真实 headless Chromium（`overflow`/`style`/`console`）或做真实 WCAG + 计算（`contrast`），一行 JSON 打到 stdout。退出码 0 才代表检查真的跑成了—— + 把这行 JSON（或从中摘出的关键数字）原样写进下面 Step 5 判词文件的 + `dimensions[].evidence`；退出码非 0 时不能拿输出（如果有）冒充证据。 + 工具详细说明见脚本文件头部注释。 2. 对照 `rubric` 的分档给分。**rubric 里没有的档位就近取整，不要自创评分体系。** 3. 找不到证据 → **给 0 分**，理由写「未找到证据」。 > 🚨 **找不到证据不等于默认及格。** 反过来也一样：不许因为"看起来做了不少"就给高分。 + > 🚨 **`evidence_hint` 命中上面那类关键词的维度，执行器会强制要求这份证据** + > （`goal-contract.js` 的 evidence 闸门，认这份判词文件 `dimensions[].evidence`， + > 或 `attempt-.scores.json` 里的 `evidence.`，两处都行，取其一）—— + > 光给分数没有证据，这一维直接判不过，哪怕分数本身达标。 4. 维度名**逐字照抄**，一个字都不能改——执行器按名字取分，名字对不上就是 `missing score in target`，该维度直接判不过，而你根本不会看到这个错误。 diff --git a/web/src/components/graph-console/GraphRunPane.vue b/web/src/components/graph-console/GraphRunPane.vue index 664e699..975c0c0 100644 --- a/web/src/components/graph-console/GraphRunPane.vue +++ b/web/src/components/graph-console/GraphRunPane.vue @@ -105,6 +105,8 @@ const companyRoot = ref(null); // 不要合并：graph_files 只索引节点私有目录，projectRoot 才是代码真正写的地方。 // 没有记录（节点还没跑到 ensureProjectRoot 那一步）时保持 null。 const projectRoot = ref(null); +// 35 号文档提案 B：这个节点最近一次 benchmark 判决按维度标了有没有证据支撑，纯展示。 +const benchmarkVerdicts = ref([]); const events = ref([]); // SSE 事件缓冲，供详情面板的「运行日志」tab 消费 const selectedDefId = ref(null); const loading = ref(false); @@ -213,6 +215,7 @@ async function load() { taskFiles.value = Array.isArray(j.taskFiles) ? j.taskFiles : null; companyRoot.value = j.companyRoot || null; projectRoot.value = j.projectRoot || null; + benchmarkVerdicts.value = Array.isArray(j.benchmarkVerdicts) ? j.benchmarkVerdicts : []; mode.value = j.run?.execution_mode || 'manual'; push(); } catch (e) { @@ -232,6 +235,7 @@ function push() { // R-19：右栏「输入输出」tab 的实例侧数据，与 nodes/edges 走同一条 load taskFiles: taskFiles.value, companyRoot: companyRoot.value, projectRoot: projectRoot.value, + benchmarkVerdicts: benchmarkVerdicts.value, }); } diff --git a/web/src/components/graph-console/NodeDetailPanel.vue b/web/src/components/graph-console/NodeDetailPanel.vue index cef4961..7894cfb 100644 --- a/web/src/components/graph-console/NodeDetailPanel.vue +++ b/web/src/components/graph-console/NodeDetailPanel.vue @@ -102,6 +102,27 @@ + + +

回退记录（{{ myLoops.length }}）

  • @@ -339,6 +360,8 @@ const props = defineProps({ companyId: { type: String, default: '' }, // 「LLM 原文」tab 要它拼端点：/api/graph-runs/:runId/nodes/:nodeRunId/transcript runId: { type: String, default: '' }, + // 35 号文档提案 B：GET /api/graph-runs/:id 的 benchmarkVerdicts 原样转发 + benchmarkVerdicts: { type: Array, default: () => [] }, }); defineEmits(['dive']); @@ -411,6 +434,14 @@ const myLoops = computed(() => ? props.loopRecords.filter((r) => r.from_node_run_id === props.node.id || r.to_node_run_id === props.node.id) : []); +// 35 号文档提案 B：这个节点最近一次的 benchmark 判决（同一节点可能跑过多轮 attempt， +// 取最新那条——index.ndjson 是写入序，最后一条就是最近一次） +const myVerdict = computed(() => { + if (!props.node) return null; + const mine = props.benchmarkVerdicts.filter((v) => v.nodeRunId === props.node.id); + return mine.length ? mine[mine.length - 1] : null; +}); + /** * 本节点这一轮实际拿到的「应该如何修复」——它就是下一轮输入与原始产出的差量。 * @@ -683,6 +714,18 @@ button.path.as-link:focus-visible { outline:2px solid #1d4ed8; outline-offset:1p .c-pill.elsewhere { background:#f4f4f6; color:#666; border-color:#dcdce2; cursor:help; } .c-pill.opt { opacity:.85; } +.verdict-dims { list-style:none; padding:0; margin:0 0 4px; } +.verdict-dims > li { + display:flex; align-items:center; gap:8px; font-size:11px; + border:1px solid #e4e4e8; background:#fafafa; border-radius:5px; padding:5px 9px; margin-bottom:5px; +} +.verdict-dims > li.warn { border-color:#f0dcc0; background:#fffaf3; } +.dim-name { font-family: ui-monospace, monospace; color:#333; } +.dim-score { color:#888; margin-left:auto; } +.dim-flag { font-size:10px; border-radius:8px; padding:1px 7px; white-space:nowrap; } +.dim-flag.ok { background:#e8f6ec; color:#1a6b38; } +.dim-flag.missing { background:#fff4e5; color:#8a4b00; } + .kv dd.wrap { white-space:pre-wrap; color:#444; } .list { padding-left:16px; margin:0; } .muted { color:#999; font-size:12px; } diff --git a/web/src/views/GraphConsole.vue b/web/src/views/GraphConsole.vue index 2a86a4b..7d2733f 100644 --- a/web/src/views/GraphConsole.vue +++ b/web/src/views/GraphConsole.vue @@ -162,6 +162,7 @@ :nodes="selectedState.nodes || []" :edges="selectedState.edges || []" :files="selectedState.taskFiles || null" + :benchmark-verdicts="selectedState.benchmarkVerdicts || []" :company-root="selectedState.companyRoot || null" :company-id="companyId" :run-id="selectedRunId || ''" -- libgit2 0.21.4