From 0c562d9d0b65a96611bc00f1b316436354a9306b Mon Sep 17 00:00:00 2001 From: yibo Date: Sun, 23 Aug 2026 11:19:19 +0800 Subject: [PATCH] fix(graph-engine): 补墙钟预算看门狗，堵 manual/auto 两条驱动路径失活后无人发现超时的口子 --- server/index.js | 14 ++++++++++++++ server/services/graph-engine.js | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ server/tests/unit/graph-engine-wallclock-watchdog.test.js | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 0 deletions(-) create mode 100644 server/tests/unit/graph-engine-wallclock-watchdog.test.js diff --git a/server/index.js b/server/index.js index 3fbba99..8fd9044 100644 --- a/server/index.js +++ b/server/index.js @@ -171,6 +171,20 @@ async function start() { } catch (err) { console.error('[Startup] 任务队列扫描失败:', err.message); } + // 27-P4 §4.4：墙钟预算看门狗。manual 模式没人点「下一步」、或 auto 模式的驱动循环 + // （纯内存 promise 链）随进程重启丢失时，run_status='running' 的 run 从此再也不会 + // 触发 checkBudget —— wallclock_ms 早超了也没人发现（T-msx6toej 卡 135 小时才被人工揪出）。 + // 起跑前先补一次全量扫描，再起 60s 兜底定时器，同 task-scheduler 的形状。 + try { + const engine = require('./services/graph-engine'); + const swept = engine.sweepWallclockBudgets(); + if (swept.suspended.length) { + console.log(`[Startup] ⏱️ 墙钟看门狗：挂起 ${swept.suspended.length} 条超时仍在跑的 run`); + } + engine.startWallclockWatchdog(); + } catch (err) { + console.error('[Startup] 墙钟看门狗启动失败:', err.message); + } const heartbeatService = require('./services/heartbeat-service'); heartbeatService.start(); // 公司级心跳 (Pipeline 自动驱动) diff --git a/server/services/graph-engine.js b/server/services/graph-engine.js index 7963c93..7e02dee 100644 --- a/server/services/graph-engine.js +++ b/server/services/graph-engine.js @@ -2693,6 +2693,75 @@ async function resumeBillingClearedRuns(opts = {}) { } /** + * 墙钟预算的兜底扫描——「7×24 无人值守缺看门狗」（27-P4 §4.4）的又一个具体实例， + * 这次挡的是 `wallclock_ms` 那一维而不是欠费。 + * + * `checkBudget` 只在 `advanceOne`/`advanceBatch` 拾到 ready 节点时才查（§四.A.3 的 + * 「拾到了才查」，即将真花钱时才查，不在空转时误判）——这对**正在被推进**的 run 没问题， + * 出问题的是**没人再推进它**的 run：manual 模式没人点「下一步」（`POST /graph-runs/:id/next`）， + * 或 auto 模式的驱动循环（`runAuto`）是纯内存 promise 链，进程一重启就消失、 + * 没有任何东西在启动时把它接回去。两条路径的共同症状是 `run_status='running'` + * 却再也没有任何调用会触发 checkBudget——wallclock_ms 早就超了，界面上却和正常在跑 + * 一模一样，直到有人手工发现。 + * + * 实例：`T-msx6toej`/`run_56ea2c598153`，manual 模式，2026-08-17 12:06 起跑， + * `wallclock_ms` 预算 2 小时，卡到 2026-08-23 才被发现——中途没有任何一次 checkBudget。 + * + * 只扫 root run：`budget_json` 恒挂在 root（`readBudget` 的口径，子 run 永远是 null）， + * 扫子 run 拿到的预算池和扫 root 是同一份，多扫一遍纯浪费。 + * + * 复用 `checkBudget` + `suspendRunTree`——跟 `advanceOne` 判耗尽时走的是同一条路径， + * 不发明第二种「超时」形态。 + * + * @param {object} [opts] + * @param {string} [opts.companyId] 只扫这一家公司；不传扫全部 + * @returns {{scanned: number, suspended: string[]}} + */ +function sweepWallclockBudgets(opts = {}) { + const rows = getDb() + .prepare( + opts.companyId + ? `SELECT * FROM graph_runs WHERE run_status = 'running' AND parent_run_id IS NULL + AND budget_json LIKE '%wallclock_ms%' AND company_id = ?` + : `SELECT * FROM graph_runs WHERE run_status = 'running' AND parent_run_id IS NULL + AND budget_json LIKE '%wallclock_ms%'` + ) + .all(...(opts.companyId ? [opts.companyId] : [])); + + const suspended = []; + for (const run of rows) { + try { + const gate = checkBudget(run); + if (!gate.ok && gate.exhausted.includes('wallclock_ms')) { + suspendRunTree(run); + suspended.push(run.id); + } + } catch (err) { + console.error(`[graph-engine] sweepWallclockBudgets run=${run.id} 异常: ${err.message}`); + } + } + return { scanned: rows.length, suspended }; +} + +const WALLCLOCK_SWEEP_INTERVAL_MS = 60 * 1000; +let wallclockSweepTimer = null; + +/** 起兜底定时器。`unref()`——不该拖着进程不退出；幂等——重复调只留一个（同 task-scheduler 的形状）。 */ +function startWallclockWatchdog(intervalMs = WALLCLOCK_SWEEP_INTERVAL_MS) { + if (wallclockSweepTimer) return wallclockSweepTimer; + wallclockSweepTimer = setInterval(() => { + try { sweepWallclockBudgets(); } catch (err) { console.error(`[graph-engine] 墙钟看门狗异常: ${err.message}`); } + }, intervalMs); + if (typeof wallclockSweepTimer.unref === 'function') wallclockSweepTimer.unref(); + return wallclockSweepTimer; +} + +function stopWallclockWatchdog() { + if (wallclockSweepTimer) clearInterval(wallclockSweepTimer); + wallclockSweepTimer = null; +} + +/** * 扣减（「只增不减」，§四.A.3）。始终写 root run 那一行。 * ponytail: 读-改-写不是原子的，当前单进程串行驱动（runAuto 一次只推一个节点）下够用； * 升级路径是把自增下沉成一条 `UPDATE ... SET budget_json = json_set(...)` 的 SQL。 @@ -4379,6 +4448,11 @@ module.exports = { setBillingGate, resumeBillingClearedRuns, + // 27-P4 §4.4：墙钟预算看门狗（manual 模式无人点「下一步」/ auto 驱动循环随进程重启丢失时的兜底） + sweepWallclockBudgets, + startWallclockWatchdog, + stopWallclockWatchdog, + // 重试级联判据（§四.C 末段） diffContracts, applyRetryCascade, diff --git a/server/tests/unit/graph-engine-wallclock-watchdog.test.js b/server/tests/unit/graph-engine-wallclock-watchdog.test.js new file mode 100644 index 0000000..f067c96 --- /dev/null +++ b/server/tests/unit/graph-engine-wallclock-watchdog.test.js @@ -0,0 +1,80 @@ +/** + * Unit Test — 墙钟预算看门狗（`sweepWallclockBudgets`） + * + * 挡的是什么：`checkBudget` 只在 `advanceOne`/`advanceBatch` 拾到 ready 节点时才查—— + * manual 模式没人点「下一步」、或 auto 驱动循环随进程重启丢失时，`run_status='running'` + * 的 run 从此再也不会触发一次 checkBudget，wallclock_ms 超没超都没人知道。 + * 真事：T-msx6toej/run_56ea2c598153，manual 模式，卡 135 小时才被人工发现。 + */ +import { describe, it, expect, afterAll, beforeEach } from 'vitest'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'lemonboss-wc-watchdog-')); +process.env.GRAPH_DB_PATH = path.join(TMP, 'graph.sqlite'); + +const engine = require('../../services/graph-engine'); +const wdb = require('../../services/work-graph-db'); + +const HOUR = 3600_000; +const budget = (spent) => JSON.stringify({ wallclock_ms: 2 * HOUR, total_agent_calls: 100, spent }); + +function seedRun(id, spentObj, status = 'running') { + const db = wdb.getDb(); + const t = '2026-08-17T12:06:00.000Z'; + db.prepare(`INSERT INTO graph_runs + (id, graph_kind, graph_id, company_id, root_run_id, execution_mode, run_status, budget_json, created_at, updated_at) + VALUES (?, 'org', 'g1', 'co1', ?, 'manual', ?, ?, ?, ?)`) + .run(id, id, status, budget(spentObj), t, t); + return wdb.getRun(id); +} + +describe('墙钟看门狗 · sweepWallclockBudgets', () => { + afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); + beforeEach(() => wdb.getDb().prepare('DELETE FROM graph_runs').run()); + + it('没人再推进、wallclock 早超的 running root run：扫一遍就被挂起', () => { + // started_at 是 6 天前的绝对时刻，没人点过「下一步」——checkBudget 从没被调过 + seedRun('r_stale', { agent_calls: 4, tokens: 53802, started_at: '2026-08-17T12:06:06.128Z' }, 'running'); + + const out = engine.sweepWallclockBudgets(); + + expect(out.suspended).toEqual(['r_stale']); + const after = wdb.getRun('r_stale'); + expect(after.run_status).toBe('blocked'); + expect(after.block_reason).toBe('budget-exhausted'); + }); + + it('还在预算内的 running run：扫一遍原地不动', () => { + seedRun('r_fresh', { agent_calls: 1, tokens: 100, active_ms: 0, started_at: null }, 'running'); + // 这一条本身没有 started_at（从未真正计时过），wallclockElapsedMs = active_ms = 0 + + const out = engine.sweepWallclockBudgets(); + + expect(out.suspended).toEqual([]); + expect(wdb.getRun('r_fresh').run_status).toBe('running'); + }); + + it('挂起期间（blocked）不算「还在跑」——只扫 running，不会被误判', () => { + seedRun('r_blocked', { agent_calls: 7, started_at: '2026-08-17T12:06:06.128Z' }, 'blocked'); + + const out = engine.sweepWallclockBudgets(); + + expect(out.scanned).toBe(0); + expect(wdb.getRun('r_blocked').run_status).toBe('blocked'); + }); + + it('没配 wallclock_ms 的 run：不在扫描范围内', () => { + const db = wdb.getDb(); + db.prepare(`INSERT INTO graph_runs + (id, graph_kind, graph_id, company_id, root_run_id, execution_mode, run_status, budget_json, created_at, updated_at) + VALUES ('r_nowc','org','g1','co1','r_nowc','manual','running', ?, ?, ?)`) + .run(JSON.stringify({ total_agent_calls: 10, spent: { agent_calls: 1 } }), '2026-08-17T12:06:00.000Z', '2026-08-17T12:06:00.000Z'); + + const out = engine.sweepWallclockBudgets(); + + expect(out.scanned).toBe(0); + expect(wdb.getRun('r_nowc').run_status).toBe('running'); + }); +}); -- libgit2 0.21.4