Commit 9a16416ac63a2e9113c8aa6065f67f13f042d34b
1 parent
1eadf743
fix(graph-run): 补上 POST /:id/budget 端点,给 blocked run 追加预算
run_b962ea87f58f 卡在 wallclock_ms 耗尽的锚点未结清问题上,需要人工追加预算 才能继续。extendBudget 只增不减、只加到已配额的维度上,追加时顺带结清卡死的 墙钟锚点(否则 startRunSync 一启动就把挂起时长记进新预算里)。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Showing
5 changed files
with
165 additions
and
1 deletions
server/routes/graph-run.js
| ... | ... | @@ -328,6 +328,39 @@ router.post('/:id/nodes/:nodeId/retry', h((req, res) => { |
| 328 | 328 | res.json(engine.retryNode(run.id, nodeRun.id)); |
| 329 | 329 | })); |
| 330 | 330 | |
| 331 | +/** | |
| 332 | + * POST /api/graph-runs/:id/budget — 「追加预算」(运维操作,不在 §四.C 五个操作之列) | |
| 333 | + * body: { total_agent_calls?, total_tokens?, wallclock_ms?, resume?: boolean } | |
| 334 | + * 可用条件:run_status = 'blocked' | |
| 335 | + * | |
| 336 | + * 只在这一个状态开放:预算耗尽 / 墙钟看门狗(`sweepWallclockBudgets`)打到 blocked 的 run | |
| 337 | + * 才需要人工追加额度,其它状态下加预算没有对应的业务场景(想在 running 时顺手提高上限, | |
| 338 | + * 应该是另一个决策要不要做的问题,这里不顺手放开)。 | |
| 339 | + * `resume: true` 时追加完立即调 `/start` 同一条路径拉回 running——两步拆成一个请求, | |
| 340 | + * 免得调用方在这两步之间手动再打一次 `/start`。 | |
| 341 | + */ | |
| 342 | +router.post('/:id/budget', h((req, res) => { | |
| 343 | + const run = loadRun(req, res); | |
| 344 | + if (!run) return; | |
| 345 | + if (!ensureStatus(res, run, '追加预算', ['blocked'])) return; | |
| 346 | + | |
| 347 | + const { total_agent_calls, total_tokens, wallclock_ms, resume } = req.body || {}; | |
| 348 | + let out; | |
| 349 | + try { | |
| 350 | + out = engine.extendBudget(run.id, { total_agent_calls, total_tokens, wallclock_ms }); | |
| 351 | + } catch (err) { | |
| 352 | + return res.status(400).json({ error: err.message }); | |
| 353 | + } | |
| 354 | + | |
| 355 | + if (!resume) return res.json(out); | |
| 356 | + | |
| 357 | + // 复用「启动」同一条路径(admitOrRequeue 的坑满退回队列、清软信号……一样都不重写) | |
| 358 | + const fresh = loadRun(req, res); | |
| 359 | + if (!fresh) return; | |
| 360 | + const { driver, ...startOut } = engine.startRun(fresh.id, {}); | |
| 361 | + res.json({ ...out, resumed: { ...startOut, auto: Boolean(driver) } }); | |
| 362 | +})); | |
| 363 | + | |
| 331 | 364 | // ──────────────────────────────────────────────────────────────── |
| 332 | 365 | // 只读查询类端点 |
| 333 | 366 | // ──────────────────────────────────────────────────────────────── | ... | ... |
server/services/graph-engine.js
| ... | ... | @@ -2845,6 +2845,60 @@ function spendBudget(run, { agentCalls = 0, tokens = 0 } = {}) { |
| 2845 | 2845 | } |
| 2846 | 2846 | |
| 2847 | 2847 | /** |
| 2848 | + * 运维操作:给一条 blocked 的 run 追加预算上限,为「启动」铺路。 | |
| 2849 | + * | |
| 2850 | + * 「只增不减」(§四.A.3 的既有约定)在这里延伸成:只吃正数增量、只加到已经配了额度的 | |
| 2851 | + * 那一维上(`b[key] == null` = 那一维本来就不限额,追加没有意义,忽略而不是凭空开一维新限额)。 | |
| 2852 | + * | |
| 2853 | + * 顺手结清卡死的墙钟锚点——不做这一步的话,`startRunSync` 一「启动」就会走 | |
| 2854 | + * `settleWallclock(runId,'running')`:`spent.started_at` 非空会被判成「树里已经有别的 run | |
| 2855 | + * 在跑,别重锚」(见该函数注释),于是沿用的还是 blocked 之前那个旧锚点,预算刚加完 | |
| 2856 | + * 下一次 `checkBudget` 立刻用「挂起等待的时长」把新加的额度吃光,run 秒进秒出。 | |
| 2857 | + * 这里手动做一次 `settleWallclock` 本该在「离开 running」那一刻做的事:把锚点到现在的 | |
| 2858 | + * 时长并进 `active_ms`、断开锚点,让「启动」重新开一段干净的计时。 | |
| 2859 | + */ | |
| 2860 | +function extendBudget(runId, patch = {}) { | |
| 2861 | + const run = getRun(runId); | |
| 2862 | + if (!run) throw new Error(`graph_runs 不存在: ${runId}`); | |
| 2863 | + const held = readBudget(run); | |
| 2864 | + if (!held) throw new Error(`run ${runId} 没有挂 budget_json,没有预算可加`); | |
| 2865 | + | |
| 2866 | + const b = held.budget; | |
| 2867 | + if (!b.spent) b.spent = {}; | |
| 2868 | + const before = { ...b, spent: { ...b.spent } }; | |
| 2869 | + | |
| 2870 | + const keys = ['total_agent_calls', 'total_tokens', 'wallclock_ms']; | |
| 2871 | + const applied = {}; | |
| 2872 | + for (const key of keys) { | |
| 2873 | + if (patch[key] == null) continue; | |
| 2874 | + if (typeof patch[key] !== 'number' || !(patch[key] > 0)) { | |
| 2875 | + throw new Error(`${key} 增量必须是正数,收到: ${patch[key]}`); | |
| 2876 | + } | |
| 2877 | + if (b[key] == null) continue; // 该维度本来就不限额,追加没有意义 | |
| 2878 | + b[key] += patch[key]; | |
| 2879 | + applied[key] = patch[key]; | |
| 2880 | + } | |
| 2881 | + if (Object.keys(applied).length === 0) { | |
| 2882 | + throw new Error('没有任何一维预算被追加(增量缺失,或目标维度本来就不限额)'); | |
| 2883 | + } | |
| 2884 | + | |
| 2885 | + if (b.spent.started_at) { | |
| 2886 | + b.spent.active_ms = (b.spent.active_ms || 0) + Math.max(0, Date.now() - Date.parse(b.spent.started_at)); | |
| 2887 | + b.spent.started_at = null; | |
| 2888 | + } | |
| 2889 | + | |
| 2890 | + getDb() | |
| 2891 | + .prepare(`UPDATE graph_runs SET budget_json = ?, updated_at = ? WHERE id = ?`) | |
| 2892 | + .run(JSON.stringify(b), now(), held.rootRunId); | |
| 2893 | + | |
| 2894 | + sse.emitEvent(runId, 'budget.extended', '运维追加预算并结清墙钟锚点', { | |
| 2895 | + root_run_id: held.rootRunId, applied, before, after: b, | |
| 2896 | + }, { ref: held.rootRunId }); | |
| 2897 | + | |
| 2898 | + return { ok: true, rootRunId: held.rootRunId, applied, budget: b }; | |
| 2899 | +} | |
| 2900 | + | |
| 2901 | +/** | |
| 2848 | 2902 | * 资源类原因(预算耗尽 / 账户欠费)→ **整棵 run 树挂起等人工**,不是失败(§四.A.3 / 27-P4 §4.4): |
| 2849 | 2903 | * root run → blocked,树上所有非终态 run 一律置 pause_requested=1。 |
| 2850 | 2904 | * 同样只做一次 listRunTree 索引查询。 |
| ... | ... | @@ -4495,6 +4549,7 @@ module.exports = { |
| 4495 | 4549 | // 任务级总预算(§四.A.3) |
| 4496 | 4550 | checkBudget, |
| 4497 | 4551 | spendBudget, |
| 4552 | + extendBudget, | |
| 4498 | 4553 | settleWallclock, |
| 4499 | 4554 | wallclockElapsedMs, |
| 4500 | 4555 | suspendRunTree, | ... | ... |
server/tests/unit/graph-console-node-signals.test.js
| ... | ... | @@ -77,6 +77,7 @@ describe('控制台节点信号(W12-UI 读回层)', () => { |
| 77 | 77 | 'loop.terminated': { cap: 'edge' }, |
| 78 | 78 | 'loop.cross_layer': {}, |
| 79 | 79 | 'budget.spent': { delta: { agent_calls: 1, tokens: 42 } }, |
| 80 | + 'budget.extended': { applied: { wallclock_ms: 14_400_000 } }, | |
| 80 | 81 | 'plan.call': { agent: 'workgraph-planning', round: 1 }, |
| 81 | 82 | 'plan.done': { work_graph_id: 'wg-1', round: 1, nodes: 3, edges: 2 }, |
| 82 | 83 | 'plan.rejected': { round: 1, errors: [{ rule: 'reference', detail: '图不存在' }] }, | ... | ... |
server/tests/unit/graph-engine-extend-budget.test.js
0 → 100644
| 1 | +/** | |
| 2 | + * Unit Test — engine.extendBudget:给 blocked run 追加预算 + 结清卡死的墙钟锚点 | |
| 3 | + * | |
| 4 | + * 背景:run_b962ea87f58f 因 wallclock_ms 耗尽被 block,`spent.started_at` 卡在 block | |
| 5 | + * 之前的旧时刻没有断开(该 run 被 block 那一刻树里还有别的 run 在跑,settleWallclock | |
| 6 | + * 判定「不结算」,见 budget-wallclock-active.test.js 那条同名用例)。 | |
| 7 | + * 只加 wallclock_ms 上限不够:`startRunSync` 一启动就走 settleWallclock('running'), | |
| 8 | + * 见到 started_at 非空会拒绝重锚,新加的额度立刻被「挂起等待的时长」吃光。 | |
| 9 | + * 这个测试钉住 extendBudget 必须同时把这个锚点结清。 | |
| 10 | + */ | |
| 11 | +import { describe, it, expect, afterAll, beforeEach } from 'vitest'; | |
| 12 | +import fs from 'fs'; | |
| 13 | +import os from 'os'; | |
| 14 | +import path from 'path'; | |
| 15 | + | |
| 16 | +const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'lemonboss-extend-budget-')); | |
| 17 | +process.env.GRAPH_DB_PATH = path.join(TMP, 'graph.sqlite'); | |
| 18 | + | |
| 19 | +const engine = require('../../services/graph-engine'); | |
| 20 | +const wdb = require('../../services/work-graph-db'); | |
| 21 | + | |
| 22 | +const HOUR = 3600_000; | |
| 23 | + | |
| 24 | +function seedRun(id, budgetObj, status = 'blocked') { | |
| 25 | + const db = wdb.getDb(); | |
| 26 | + const t = '2026-08-23T05:23:52.886Z'; | |
| 27 | + db.prepare(`INSERT INTO graph_runs | |
| 28 | + (id, graph_kind, graph_id, company_id, root_run_id, execution_mode, run_status, budget_json, created_at, updated_at) | |
| 29 | + VALUES (?, 'org', 'g1', 'co1', ?, 'manual', ?, ?, ?, ?)`) | |
| 30 | + .run(id, id, status, JSON.stringify(budgetObj), t, t); | |
| 31 | + return wdb.getRun(id); | |
| 32 | +} | |
| 33 | + | |
| 34 | +describe('extendBudget · 追加预算并结清卡死的墙钟锚点', () => { | |
| 35 | + afterAll(() => fs.rmSync(TMP, { recursive: true, force: true })); | |
| 36 | + beforeEach(() => wdb.getDb().prepare('DELETE FROM graph_runs').run()); | |
| 37 | + | |
| 38 | + it('run_b962ea87f58f 的真实形态:追加后立即可用,且锚点被断开', () => { | |
| 39 | + const stuckStartedAt = '2026-08-23T05:23:52.886Z'; // 距 now() 已远超 2h | |
| 40 | + const run = seedRun('r_stuck', { | |
| 41 | + total_agent_calls: 100, total_tokens: 5_000_000, wallclock_ms: 2 * HOUR, | |
| 42 | + spent: { agent_calls: 35, tokens: 249_324, active_ms: 0, started_at: stuckStartedAt }, | |
| 43 | + }); | |
| 44 | + | |
| 45 | + // 追加前:卡死的锚点让墙钟早就判耗尽 | |
| 46 | + expect(engine.checkBudget(run).exhausted).toContain('wallclock_ms'); | |
| 47 | + | |
| 48 | + const out = engine.extendBudget(run.id, { wallclock_ms: 4 * HOUR }); | |
| 49 | + expect(out.ok).toBe(true); | |
| 50 | + expect(out.budget.wallclock_ms).toBe(6 * HOUR); | |
| 51 | + expect(out.budget.spent.started_at).toBeNull(); // 锚点必须断开 | |
| 52 | + expect(out.budget.spent.active_ms).toBeGreaterThan(0); // 挂起时长已并入 active_ms | |
| 53 | + | |
| 54 | + const fresh = wdb.getRun('r_stuck'); | |
| 55 | + expect(engine.checkBudget(fresh).ok).toBe(true); | |
| 56 | + | |
| 57 | + // 「启动」重锚不会被旧锚点拒绝——settleWallclock('running') 应该重新开一段计时 | |
| 58 | + const settled = engine.settleWallclock('r_stuck', 'running'); | |
| 59 | + expect(settled.started_at).toBeTruthy(); | |
| 60 | + }); | |
| 61 | + | |
| 62 | + it('目标维度本来就不限额时忽略该维度;一维都没追加成就报错', () => { | |
| 63 | + seedRun('r_partial', { total_agent_calls: 100, spent: { agent_calls: 5 } }); | |
| 64 | + // wallclock_ms 本来就没配额度,追加它是 no-op;total_agent_calls 有配额,真正生效 | |
| 65 | + const out = engine.extendBudget('r_partial', { wallclock_ms: HOUR, total_agent_calls: 50 }); | |
| 66 | + expect(out.applied).toEqual({ total_agent_calls: 50 }); | |
| 67 | + expect(out.budget.total_agent_calls).toBe(150); | |
| 68 | + expect(out.budget.wallclock_ms).toBeUndefined(); | |
| 69 | + | |
| 70 | + seedRun('r_none', { total_agent_calls: 100, spent: {} }); | |
| 71 | + expect(() => engine.extendBudget('r_none', { wallclock_ms: HOUR })).toThrow(); | |
| 72 | + }); | |
| 73 | +}); | ... | ... |
web/src/components/graph-console/node-signals.js
| ... | ... | @@ -20,7 +20,7 @@ export const RUN_EVENT_KINDS = [ |
| 20 | 20 | 'agent.call', 'node.output', 'node.error', |
| 21 | 21 | 'node.triage', 'node.contract', 'node.boundary', 'node.subgraph', |
| 22 | 22 | 'loop.registered', 'loop.terminated', 'loop.cross_layer', |
| 23 | - 'budget.spent', | |
| 23 | + 'budget.spent', 'budget.extended', | |
| 24 | 24 | 'plan.call', 'plan.done', 'plan.rejected', |
| 25 | 25 | ]; |
| 26 | 26 | |
| ... | ... | @@ -94,6 +94,8 @@ export function describeEvent(e) { |
| 94 | 94 | return '跨层定点回退'; |
| 95 | 95 | case 'budget.spent': |
| 96 | 96 | return `预算扣减 +${(d.delta && d.delta.agent_calls) || 0} 次调用 / +${(d.delta && d.delta.tokens) || 0} tokens`; |
| 97 | + case 'budget.extended': | |
| 98 | + return `运维追加预算:${Object.entries(d.applied || {}).map(([k, v]) => `${k} +${v}`).join('、') || '?'}`; | |
| 97 | 99 | case 'plan.call': |
| 98 | 100 | return `调 ${d.agent || 'workgraph-planning'} 规划子图(第 ${d.round ?? '?'} 轮)`; |
| 99 | 101 | case 'plan.done': | ... | ... |