graph-engine-wallclock-watchdog.test.js
3.52 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
/**
* Unit Test — 墙钟预算看门狗（`sweepWallclockBudgets`）
*
* 挡的是什么：`checkBudget` 只在 `advanceOne`/`advanceBatch` 拾到 ready 节点时才查——
* manual 模式没人点「下一步」、或 auto 驱动循环随进程重启丢失时，`run_status='running'`
* 的 run 从此再也不会触发一次 checkBudget，wallclock_ms 超没超都没人知道。
* 真事：T-msx6toej/run_56ea2c598153，manual 模式，卡 135 小时才被人工发现。
*/
import { describe, it, expect, afterAll, beforeEach } from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
const TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'lemonboss-wc-watchdog-'));
process.env.GRAPH_DB_PATH = path.join(TMP, 'graph.sqlite');
const engine = require('../../services/graph-engine');
const wdb = require('../../services/work-graph-db');
const HOUR = 3600_000;
const budget = (spent) => JSON.stringify({ wallclock_ms: 2 * HOUR, total_agent_calls: 100, spent });
function seedRun(id, spentObj, status = 'running') {
const db = wdb.getDb();
const t = '2026-08-17T12:06:00.000Z';
db.prepare(`INSERT INTO graph_runs
(id, graph_kind, graph_id, company_id, root_run_id, execution_mode, run_status, budget_json, created_at, updated_at)
VALUES (?, 'org', 'g1', 'co1', ?, 'manual', ?, ?, ?, ?)`)
.run(id, id, status, budget(spentObj), t, t);
return wdb.getRun(id);
}
describe('墙钟看门狗 · sweepWallclockBudgets', () => {
afterAll(() => fs.rmSync(TMP, { recursive: true, force: true }));
beforeEach(() => wdb.getDb().prepare('DELETE FROM graph_runs').run());
it('没人再推进、wallclock 早超的 running root run：扫一遍就被挂起', () => {
// started_at 是 6 天前的绝对时刻，没人点过「下一步」——checkBudget 从没被调过
seedRun('r_stale', { agent_calls: 4, tokens: 53802, started_at: '2026-08-17T12:06:06.128Z' }, 'running');
const out = engine.sweepWallclockBudgets();
expect(out.suspended).toEqual(['r_stale']);
const after = wdb.getRun('r_stale');
expect(after.run_status).toBe('blocked');
expect(after.block_reason).toBe('budget-exhausted');
});
it('还在预算内的 running run：扫一遍原地不动', () => {
seedRun('r_fresh', { agent_calls: 1, tokens: 100, active_ms: 0, started_at: null }, 'running');
// 这一条本身没有 started_at（从未真正计时过），wallclockElapsedMs = active_ms = 0
const out = engine.sweepWallclockBudgets();
expect(out.suspended).toEqual([]);
expect(wdb.getRun('r_fresh').run_status).toBe('running');
});
it('挂起期间（blocked）不算「还在跑」——只扫 running，不会被误判', () => {
seedRun('r_blocked', { agent_calls: 7, started_at: '2026-08-17T12:06:06.128Z' }, 'blocked');
const out = engine.sweepWallclockBudgets();
expect(out.scanned).toBe(0);
expect(wdb.getRun('r_blocked').run_status).toBe('blocked');
});
it('没配 wallclock_ms 的 run：不在扫描范围内', () => {
const db = wdb.getDb();
db.prepare(`INSERT INTO graph_runs
(id, graph_kind, graph_id, company_id, root_run_id, execution_mode, run_status, budget_json, created_at, updated_at)
VALUES ('r_nowc','org','g1','co1','r_nowc','manual','running', ?, ?, ?)`)
.run(JSON.stringify({ total_agent_calls: 10, spent: { agent_calls: 1 } }), '2026-08-17T12:06:00.000Z', '2026-08-17T12:06:00.000Z');
const out = engine.sweepWallclockBudgets();
expect(out.scanned).toBe(0);
expect(wdb.getRun('r_nowc').run_status).toBe('running');
});
});