Commit ef20ceb6efa1a520056f7b07082a912557719ea6
1 parent
f64ddc23
fix(graph-engine): reopenRunForRetry/孤儿清扫重挂 auto 驱动循环 (T-mtjh8sdt)
reopenRunForRetry 与 _reapStaleRunningNodes(开机孤儿清扫 + 45号文档持续性存活 扫描共用)把 run 从 blocked/ready 拉回 running 走的是裸 admitOrRequeue,不是 startRun,只改了状态从没重新挂上 runAuto 自驱动循环——手动/自动重试推一步后没人 接着推,agentRunId 仍是 null,backward 重试每次都在这里再卡一次。两处都补上 startRun 收尾那段逻辑:真的推成 running 且 execution_mode==='auto' 才挂 driver。 Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Showing
1 changed file
with
30 additions
and
0 deletions
server/services/graph-engine.js
| @@ -3204,6 +3204,7 @@ function stopWallclockWatchdog() { | @@ -3204,6 +3204,7 @@ function stopWallclockWatchdog() { | ||
| 3204 | */ | 3204 | */ |
| 3205 | function _reapStaleRunningNodes(rows, summary) { | 3205 | function _reapStaleRunningNodes(rows, summary) { |
| 3206 | const swept = []; | 3206 | const swept = []; |
| 3207 | + const runIdsToRedrive = new Set(); | ||
| 3207 | for (const row of rows) { | 3208 | for (const row of rows) { |
| 3208 | try { | 3209 | try { |
| 3209 | const run = getRun(row.run_id); | 3210 | const run = getRun(row.run_id); |
| @@ -3220,10 +3221,26 @@ function _reapStaleRunningNodes(rows, summary) { | @@ -3220,10 +3221,26 @@ function _reapStaleRunningNodes(rows, summary) { | ||
| 3220 | const out = { activated: [], backwardEdgeId: null, loop: null, contractDiff: null, crossLayer: null, reset: [], infraRerun: false }; | 3221 | const out = { activated: [], backwardEdgeId: null, loop: null, contractDiff: null, crossLayer: null, reset: [], infraRerun: false }; |
| 3221 | routeFailure(row.run_id, def, nodeRun, out, null); | 3222 | routeFailure(row.run_id, def, nodeRun, out, null); |
| 3222 | swept.push({ runId: row.run_id, nodeRunId: row.id, infraRerun: out.infraRerun }); | 3223 | swept.push({ runId: row.run_id, nodeRunId: row.id, infraRerun: out.infraRerun }); |
| 3224 | + runIdsToRedrive.add(row.run_id); | ||
| 3223 | } catch (err) { | 3225 | } catch (err) { |
| 3224 | console.error(`[graph-engine] _reapStaleRunningNodes node=${row.id} 异常: ${err.message}`); | 3226 | console.error(`[graph-engine] _reapStaleRunningNodes node=${row.id} 异常: ${err.message}`); |
| 3225 | } | 3227 | } |
| 3226 | } | 3228 | } |
| 3229 | + // T-mtjh8sdt 同一个洞的第二个入口:routeFailure 把节点重置回 ready/running 只改了状态, | ||
| 3230 | + // 不会重新挂上 runAuto 的自驱动循环(reopenRunForRetry 那次已经补过人工重试路径,这里 | ||
| 3231 | + // 是开机孤儿清扫 sweepOrphanedRunningNodes 与 45 号文档持续性存活扫描 | ||
| 3232 | + // sweepStaleRunningAgentCalls 共用的结算内层循环,同一缺陷的另一条触发路径)。 | ||
| 3233 | + // 异步补、不 await:sweepOrphanedRunningNodes 在服务器启动路径上同步调用,扫描可能 | ||
| 3234 | + // 命中好几个 run,不能让重新驱动的 agent 调用顶着启动耗时走;按 run 去重,避免同一个 | ||
| 3235 | + // run 里多个孤儿节点各自触发一次 runAuto。 | ||
| 3236 | + for (const runId of runIdsToRedrive) { | ||
| 3237 | + const after = getRun(runId); | ||
| 3238 | + if (after && after.run_status === 'running' && after.execution_mode === 'auto') { | ||
| 3239 | + runAuto(runId).catch((err) => { | ||
| 3240 | + console.error(`[graph-engine] 孤儿清扫重挂 auto 驱动异常 run=${runId}: ${err.message}`); | ||
| 3241 | + }); | ||
| 3242 | + } | ||
| 3243 | + } | ||
| 3227 | return { scanned: rows.length, swept }; | 3244 | return { scanned: rows.length, swept }; |
| 3228 | } | 3245 | } |
| 3229 | 3246 | ||
| @@ -3802,6 +3819,13 @@ function descendantRuns(run) { | @@ -3802,6 +3819,13 @@ function descendantRuns(run) { | ||
| 3802 | * | 3819 | * |
| 3803 | * @returns {void} 没有「失败」这个出口:要么推进了,要么被别人抢先推走了(都可以继续动节点), | 3820 | * @returns {void} 没有「失败」这个出口:要么推进了,要么被别人抢先推走了(都可以继续动节点), |
| 3804 | * 要么抛 RunSlotBusyError(整条重试作废,节点一个字都没动)。 | 3821 | * 要么抛 RunSlotBusyError(整条重试作废,节点一个字都没动)。 |
| 3822 | + * | ||
| 3823 | + * **auto 模式重挂驱动循环**(T-mtjh8sdt):这里把 run 从 blocked/ready 拉回 running 走的是 | ||
| 3824 | + * 裸 `admitOrRequeue`,不是 `startRun`——所以以前只改了状态,从没重新挂上 `runAuto` 的自驱动 | ||
| 3825 | + * 循环。手动 `nextStep` 推一步后没人接着推,agentRunId 仍是 null,backward 重试每次都会在 | ||
| 3826 | + * 这里再卡一次。`startRun` 的收尾那段(判 execution_mode==='auto' 才挂 driver)搬过来一份: | ||
| 3827 | + * 只在真的把 run 推成了 running 且 execution_mode 是 auto 时才挂,manual 模式或没推动(requeued/ | ||
| 3828 | + * lost)都不挂,语义与 startRun 对齐。 | ||
| 3805 | */ | 3829 | */ |
| 3806 | function reopenRunForRetry(runId, reopenRun) { | 3830 | function reopenRunForRetry(runId, reopenRun) { |
| 3807 | if (!reopenRun) return; | 3831 | if (!reopenRun) return; |
| @@ -3813,6 +3837,12 @@ function reopenRunForRetry(runId, reopenRun) { | @@ -3813,6 +3837,12 @@ function reopenRunForRetry(runId, reopenRun) { | ||
| 3813 | if (cur === 'blocked' || cur === 'ready') { | 3837 | if (cur === 'blocked' || cur === 'ready') { |
| 3814 | admitOrRequeue(runId, cur, 'running', { summary: `人工重试重开链路:${cur} → running` }); | 3838 | admitOrRequeue(runId, cur, 'running', { summary: `人工重试重开链路:${cur} → running` }); |
| 3815 | } | 3839 | } |
| 3840 | + const after = getRun(runId); | ||
| 3841 | + if (after && after.run_status === 'running' && after.execution_mode === 'auto') { | ||
| 3842 | + runAuto(runId).catch((err) => { | ||
| 3843 | + console.error(`[graph-engine] 人工重试重挂 auto 驱动异常 run=${runId}: ${err.message}`); | ||
| 3844 | + }); | ||
| 3845 | + } | ||
| 3816 | } | 3846 | } |
| 3817 | 3847 | ||
| 3818 | /** | 3848 | /** |