Commit a9c0f8bbbd4bcbe3d5b85938f3dad8802255201f
1 parent
01a9a58b
run_goal_to_completion: any unrouted exception now blocks the task, doesn't just crash
Found for real: a work_graph_execute.WorkGraphExecuteError ("API Error:
Response stalled mid-stream") wasn't in _ROUTABLE_EXCEPTIONS, so it
propagated uncaught and crashed the calling process entirely -- leaving the
task neither running nor blocked_pending_human, just stuck with
blocked_reason=NULL. run_scheduler_once's own try/except only protects
OTHER goals in the same batch from one goal's failure; it never applied the
task-level blocked_pending_human consequence either.
Add a catch-all mirroring task_execution._mark_blocked's existing fallback
for its own (Start-button) entry point, so both ways of running a task now
guarantee it always lands on a real terminal-ish status with the real error
message persisted, never a caller-crashes-and-the-task-is-just-stuck outcome
nothing can manual_resume from.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Showing
2 changed files
with
41 additions
and
7 deletions
src/scheduler.py
| @@ -177,9 +177,21 @@ def run_goal_to_completion(db_path, target_folder_path, goal_text): | @@ -177,9 +177,21 @@ def run_goal_to_completion(db_path, target_folder_path, goal_text): | ||
| 177 | run_task_lifecycle raises partway through -- needed to route | 177 | run_task_lifecycle raises partway through -- needed to route |
| 178 | CorruptPersistedStateError/ClassifierBlockedError/UsageLimitWaitTimeoutError | 178 | CorruptPersistedStateError/ClassifierBlockedError/UsageLimitWaitTimeoutError |
| 179 | to task.status='blocked_pending_human' (see module docstring's "Second | 179 | to task.status='blocked_pending_human' (see module docstring's "Second |
| 180 | - gap" note). Any OTHER exception still propagates uncaught, same as | ||
| 181 | - before -- run_scheduler_once's own try/except is what keeps one Goal's | ||
| 182 | - unexpected failure from stopping the rest. | 180 | + gap" note). |
| 181 | + | ||
| 182 | + **2026-08-04**: any OTHER exception (e.g. work_graph_execute. | ||
| 183 | + WorkGraphExecuteError from a real API hiccup -- found for real: a | ||
| 184 | + "Response stalled mid-stream" error crashed the calling process | ||
| 185 | + entirely) used to propagate uncaught, same as before this note -- | ||
| 186 | + `run_scheduler_once`'s own try/except only stops it from taking down | ||
| 187 | + OTHER goals in the same batch, it does NOT apply the task-level | ||
| 188 | + blocked_pending_human consequence either, so the task itself was left | ||
| 189 | + stuck in limbo: not running, not blocked, `blocked_reason` still NULL, | ||
| 190 | + nothing a human could `manual_resume` from the console. This mirrors | ||
| 191 | + the exact fallback `task_execution._mark_blocked()` already has for its | ||
| 192 | + own (Start-button) entry point -- both entry points now guarantee a | ||
| 193 | + task always lands on a real terminal-ish status with a real error | ||
| 194 | + detail, never a caller-crashes-and-the-task-is-just-stuck outcome. | ||
| 183 | """ | 195 | """ |
| 184 | conn = schema.connect(db_path) | 196 | conn = schema.connect(db_path) |
| 185 | conn.execute("PRAGMA busy_timeout = {}".format(_BUSY_TIMEOUT_MS)) | 197 | conn.execute("PRAGMA busy_timeout = {}".format(_BUSY_TIMEOUT_MS)) |
| @@ -194,6 +206,9 @@ def run_goal_to_completion(db_path, target_folder_path, goal_text): | @@ -194,6 +206,9 @@ def run_goal_to_completion(db_path, target_folder_path, goal_text): | ||
| 194 | blocked_reason = _blocked_reason_for(e) | 206 | blocked_reason = _blocked_reason_for(e) |
| 195 | _block_task(conn, task_id, blocked_reason, detail=str(e)) | 207 | _block_task(conn, task_id, blocked_reason, detail=str(e)) |
| 196 | return {"task_id": task_id, "final_status": "blocked_pending_human", "blocked_reason": blocked_reason} | 208 | return {"task_id": task_id, "final_status": "blocked_pending_human", "blocked_reason": blocked_reason} |
| 209 | + except Exception as e: | ||
| 210 | + _block_task(conn, task_id, "execution_error", detail=str(e)) | ||
| 211 | + return {"task_id": task_id, "final_status": "blocked_pending_human", "blocked_reason": "execution_error"} | ||
| 197 | return finish_task_lifecycle(conn, result, target_folder_path, goal_text) | 212 | return finish_task_lifecycle(conn, result, target_folder_path, goal_text) |
| 198 | finally: | 213 | finally: |
| 199 | conn.close() | 214 | conn.close() |
src/test_scheduler.py
| @@ -350,10 +350,29 @@ class ConsistencyCheckFailedRoutingTest(RoutableExceptionTestBase): | @@ -350,10 +350,29 @@ class ConsistencyCheckFailedRoutingTest(RoutableExceptionTestBase): | ||
| 350 | self._assert_task_blocked(result["task_id"], "prd_consistency_check_failed") | 350 | self._assert_task_blocked(result["task_id"], "prd_consistency_check_failed") |
| 351 | 351 | ||
| 352 | 352 | ||
| 353 | -class UnroutedExceptionStillPropagatesTest(RoutableExceptionTestBase): | ||
| 354 | - def test_other_exceptions_are_not_swallowed(self): | ||
| 355 | - with self.assertRaises(RuntimeError): | ||
| 356 | - self._run_with_raising_lifecycle(RuntimeError("something else entirely")) | 353 | +class UnroutedExceptionStillBlocksTaskTest(RoutableExceptionTestBase): |
| 354 | + def test_other_exceptions_block_the_task_instead_of_leaving_it_stuck(self): | ||
| 355 | + # 2026-08-04 regression: this used to propagate uncaught, leaving | ||
| 356 | + # the task neither running nor blocked_pending_human -- a real | ||
| 357 | + # work_graph_execute.WorkGraphExecuteError (API "Response stalled | ||
| 358 | + # mid-stream") crashed the calling process and left the task stuck | ||
| 359 | + # in limbo, nothing a human could resume from the console. Mirrors | ||
| 360 | + # task_execution._mark_blocked's same fallback for its own | ||
| 361 | + # (Start-button) entry point. | ||
| 362 | + result = self._run_with_raising_lifecycle(RuntimeError("something else entirely")) | ||
| 363 | + self.assertEqual(result["final_status"], "blocked_pending_human") | ||
| 364 | + self.assertEqual(result["blocked_reason"], "execution_error") | ||
| 365 | + self._assert_task_blocked(result["task_id"], "execution_error") | ||
| 366 | + | ||
| 367 | + def test_real_exception_message_is_persisted_for_diagnosis(self): | ||
| 368 | + result = self._run_with_raising_lifecycle(RuntimeError("stalled mid-stream")) | ||
| 369 | + conn = schema.connect(self.db_path) | ||
| 370 | + payload = conn.execute( | ||
| 371 | + "SELECT payload FROM execution_event WHERE task_id = ? ORDER BY rowid DESC LIMIT 1", | ||
| 372 | + (result["task_id"],), | ||
| 373 | + ).fetchone()[0] | ||
| 374 | + conn.close() | ||
| 375 | + self.assertEqual(json.loads(payload)["detail"], "stalled mid-stream") | ||
| 357 | 376 | ||
| 358 | 377 | ||
| 359 | def _fix_loop_trigger(org_node, prompt, target_folder_path, timeout=300): | 378 | def _fix_loop_trigger(org_node, prompt, target_folder_path, timeout=300): |