"""task_reconciliation.py tests (2026-08-06, real 15h-stuck-task incident): mocked at the subprocess boundary for reconcile_once, real sqlite for candidate-selection logic -- same split as test_task_execution.py. Run: python3 -m unittest src.test_task_reconciliation -v """ import subprocess import tempfile import unittest from datetime import datetime, timezone from unittest.mock import patch from src import project_scaffold, schema, task_execution, task_reconciliation _NOW_EPOCH = 2_000_000_000 # fixed reference point, independent of wall clock def _ts(epoch_seconds): return datetime.fromtimestamp(epoch_seconds, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S") class ReconciliationTestBase(unittest.TestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) subprocess.run(["git", "init"], cwd=self.tmp.name, check=True, capture_output=True) self.conn = schema.connect(":memory:") schema.create_schema(self.conn) project = project_scaffold.create_project(self.conn, self.tmp.name, init_git_if_missing=False) self.project_id = project["project_id"] def tearDown(self): self.conn.close() def _insert_task(self, task_id, status, created_at_epoch, assigned_org_node="planner"): self.conn.execute( "INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) " "VALUES (?, ?, 'a goal', 'goal', ?, ?, ?)", (task_id, self.project_id, status, assigned_org_node, _ts(created_at_epoch)), ) self.conn.commit() def _insert_event(self, task_id, created_at_epoch, event_type="log_appended"): self.conn.execute( "INSERT INTO execution_event (event_id, project_id, task_id, event_type, payload, created_at) " "VALUES (?, ?, ?, ?, '{}', ?)", ("ev-" + task_id + "-" + str(created_at_epoch), self.project_id, task_id, event_type, _ts(created_at_epoch)), ) self.conn.commit() class FindStaleResumableTasksTest(ReconciliationTestBase): def test_finds_task_with_no_activity_at_all(self): self._insert_task("task-1", "coding", created_at_epoch=_NOW_EPOCH - 3600) result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual([c["task_id"] for c in result], ["task-1"]) def test_excludes_task_with_recent_event(self): self._insert_task("task-1", "validating", created_at_epoch=_NOW_EPOCH - 7200) self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 60) # 1 min ago -- fresh result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual(result, []) def test_uses_most_recent_event_not_earliest(self): self._insert_task("task-1", "coding", created_at_epoch=_NOW_EPOCH - 7200) self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 5000) # old self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 60) # recent -- MAX() must win result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual(result, []) def test_stale_event_still_flagged(self): self._insert_task("task-1", "coding", created_at_epoch=_NOW_EPOCH - 7200) self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 5000) # > 1800s ago result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual([c["task_id"] for c in result], ["task-1"]) def test_excludes_non_resumable_status(self): self._insert_task("task-1", "done", created_at_epoch=_NOW_EPOCH - 100000) self._insert_task("task-2", "blocked_pending_human", created_at_epoch=_NOW_EPOCH - 100000) result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual(result, []) def test_excludes_in_flight_task(self): self._insert_task("task-1", "validating", created_at_epoch=_NOW_EPOCH - 100000) with patch.object(task_execution, "is_in_flight", return_value=True): result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual(result, []) def test_candidate_carries_target_folder_path_and_goal(self): self._insert_task("task-1", "queued", created_at_epoch=_NOW_EPOCH - 100000) result = task_reconciliation.find_stale_resumable_tasks( self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual(result, [{"task_id": "task-1", "target_folder_path": self.tmp.name, "goal_text": "a goal"}]) class ReconcileOnceTest(unittest.TestCase): """Real file db (own connection per dispatched thread, same reasoning as StartTaskExecutionInBackgroundTest) -- dispatch itself is mocked so this stays at the "did we correctly decide to call it" boundary, not a re-test of run_or_resume_task_lifecycle's own mechanics.""" def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) subprocess.run(["git", "init"], cwd=self.tmp.name, check=True, capture_output=True) self.tmp_db = tempfile.NamedTemporaryFile(suffix=".db", delete=False) self.tmp_db.close() self.db_path = self.tmp_db.name self.addCleanup(lambda: __import__("os").path.exists(self.db_path) and __import__("os").remove(self.db_path)) conn = schema.connect(self.db_path) schema.create_schema(conn) project = project_scaffold.create_project(conn, self.tmp.name, init_git_if_missing=False) self.project_id = project["project_id"] conn.execute( "INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) " "VALUES ('stale-1', ?, 'goal 1', 'goal', 'coding', 'coder', ?)", (self.project_id, _ts(_NOW_EPOCH - 100000)), ) conn.execute( "INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) " "VALUES ('fresh-1', ?, 'goal 2', 'goal', 'coding', 'coder', ?)", (self.project_id, _ts(_NOW_EPOCH - 10)), ) conn.commit() conn.close() def test_dispatches_only_stale_candidates(self): with patch.object(task_execution, "start_task_execution_in_background") as mock_start: result = task_reconciliation.reconcile_once( self.db_path, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual([c["task_id"] for c in result], ["stale-1"]) mock_start.assert_called_once_with(self.db_path, "stale-1", self.tmp.name, "goal 1") def test_one_dispatch_failure_does_not_block_others(self): conn = schema.connect(self.db_path) conn.execute( "INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) " "VALUES ('stale-2', ?, 'goal 3', 'goal', 'validating', 'validator', ?)", (self.project_id, _ts(_NOW_EPOCH - 100000)), ) conn.commit() conn.close() with patch.object( task_execution, "start_task_execution_in_background", side_effect=[task_execution.TaskExecutionError("already in flight"), None], ) as mock_start: result = task_reconciliation.reconcile_once( self.db_path, stale_after_seconds=1800, now=lambda: _NOW_EPOCH ) self.assertEqual(len(result), 2) self.assertEqual(mock_start.call_count, 2) if __name__ == "__main__": unittest.main()