test_task_reconciliation.py
7.91 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
"""task_reconciliation.py tests (2026-08-06, real 15h-stuck-task incident):
mocked at the subprocess boundary for reconcile_once, real sqlite for
candidate-selection logic -- same split as test_task_execution.py.
Run: python3 -m unittest src.test_task_reconciliation -v
"""
import subprocess
import tempfile
import unittest
from datetime import datetime, timezone
from unittest.mock import patch
from src import project_scaffold, schema, task_execution, task_reconciliation
_NOW_EPOCH = 2_000_000_000 # fixed reference point, independent of wall clock
def _ts(epoch_seconds):
return datetime.fromtimestamp(epoch_seconds, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S")
class ReconciliationTestBase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
subprocess.run(["git", "init"], cwd=self.tmp.name, check=True, capture_output=True)
self.conn = schema.connect(":memory:")
schema.create_schema(self.conn)
project = project_scaffold.create_project(self.conn, self.tmp.name, init_git_if_missing=False)
self.project_id = project["project_id"]
def tearDown(self):
self.conn.close()
def _insert_task(self, task_id, status, created_at_epoch, assigned_org_node="planner"):
self.conn.execute(
"INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) "
"VALUES (?, ?, 'a goal', 'goal', ?, ?, ?)",
(task_id, self.project_id, status, assigned_org_node, _ts(created_at_epoch)),
)
self.conn.commit()
def _insert_event(self, task_id, created_at_epoch, event_type="log_appended"):
self.conn.execute(
"INSERT INTO execution_event (event_id, project_id, task_id, event_type, payload, created_at) "
"VALUES (?, ?, ?, ?, '{}', ?)",
("ev-" + task_id + "-" + str(created_at_epoch), self.project_id, task_id, event_type, _ts(created_at_epoch)),
)
self.conn.commit()
class FindStaleResumableTasksTest(ReconciliationTestBase):
def test_finds_task_with_no_activity_at_all(self):
self._insert_task("task-1", "coding", created_at_epoch=_NOW_EPOCH - 3600)
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual([c["task_id"] for c in result], ["task-1"])
def test_excludes_task_with_recent_event(self):
self._insert_task("task-1", "validating", created_at_epoch=_NOW_EPOCH - 7200)
self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 60) # 1 min ago -- fresh
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual(result, [])
def test_uses_most_recent_event_not_earliest(self):
self._insert_task("task-1", "coding", created_at_epoch=_NOW_EPOCH - 7200)
self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 5000) # old
self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 60) # recent -- MAX() must win
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual(result, [])
def test_stale_event_still_flagged(self):
self._insert_task("task-1", "coding", created_at_epoch=_NOW_EPOCH - 7200)
self._insert_event("task-1", created_at_epoch=_NOW_EPOCH - 5000) # > 1800s ago
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual([c["task_id"] for c in result], ["task-1"])
def test_excludes_non_resumable_status(self):
self._insert_task("task-1", "done", created_at_epoch=_NOW_EPOCH - 100000)
self._insert_task("task-2", "blocked_pending_human", created_at_epoch=_NOW_EPOCH - 100000)
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual(result, [])
def test_excludes_in_flight_task(self):
self._insert_task("task-1", "validating", created_at_epoch=_NOW_EPOCH - 100000)
with patch.object(task_execution, "is_in_flight", return_value=True):
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual(result, [])
def test_candidate_carries_target_folder_path_and_goal(self):
self._insert_task("task-1", "queued", created_at_epoch=_NOW_EPOCH - 100000)
result = task_reconciliation.find_stale_resumable_tasks(
self.conn, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual(result, [{"task_id": "task-1", "target_folder_path": self.tmp.name, "goal_text": "a goal"}])
class ReconcileOnceTest(unittest.TestCase):
"""Real file db (own connection per dispatched thread, same reasoning
as StartTaskExecutionInBackgroundTest) -- dispatch itself is mocked so
this stays at the "did we correctly decide to call it" boundary, not
a re-test of run_or_resume_task_lifecycle's own mechanics."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
subprocess.run(["git", "init"], cwd=self.tmp.name, check=True, capture_output=True)
self.tmp_db = tempfile.NamedTemporaryFile(suffix=".db", delete=False)
self.tmp_db.close()
self.db_path = self.tmp_db.name
self.addCleanup(lambda: __import__("os").path.exists(self.db_path) and __import__("os").remove(self.db_path))
conn = schema.connect(self.db_path)
schema.create_schema(conn)
project = project_scaffold.create_project(conn, self.tmp.name, init_git_if_missing=False)
self.project_id = project["project_id"]
conn.execute(
"INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) "
"VALUES ('stale-1', ?, 'goal 1', 'goal', 'coding', 'coder', ?)",
(self.project_id, _ts(_NOW_EPOCH - 100000)),
)
conn.execute(
"INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) "
"VALUES ('fresh-1', ?, 'goal 2', 'goal', 'coding', 'coder', ?)",
(self.project_id, _ts(_NOW_EPOCH - 10)),
)
conn.commit()
conn.close()
def test_dispatches_only_stale_candidates(self):
with patch.object(task_execution, "start_task_execution_in_background") as mock_start:
result = task_reconciliation.reconcile_once(
self.db_path, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual([c["task_id"] for c in result], ["stale-1"])
mock_start.assert_called_once_with(self.db_path, "stale-1", self.tmp.name, "goal 1")
def test_one_dispatch_failure_does_not_block_others(self):
conn = schema.connect(self.db_path)
conn.execute(
"INSERT INTO task (task_id, project_id, goal_text, task_type, status, assigned_org_node, created_at) "
"VALUES ('stale-2', ?, 'goal 3', 'goal', 'validating', 'validator', ?)",
(self.project_id, _ts(_NOW_EPOCH - 100000)),
)
conn.commit()
conn.close()
with patch.object(
task_execution, "start_task_execution_in_background",
side_effect=[task_execution.TaskExecutionError("already in flight"), None],
) as mock_start:
result = task_reconciliation.reconcile_once(
self.db_path, stale_after_seconds=1800, now=lambda: _NOW_EPOCH
)
self.assertEqual(len(result), 2)
self.assertEqual(mock_start.call_count, 2)
if __name__ == "__main__":
unittest.main()