test_org_level_loop_e2e.py
3.98 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
"""U-017's real-call coverage: architecture doc §11 MVP acceptance criterion
4, for real -- a deliberately injected off-by-one wall-collision bug (same
scenario shape as U-015's real test, but this time actually driving a real
fix instead of just judging), routed, fixed by a real Coder call editing the
actual file in the repo, then confirmed by a real Validator call that reads
the fixed file. Gated behind RUN_REAL_AGENT_E2E=1 (CLAUDE.md Testing
constraint).
Run: RUN_REAL_AGENT_E2E=1 python3 -m unittest src.test_org_level_loop_e2e -v
"""
import subprocess
import tempfile
import unittest
from src import agent_profile, org_level_loop, schema
from src.test_support import skip_unless_env
_SDD_SNIPPET = (
"## Wall collision (SDD §3.2)\n"
"The board is a grid with valid coordinates 0..N-1 in each axis. A collision (game "
"over) MUST be triggered the instant the head's x or y coordinate is < 0 or >= N."
)
_BUG_DESCRIPTION = (
"When the snake's head reaches the rightmost or bottommost column, the game does not "
"end immediately -- the head is visually one cell outside the board before 'wall "
"collision' triggers. See collision.py in this repo."
)
_BUGGY_CODE = (
"def check_wall_collision(head_x, head_y, board_size):\n"
" # BUG: uses > / < instead of >= / <=, so the head is allowed to sit one cell\n"
" # past the actual boundary before a collision is registered.\n"
" return head_x < 0 or head_y < 0 or head_x > board_size or head_y > board_size\n"
)
@skip_unless_env("RUN_REAL_AGENT_E2E")
class RealOrgLevelLoopTest(unittest.TestCase):
def test_real_fix_loop_reaches_done(self):
with tempfile.TemporaryDirectory() as tmp:
subprocess.run(["git", "init"], cwd=tmp, check=True, capture_output=True)
agent_profile.deploy_agent_definitions(tmp)
with open(tmp + "/collision.py", "w") as f:
f.write(_BUGGY_CODE)
subprocess.run(["git", "add", "collision.py"], cwd=tmp, check=True, capture_output=True)
subprocess.run(
["git", "-c", "user.name=t", "-c", "user.email=t@t.com", "commit", "-m", "seed buggy collision.py"],
cwd=tmp, check=True, capture_output=True,
)
conn = schema.connect(":memory:")
schema.create_schema(conn)
conn.execute(
"INSERT INTO project (project_id, name, target_folder_path, has_git_repo, status) "
"VALUES ('proj-1', 'P', ?, 1, 'active')",
(tmp,),
)
conn.execute(
"INSERT INTO task (task_id, project_id, goal_text, task_type, status, loop_round, assigned_org_node) "
"VALUES ('task-1', 'proj-1', '实现贪吃蛇游戏', 'goal', 'validating', 0, 'validator')"
)
conn.commit()
result = org_level_loop.run_fix_loop(
conn, "task-1", tmp,
bug_description=_BUG_DESCRIPTION,
context="SDD:\n{}\n\nImplementation:\n{}".format(_SDD_SNIPPET, _BUGGY_CODE),
fix_prompt=(
"Fix the off-by-one wall-collision bug in collision.py per this SDD "
"section, then commit the fix:\n\n" + _SDD_SNIPPET
),
verify_prompt=(
"Read collision.py in this repo and check whether check_wall_collision now "
"correctly triggers at the boundary (>= N / < 0, not > N / < 0), per this SDD "
"section:\n\n" + _SDD_SNIPPET + "\n\nReply with 'VERDICT: PASS' or 'VERDICT: FAIL' "
"and your reasoning."
),
timeout=300,
)
self.assertEqual(result["root_cause_category"], "coder")
self.assertEqual(result["route"], "fast_lane")
self.assertTrue(result["verified"], "validator did not confirm the fix: " + result["verify_response"])
self.assertEqual(result["final_status"], "done")
if __name__ == "__main__":
unittest.main()