test_root_cause_trace_e2e.py
2.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
"""U-015's real-call coverage: REQ-04's happy case, for real -- a deliberately
injected "wall collision misjudged" bug, judged by 3 real Validator calls
(one per perspective), expecting >=2/3 to land on root_cause_category=coder
(the SDD clearly specifies wall-collision behavior; the code clearly
deviates from it -- this should NOT look like a planning gap or a skill
defect to a careful reader). Gated behind RUN_REAL_AGENT_E2E=1 (CLAUDE.md
Testing constraint).
Run: RUN_REAL_AGENT_E2E=1 python3 -m unittest src.test_root_cause_trace_e2e -v
"""
import subprocess
import tempfile
import unittest
from src import agent_profile, org_node_trigger, root_cause_trace, schema
from src.test_support import skip_unless_env
_SDD_SNIPPET = (
"## Wall collision (SDD §3.2)\n"
"The board is a grid with valid coordinates 0..N-1 in each axis. The snake's head "
"position is checked against the board bounds every tick. A collision (game over) "
"MUST be triggered the instant the head's x or y coordinate is < 0 or >= N (i.e. any "
"coordinate outside the valid 0..N-1 range, inclusive of the boundary cells "
"themselves being valid, exclusive of anything past them)."
)
_BUGGY_CODE_SNIPPET = (
"def check_wall_collision(head_x, head_y, board_size):\n"
" # BUG: uses > / < instead of >= / <=, so the head is allowed to sit one cell\n"
" # past the actual boundary before a collision is registered.\n"
" return head_x < 0 or head_y < 0 or head_x > board_size or head_y > board_size\n"
)
_BUG_DESCRIPTION = (
"When the snake's head reaches the rightmost or bottommost column, the game does not "
"end immediately -- the head is visually one cell outside the board before 'wall "
"collision' triggers."
)
@skip_unless_env("RUN_REAL_AGENT_E2E")
class RealRootCauseTraceTest(unittest.TestCase):
def test_three_real_perspectives_agree_on_coder(self):
with tempfile.TemporaryDirectory() as tmp:
subprocess.run(["git", "init"], cwd=tmp, check=True, capture_output=True)
agent_profile.deploy_agent_definitions(tmp)
context = "SDD:\n{}\n\nImplementation:\n{}".format(_SDD_SNIPPET, _BUGGY_CODE_SNIPPET)
perspective_fns = [
root_cause_trace.real_validator_perspective(
name, _BUG_DESCRIPTION, context, tmp, trigger_fn=org_node_trigger.trigger_org_node
)
for name in ("correctness", "requirement_completeness", "skill_defect")
]
result = root_cause_trace.judge_root_cause(perspective_fns)
self.assertEqual(result["root_cause_category"], "coder")
self.assertTrue(result["root_cause_detail"])
if __name__ == "__main__":
unittest.main()