test_org_node_trigger_e2e.py
6.09 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
"""trigger_org_node_structured's real-call coverage (2026-08,
work-graph-node-execution-deblackbox-requirements.md FR-4 -- the migration
from Dynamic Workflow scripts to flat `claude --agent` calls for the three
call sites in work_graph_orchestrate.py/e2e_pipeline.py that used to render
one-off `.js` Workflow scripts purely to get "agent identity + schema-shaped
output"). Gated behind RUN_REAL_AGENT_E2E=1 (CLAUDE.md Testing constraint) --
costs real API usage, skipped by default, same as src/test_agent_e2e.py.
Run: RUN_REAL_AGENT_E2E=1 python3 -m unittest src.test_org_node_trigger_e2e -v
**Real result, recorded 2026-08-13** (one real call, `--agent validator
--model claude-haiku-4-5`, target org_node's persona = agents/validator/SOUL.md
via .claude/agents/validator.md): all three FR-4 questions confirmed together
in a single real subprocess call (spied via `wraps=subprocess.run` so the
production `trigger_org_node_structured` code path runs completely for real,
while this test still gets to inspect the raw CompletedProcess underneath it):
1. **Persona loaded correctly** -- the self-identify answer named itself an
independent QA/verification agent (PASS/FAIL/INCONCLUSIVE judgment
framework, "does not write or fix the Coder's business code"), matching
agents/validator/SOUL.md's own red lines, not the generic Claude Code
assistant persona. Same technique U-031 used to validate the (now-retired)
Dynamic Workflow `agentType` path -- ask the agent to self-identify, check
for persona-specific vocabulary, not a canned/generic answer.
2. **`--model` override took effect for real** -- `claude --agent <org_node>`
combined with `--model claude-haiku-4-5` is a genuinely different CLI
combination from anything DSL-internal Workflow options ever validated
(05 §5.4.1.5's "two different paths" lesson) -- confirmed empirically, not
assumed: the raw JSON response's own `modelUsage` key names
"claude-haiku-4-5" as the model that actually served the request, not
validator's role-declared default (`claude-sonnet-5`, roles/validator/
role.json).
3. **```json fence extraction is stable on real model output** -- the
real free-text response (identity narrative + a trailing ```json fenced
block, since `--json-schema` is never combined with `--agent`, see 05
§5.4.1.1) parsed cleanly via `extract_json_payload`/`trigger_org_node_
structured`'s required-keys matching, returning the schema-shaped dict
with no manual fallback needed -- the same extraction algorithm
`work_graph_plan.py::plan_work_graph()` already validated for real in
production (`_extract_graph_manifest`), now proven for `_JSON_FENCE_
INSTRUCTION`'s more generic prompt wording too.
"""
import json
import subprocess
import tempfile
import unittest
from unittest.mock import patch
from src import agent_profile, org_node_trigger, project_scaffold, schema
from src.test_support import skip_unless_env
_SELF_IDENTIFY_SCHEMA = {
"type": "object",
"required": ["job", "never_does"],
"properties": {
"job": {"type": "string", "description": "one short sentence describing your role"},
"never_does": {"type": "string", "description": "one short sentence: something you never do"},
},
}
@skip_unless_env("RUN_REAL_AGENT_E2E")
class RealTriggerOrgNodeStructuredTest(unittest.TestCase):
def test_identity_model_override_and_json_extraction_all_real(self):
with tempfile.TemporaryDirectory() as tmp:
subprocess.run(["git", "init"], cwd=tmp, check=True, capture_output=True)
conn = schema.connect(":memory:")
schema.create_schema(conn)
project_scaffold.create_project(conn, tmp, init_git_if_missing=False)
agent_profile.deploy_agent_definitions(tmp)
real_run = subprocess.run
captured = {}
def spying_run(*args, **kwargs):
result = real_run(*args, **kwargs)
captured["result"] = result
return result
# wraps the real subprocess.run rather than replacing it -- this
# call goes out to the real `claude` CLI exactly like production
# code would, this test just also gets to inspect the raw
# CompletedProcess trigger_org_node_structured's own return value
# (the extracted payload alone) doesn't expose.
with patch("src.org_node_trigger.subprocess.run", side_effect=spying_run):
extracted = org_node_trigger.trigger_org_node_structured(
"validator",
"In one short sentence each: what is your job, and what do you never do? "
"Answer only, take no other action.",
tmp,
_SELF_IDENTIFY_SCHEMA,
model="claude-haiku-4-5",
timeout=120,
)
# (3) JSON fence extraction worked on real model output -- a dict
# with both required keys, not None/a StructuredOutputExtractionError.
self.assertIn("job", extracted)
self.assertIn("never_does", extracted)
# (1) persona genuinely loaded -- validator's own red lines
# (agents/validator/SOUL.md), not a generic assistant answer.
identity_text = extracted["job"] + " " + extracted["never_does"]
self.assertTrue(
any(kw in identity_text for kw in (
"QA", "验收", "PASS", "FAIL", "INCONCLUSIVE", "independent", "verif", "Verif",
)),
"validator's answer doesn't look like the migrated QA identity: " + identity_text,
)
# (2) --model override genuinely took effect -- the real response's
# own modelUsage key names the override model, not validator's
# role-declared default (claude-sonnet-5, roles/validator/role.json).
raw_payload = json.loads(captured["result"].stdout)
model_usage = raw_payload.get("modelUsage") or {}
self.assertIn(
"claude-haiku-4-5", model_usage,
"--model override not reflected in the real response's modelUsage: " + json.dumps(raw_payload)[:2000],
)
if __name__ == "__main__":
unittest.main()