test_step6_readback_existence_only.py
6.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
#!/usr/bin/env python3
"""VERIFY: Step5「确认性吸收」回读强化（First Delivery 纪律#5, learn_queue #1330, 2026-10-12）
学到的知识（原文 L8）：「模型说「成功」不算成功——结果要看任务结束后的实际状态，
核对必须回读业务系统。」映射到 Hermes：`_verify_and_apply` 的确认性吸收 grounding 分支
旧逻辑拿「target skill 目录存在」直接判 `confirmed` → PASS。
病理 WHERE=step5_verify 确认性吸收 grounding 分支
WHY=存在性是最弱奖励（Harness-R1）；「显示成功（目录在）」≠「实际状态（知识真在里面）」。
实测 12 条近期 applied 中 11 条 created_skill/updated_skill 双空。
修复：存在性 + 内容回读双证据；仅存在性命中 → 标 `existence_only` 观测量 + 降 PASS_LAX。
TDD：
TC1 (RED/复现旧病理) 旧逻辑对「目录存在但不含知识」的 target 也判 confirmed
TC2 (GREEN/真实数据) 本轮吸收 #1330 的真实 outline 对真实目标 SKILL.md → 内容回读命中
TC3 (GREEN/判别力) 拿一个存在但无关的 skill 当 target → 判 existence_only（非无条件放行）
TC4 (源码守卫/防回退) app.py 含双证据分支 + 弱证据降级谓词 + reason code 注册
TC5 (冻结重跑 spot check) 同一输入跑 3 次，结果一致（无随机漂移）
"""
import pathlib
import sqlite3
import sys
SKILLS = pathlib.Path("/data/data/com.termux/files/home/.dsh/skills")
APP = pathlib.Path("/data/data/com.termux/files/home/wechat-viewer/app.py")
DB = pathlib.Path("/data/data/com.termux/files/home/wechat-viewer/circle.db")
TARGET = "ai-project-first-delivery" # 本轮吸收的 target（去 new:/update: 前缀后）
UNRELATED = "feishu-todo-sync" # 存在但与本知识无关的 skill（负对照）
FAIL = 0
def old_readback(tgt_files, fingerprint, outline):
"""复现旧逻辑：只判存在性。"""
if tgt_files:
return "confirmed (exists on disk)" # ← 弱证据被当成交付证据
return None
def new_readback(tgt_files, fingerprint, outline):
"""复现新逻辑：存在性 + 内容回读双证据。"""
if not tgt_files:
return None
try:
txt = tgt_files[0].read_text()
if fingerprint and fingerprint in txt:
return "confirmed (exists + content readback)"
for ph in [outline[i:i + 20] for i in range(0, min(80, len(outline or "")), 20)]:
if len(ph.strip()) >= 10 and ph.strip() in txt:
return "confirmed (exists + content readback)"
except Exception:
pass
return f"confirmed (exists only; readback=existence_only, READBACK_EXISTENCE_ONLY)"
def is_lax(verified):
"""复现修复后的 PASS/PASS_LAX 谓词。"""
return any(m in v for m in ("unconfirmed", "existence_only") for v in verified)
def find(tgt):
return list(SKILLS.rglob(f"*/{tgt}/SKILL.md")) or list(SKILLS.rglob(f"{tgt}/SKILL.md"))
# ── 取本轮真实吸收数据（learning_results 最新 #1330 reference 行）──
con = sqlite3.connect(DB)
row = con.execute(
"SELECT result_summary, outline FROM learning_results "
"WHERE learn_queue_id=1330 ORDER BY id DESC LIMIT 1").fetchone()
REAL_SUMMARY, REAL_OUTLINE = (row[0] or "", row[1] or "")
REAL_FP = REAL_SUMMARY.strip()[:30]
print("=== TC1(RED/复现旧病理): 只判存在性 ===")
tgt_files = find(TARGET)
old_res = old_readback(tgt_files, REAL_FP, REAL_OUTLINE)
print(f" 旧逻辑结果: {old_res!r}")
print(f" {'✅' if old_res and 'readback' not in old_res else '❌'} "
f"旧逻辑不区分「目录在」与「知识在」→ 弱证据直接当 confirmed（病理复现）")
if not (old_res and "readback" not in old_res):
FAIL += 1
print(f" ✅ 前提事实：目标 skill 在磁盘上 {[str(p) for p in tgt_files]}")
if not tgt_files:
FAIL += 1
print("\n=== TC2(GREEN/真实数据): 存在性 + 内容回读 ===")
print(f" fingerprint(result_summary[:30])={REAL_FP!r}")
print(f" {'✅' if REAL_FP not in tgt_files[0].read_text() else '⚠️'} "
f"主指纹不命中（与 SKILL.md Troubleshooting 记载的 FINGERPRINT 病理一致，预期）")
new_res = new_readback(tgt_files, REAL_FP, REAL_OUTLINE)
print(f" 新逻辑结果: {new_res!r}")
if "content readback" not in (new_res or ""):
print(" ❌ 真实 outline 未能回读命中目标 SKILL.md")
FAIL += 1
else:
print(" ✅ 真实 outline[0:20] 回读命中 → 拿到「实际状态」证据（非仅存在性）")
print(f" {'✅' if not is_lax([new_res]) else '❌'} 强证据 → PASS（非 PASS_LAX）")
if is_lax([new_res]):
FAIL += 1
print("\n=== TC3(GREEN/判别力): 存在但无关的 target 必须判弱证据 ===")
un_files = find(UNRELATED)
neg = new_readback(un_files, REAL_FP, REAL_OUTLINE)
print(f" 负对照 {UNRELATED}: {neg!r}")
print(f" {'✅' if un_files else '❌'} 前提：该 skill 在磁盘上")
if not un_files:
FAIL += 1
print(f" {'✅' if 'existence_only' in (neg or '') else '❌'} 仅存在性 → 标 existence_only（非无条件放行）")
if "existence_only" not in (neg or ""):
FAIL += 1
print(f" {'✅' if is_lax([neg]) else '❌'} 弱证据 → PASS_LAX（Step6 会带 double-check 警告）")
if not is_lax([neg]):
FAIL += 1
print("\n=== TC4(源码守卫/防回退) ===")
src = APP.read_text(encoding="utf-8")
checks = [
("确认性分支含内容回读 _rb_hit", "_rb_hit" in src),
("强证据文案 exists + content readback", "exists + content readback" in src),
("弱证据标记 existence_only", "readback=existence_only" in src),
("reason code 定义", "READBACK_EXISTENCE_ONLY = " in src),
("reason code 注册", "VerifyReason.READBACK_EXISTENCE_ONLY:" in src),
("弱证据降级谓词含 existence_only", '_WEAK_EVIDENCE_MARKERS = ("unconfirmed", "existence_only")' in src),
]
for label, ok in checks:
print(f" {'✅' if ok else '❌'} {label}")
if not ok:
FAIL += 1
print("\n=== TC5(冻结重跑 spot check): 同一输入 3 次采样 ===")
runs = [new_readback(tgt_files, REAL_FP, REAL_OUTLINE) for _ in range(3)]
stable = len(set(runs)) == 1
print(f" 3 次结果: {runs}")
print(f" {'✅' if stable else '❌'} 3/3 一致（无随机漂移）→ 结论置信度：中（确定性纯函数，非模型调用）")
if not stable:
FAIL += 1
con.close()
print(f"\n{'✅ ALL PASS' if FAIL == 0 else f'❌ {FAIL} FAILED'}")
sys.exit(0 if FAIL == 0 else 1)