#!/usr/bin/env python3 """VERIFY: Step5「确认性吸收」回读强化(First Delivery 纪律#5, learn_queue #1330, 2026-10-12) 学到的知识(原文 L8):「模型说「成功」不算成功——结果要看任务结束后的实际状态, 核对必须回读业务系统。」映射到 Hermes:`_verify_and_apply` 的确认性吸收 grounding 分支 旧逻辑拿「target skill 目录存在」直接判 `confirmed` → PASS。 病理 WHERE=step5_verify 确认性吸收 grounding 分支 WHY=存在性是最弱奖励(Harness-R1);「显示成功(目录在)」≠「实际状态(知识真在里面)」。 实测 12 条近期 applied 中 11 条 created_skill/updated_skill 双空。 修复:存在性 + 内容回读双证据;仅存在性命中 → 标 `existence_only` 观测量 + 降 PASS_LAX。 TDD: TC1 (RED/复现旧病理) 旧逻辑对「目录存在但不含知识」的 target 也判 confirmed TC2 (GREEN/真实数据) 本轮吸收 #1330 的真实 outline 对真实目标 SKILL.md → 内容回读命中 TC3 (GREEN/判别力) 拿一个存在但无关的 skill 当 target → 判 existence_only(非无条件放行) TC4 (源码守卫/防回退) app.py 含双证据分支 + 弱证据降级谓词 + reason code 注册 TC5 (冻结重跑 spot check) 同一输入跑 3 次,结果一致(无随机漂移) """ import pathlib import sqlite3 import sys SKILLS = pathlib.Path("/data/data/com.termux/files/home/.dsh/skills") APP = pathlib.Path("/data/data/com.termux/files/home/wechat-viewer/app.py") DB = pathlib.Path("/data/data/com.termux/files/home/wechat-viewer/circle.db") TARGET = "ai-project-first-delivery" # 本轮吸收的 target(去 new:/update: 前缀后) UNRELATED = "feishu-todo-sync" # 存在但与本知识无关的 skill(负对照) FAIL = 0 def old_readback(tgt_files, fingerprint, outline): """复现旧逻辑:只判存在性。""" if tgt_files: return "confirmed (exists on disk)" # ← 弱证据被当成交付证据 return None def new_readback(tgt_files, fingerprint, outline): """复现新逻辑:存在性 + 内容回读双证据。""" if not tgt_files: return None try: txt = tgt_files[0].read_text() if fingerprint and fingerprint in txt: return "confirmed (exists + content readback)" for ph in [outline[i:i + 20] for i in range(0, min(80, len(outline or "")), 20)]: if len(ph.strip()) >= 10 and ph.strip() in txt: return "confirmed (exists + content readback)" except Exception: pass return f"confirmed (exists only; readback=existence_only, READBACK_EXISTENCE_ONLY)" def is_lax(verified): """复现修复后的 PASS/PASS_LAX 谓词。""" return any(m in v for m in ("unconfirmed", "existence_only") for v in verified) def find(tgt): return list(SKILLS.rglob(f"*/{tgt}/SKILL.md")) or list(SKILLS.rglob(f"{tgt}/SKILL.md")) # ── 取本轮真实吸收数据(learning_results 最新 #1330 reference 行)── con = sqlite3.connect(DB) row = con.execute( "SELECT result_summary, outline FROM learning_results " "WHERE learn_queue_id=1330 ORDER BY id DESC LIMIT 1").fetchone() REAL_SUMMARY, REAL_OUTLINE = (row[0] or "", row[1] or "") REAL_FP = REAL_SUMMARY.strip()[:30] print("=== TC1(RED/复现旧病理): 只判存在性 ===") tgt_files = find(TARGET) old_res = old_readback(tgt_files, REAL_FP, REAL_OUTLINE) print(f" 旧逻辑结果: {old_res!r}") print(f" {'✅' if old_res and 'readback' not in old_res else '❌'} " f"旧逻辑不区分「目录在」与「知识在」→ 弱证据直接当 confirmed(病理复现)") if not (old_res and "readback" not in old_res): FAIL += 1 print(f" ✅ 前提事实:目标 skill 在磁盘上 {[str(p) for p in tgt_files]}") if not tgt_files: FAIL += 1 print("\n=== TC2(GREEN/真实数据): 存在性 + 内容回读 ===") print(f" fingerprint(result_summary[:30])={REAL_FP!r}") print(f" {'✅' if REAL_FP not in tgt_files[0].read_text() else '⚠️'} " f"主指纹不命中(与 SKILL.md Troubleshooting 记载的 FINGERPRINT 病理一致,预期)") new_res = new_readback(tgt_files, REAL_FP, REAL_OUTLINE) print(f" 新逻辑结果: {new_res!r}") if "content readback" not in (new_res or ""): print(" ❌ 真实 outline 未能回读命中目标 SKILL.md") FAIL += 1 else: print(" ✅ 真实 outline[0:20] 回读命中 → 拿到「实际状态」证据(非仅存在性)") print(f" {'✅' if not is_lax([new_res]) else '❌'} 强证据 → PASS(非 PASS_LAX)") if is_lax([new_res]): FAIL += 1 print("\n=== TC3(GREEN/判别力): 存在但无关的 target 必须判弱证据 ===") un_files = find(UNRELATED) neg = new_readback(un_files, REAL_FP, REAL_OUTLINE) print(f" 负对照 {UNRELATED}: {neg!r}") print(f" {'✅' if un_files else '❌'} 前提:该 skill 在磁盘上") if not un_files: FAIL += 1 print(f" {'✅' if 'existence_only' in (neg or '') else '❌'} 仅存在性 → 标 existence_only(非无条件放行)") if "existence_only" not in (neg or ""): FAIL += 1 print(f" {'✅' if is_lax([neg]) else '❌'} 弱证据 → PASS_LAX(Step6 会带 double-check 警告)") if not is_lax([neg]): FAIL += 1 print("\n=== TC4(源码守卫/防回退) ===") src = APP.read_text(encoding="utf-8") checks = [ ("确认性分支含内容回读 _rb_hit", "_rb_hit" in src), ("强证据文案 exists + content readback", "exists + content readback" in src), ("弱证据标记 existence_only", "readback=existence_only" in src), ("reason code 定义", "READBACK_EXISTENCE_ONLY = " in src), ("reason code 注册", "VerifyReason.READBACK_EXISTENCE_ONLY:" in src), ("弱证据降级谓词含 existence_only", '_WEAK_EVIDENCE_MARKERS = ("unconfirmed", "existence_only")' in src), ] for label, ok in checks: print(f" {'✅' if ok else '❌'} {label}") if not ok: FAIL += 1 print("\n=== TC5(冻结重跑 spot check): 同一输入 3 次采样 ===") runs = [new_readback(tgt_files, REAL_FP, REAL_OUTLINE) for _ in range(3)] stable = len(set(runs)) == 1 print(f" 3 次结果: {runs}") print(f" {'✅' if stable else '❌'} 3/3 一致(无随机漂移)→ 结论置信度:中(确定性纯函数,非模型调用)") if not stable: FAIL += 1 con.close() print(f"\n{'✅ ALL PASS' if FAIL == 0 else f'❌ {FAIL} FAILED'}") sys.exit(0 if FAIL == 0 else 1)