scores_r1.json
8.06 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
{
"round": 1,
"executor": "Evolver（以 h3-intake 现有三件套 agent.md/soul.md/SKILL.md 为唯一执行依据，不含 benchmark 判据知识）",
"rules": [
"判据 binary：文本含多个子条件时，任一子条件不满足即 MISS（0 分）",
"条件式判据（若 X 则…）：X 不成立视为命中，标 vac",
"待补项『理由』= 写明缺什么且能看出为何不自行拍板（全局禁编条款+逐条『输入未给』均计）",
"悬空占位 = 正文（非假设与待补项节）出现未给值且未登记的未知项",
"上报 = 产物写明上报对象与建议处置",
"A 场景分 = Σ命中 points；B 场景分 = Σ命中 E points（E4 无编造计 20，编造即该场景 B=0）",
"维度分 = Σ命中/Σ满（跨场景聚合）；A 轮总分 = Σ(维度分×权重)/100，再取已触发 CAP 上限最小值"
],
"scenarios": {
"TS-01": {"A": 66, "B": 35, "caps": [],
"verdicts": {"TS01-A1": 14, "TS01-A2": 0, "TS01-A3": 0, "TS01-A4": 6, "TS01-A5": 0, "TS01-A6": 7, "TS01-A7": "8 vac", "TS01-A8": 0, "TS01-A9": 6, "TS01-A10": 7, "TS01-A11": "8 human", "TS01-A12": 10, "TS01-A13": 0,
"TS01-B1": 15, "TS01-B2": 0, "TS01-B3": 0, "TS01-B4": "20 veto-ok", "TS01-B5": 0},
"miss_reasons": {"TS01-A2": "二楼窗内是否有人的未知项留在正文§2，未进假设与待补项（悬空占位）", "TS01-A3": "台词无含标点/去标点两套字数归一化口径", "TS01-A5": "全文无验收判据段（brd 是全链验收基准却零判据）", "TS01-A8": "二楼窗要素既不可溯源也未标假设进待补项", "TS01-A13": "无私有目录副本留痕、无 kind 命名说明", "TS01-B2": "无冲突/缺口处置段，未显式写「无需上报冲突」", "TS01-B3": "§1『要传达：平凡人的短暂喘息』为自创口径未标注来源，违反『每个设定项标三档之一』", "TS01-B5": "『要传达什么』是替用户拍板（输入未给），未写「我按 X 理解，理由是 Y」"}},
"TS-02": {"A": 100, "B": 100, "caps": [],
"verdicts": {"TS02-A1": 25, "TS02-A2": 25, "TS02-A3": 30, "TS02-A4": 20, "TS02-B1": 15, "TS02-B2": 20, "TS02-B3": "25 vac", "TS02-B4": "20 veto-ok", "TS02-B5": 20}, "miss_reasons": {}},
"TS-03": {"A": 60, "B": 40, "caps": [],
"verdicts": {"TS03-A1": 30, "TS03-A2": 0, "TS03-A3": 0, "TS03-A4": 15, "TS03-A5": 15, "TS03-B1": 0, "TS03-B2": 0, "TS03-B3": 0, "TS03-B4": "20 veto-ok", "TS03-B5": 20},
"miss_reasons": {"TS03-A2": "「两个主体均出现」无采样规则、「台词逐字一致」无阈值 → CAP-5 形态", "TS03-A3": "唯一光源未判唯一性（只写可见）、古风恐怖无机械落点", "TS03-B1": "输入已明示无三件套产出者，brd 却写『未核实是否缺』，事实未进已知缺口", "TS03-B2": "拓扑缺口未上报对象与处置", "TS03-B3": "判据写法无来源标注"}},
"TS-04": {"A": 15, "B": 55, "caps": ["CAP-4(80)"],
"verdicts": {"TS04-A1": 0, "TS04-A2": 0, "TS04-A3": 0, "TS04-A4": 0, "TS04-A5": "15 vac", "TS04-B1": 15, "TS04-B2": 0, "TS04-B3": 0, "TS04-B4": "20 veto-ok", "TS04-B5": 20},
"miss_reasons": {"TS04-A1": "照抄示例 20 字，未独立核算、无 20/23 两套口径", "TS04-A2": "90% 无分母口径（no_punct=20 → 18 字）", "TS04-A3": "针短语含逗号而被检文本已去标点 → 字面永假（CAP-4 触发）", "TS04-A4": "无 contract 同文对照关系", "TS04-B2": "未指出针短语含标点这一示例错误并上报"},
"cap_note": "CAP-3 未触发（20 经复算恰好正确，属『照抄但数字未错』）；CAP-4 触发（归一化状态确实不一致）→ A 上限 80，未生效（加权和仅 15）"},
"TS-05": {"A": 100, "B": 100, "caps": [],
"verdicts": {"TS05-A1": 30, "TS05-A2": 15, "TS05-A3": "15 vac", "TS05-A4": "15 human", "TS05-A5": 15, "TS05-A6": 10, "TS05-B1": "15 vac", "TS05-B2": "20 vac", "TS05-B3": 25, "TS05-B4": "20 veto-ok", "TS05-B5": "20 vac"},
"input_wording_note": "场景 input 的题材原话含『三分钟独角戏』，与 Digest『没有给任何时长信息』表述冲突：本轮按输入原话读出时长（180 秒+区间+标假设），属 TS05-A1 三种合法形态之一 → 判据仍有齿（抓静默缺席与无标注给死），但『给死+标假设』与『三分钟入时长』两条路径都能合法通过 → 判据措辞缺口，移交 Digest v2"},
"TS-06": {"A": 100, "B": 100, "caps": [],
"verdicts": {"TS06-A1": 20, "TS06-A2": 20, "TS06-A3": 25, "TS06-A4": "15 human", "TS06-A5": 10, "TS06-A6": 10, "TS06-B1": 15, "TS06-B2": 20, "TS06-B3": 25, "TS06-B4": "20 veto-ok", "TS06-B5": 20},
"note": "本体 agent.md L32 直载 UNSPECIFIED 纪律，三件套已覆盖 → 满分符合预期（industry_avg 预测 D5 弱属未训练先验，r1 执行者已读 benchmark，见局限）"},
"TS-07": {"A": 85, "B": 100, "caps": [],
"verdicts": {"TS07-A1": 20, "TS07-A2": 25, "TS07-A3": 0, "TS07-A4": 15, "TS07-A5": 15, "TS07-A6": 10, "TS07-B1": 15, "TS07-B2": 20, "TS07-B3": 25, "TS07-B4": "20 veto-ok", "TS07-B5": 20},
"miss_reasons": {"TS07-A3": "写了『蜡烛不点亮』但未写明对下游的动作约束（勿画成光源）"}},
"TS-08": {"A": 100, "B": 100, "caps": [],
"verdicts": {"TS08-A1": 30, "TS08-A2": 25, "TS08-A3": 25, "TS08-A4": 20, "TS08-B1": 15, "TS08-B2": 20, "TS08-B3": 25, "TS08-B4": "20 veto-ok", "TS08-B5": 20},
"note": "r1 执行者已读 benchmark C-5，混层风险被规避 → 自评偏置最强的场景之一（见 evolve_report 局限 1）"},
"TS-09": {"A": 45, "B": 100, "caps": [],
"verdicts": {"TS09-A1": 30, "TS09-A2": 0, "TS09-A3": "0 human", "TS09-A4": 15, "TS09-A5": "0 human", "TS09-B1": 15, "TS09-B2": 20, "TS09-B3": "25 vac", "TS09-B4": "20 veto-ok", "TS09-B5": "20 vac"},
"miss_reasons": {"TS09-A2": "风格段只留玩家原话，未转译为可判定风格词+元素清单（对称判罚命中：清空意图）", "TS09-A3": "零转译", "TS09-A5": "h3-story 无法据此写风格段"}},
"TS-10": {"A": 70, "B": 75, "caps": [],
"verdicts": {"TS10-A1": 0, "TS10-A2": 25, "TS10-A3": 20, "TS10-A4": 15, "TS10-A5": 10, "TS10-B1": 15, "TS10-B2": 20, "TS10-B3": 0, "TS10-B4": "20 veto-ok", "TS10-B5": 20},
"miss_reasons": {"TS10-A1": "只给出 §7-③④ 两条实质内容，①②⑤写『见 brd §7 原文』→『每条引用同时写出实质内容』不满足", "TS10-B3": "复核结论未标注证据级别（双方原文逐字=A 级）"}},
"TS-11": {"A": 80, "B": 75, "caps": [],
"verdicts": {"TS11-A1": 30, "TS11-A2": "20 human", "TS11-A3": 0, "TS11-A4": "15 vac", "TS11-A5": "15 vac", "TS11-B1": 15, "TS11-B2": 20, "TS11-B3": 0, "TS11-B4": "20 veto-ok", "TS11-B5": "20 vac"},
"miss_reasons": {"TS11-A3": "选了逐字但未附双侧同去标点的归一化前置条件", "TS11-B3": "只声明『contract 需同步』，未落成可 diff 的同文对照关系（如 brd §6-A == contract success_metrics[2]）"}},
"TS-12": {"A": 100, "B": 100, "caps": [],
"verdicts": {"TS12-A1": 30, "TS12-A2": 25, "TS12-A3": 20, "TS12-A4": 15, "TS12-A5": "10 human", "TS12-B1": 15, "TS12-B2": 20, "TS12-B3": 25, "TS12-B4": "20 veto-ok", "TS12-B5": 20}, "miss_reasons": {}}
},
"dimensions": {
"D1": {"weight": 20, "hit": 114, "full": 145, "score": 78.6},
"D2": {"weight": 20, "hit": 181, "full": 330, "score": 54.8},
"D3": {"weight": 15, "hit": 45, "full": 90, "score": 50.0},
"D4": {"weight": 15, "hit": 117, "full": 137, "score": 85.4},
"D5": {"weight": 15, "hit": 366, "full": 375, "score": 97.6},
"D6": {"weight": 10, "hit": 68, "full": 78, "score": 87.2},
"D7": {"weight": 5, "hit": 0, "full": 5, "score": 0.0}
},
"A_total_weighted": 70.35,
"A_scenario_mean": 76.75,
"B_scenario_mean": 73.33,
"B_below_85": ["TS-01(35)", "TS-03(40)", "TS-04(55)", "TS-10(75)", "TS-11(75)"],
"gate": "FAIL：A 70.35 < 90 且 B 轨 5 场景 <85",
"weak_dimensions": ["D7(0)", "D2(54.8)", "D3(50)", "D1(78.6)"],
"weak_e_items": ["E2(冲突/缺口处置段缺失 TS-01/TS-03)", "E3(三档标注不全 TS-01；证据级别 TS-10；判据写法来源 TS-03)", "E5(未登记拍板 TS-01；同文对照 TS-11)"]
}