baseline_report.json
3.41 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
{
"benchmark_id": "qa-judge-baseline-001",
"timestamp": "2026-03-25T12:00:00Z",
"agent_version": "1.0.0",
"test_results": {
"total_tests": 12,
"passed": 0,
"failed": 12,
"pending": 12,
"metrics": {
"accuracy": {
"description": "评分准确性（与预期分数的偏差）",
"current": "N/A - 待测试",
"target": ">= 90%",
"status": "pending"
},
"consistency": {
"description": "多次评审同一产出的一致性",
"current": "N/A - 待测试",
"target": ">= 95%",
"status": "pending"
},
"coverage": {
"description": "检测到的缺陷覆盖率",
"current": "N/A - 待测试",
"target": ">= 85%",
"status": "pending"
},
"redline_detection": {
"description": "红线违规检出率",
"current": "N/A - 待测试",
"target": "100%",
"status": "pending"
},
"false_positive_rate": {
"description": "误报率",
"current": "N/A - 待测试",
"target": "<= 10%",
"status": "pending"
}
}
},
"capability_gaps": [
{
"id": "GAP-001",
"category": "multimodal",
"severity": "high",
"description": "缺少图像、音频、视频评审能力",
"impact": "无法评审设计稿截图、视频教程等多媒体产出",
"evidence": "当前技能仅支持 text/code/design/data，无 image/audio/video 支持"
},
{
"id": "GAP-002",
"category": "consistency",
"severity": "high",
"description": "缺少跨文档一致性检测机制",
"impact": "无法检测多个产出之间的命名不一致、逻辑冲突",
"evidence": "design_review.md 有一致性检查，但无自动化跨文档比对"
},
{
"id": "GAP-003",
"category": "automation",
"severity": "medium",
"description": "缺少自动化测试框架",
"impact": "无法批量运行测试、回归验证",
"evidence": "无测试脚本或 CI/CD 集成"
},
{
"id": "GAP-004",
"category": "scoring",
"severity": "medium",
"description": "评分算法过于简单，缺乏权重动态调整",
"impact": "评分可能不够精准，难以适应不同场景",
"evidence": "扣分标准固定，无场景自适应机制"
},
{
"id": "GAP-005",
"category": "tracking",
"severity": "medium",
"description": "缺少 Agent 能力进化追踪",
"impact": "无法量化评估重训效果",
"evidence": "无历史评分对比和能力趋势分析"
},
{
"id": "GAP-006",
"category": "explanation",
"severity": "low",
"description": "缺陷解释不够详细，缺少修复示例",
"impact": "执行 Agent 可能难以理解如何修正",
"evidence": "feedback_to_agent 字段要求简洁，但可能过于简略"
}
],
"strengths": [
{
"id": "STR-001",
"description": "完整的三级判决体系（PASS/REWORK/RETRAIN_REQUIRED）"
},
{
"id": "STR-002",
"description": "明确的红线违规检测机制"
},
{
"id": "STR-003",
"description": "结构化的 JSON 输出协议"
},
{
"id": "STR-004",
"description": "缺陷溯源分类（粗心 vs 能力缺陷）"
},
{
"id": "STR-005",
"description": "多维度评分权重体系"
}
]
}