test_cases.json
8.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
{
"test_suite": "QA Judge Capability Benchmark",
"version": "1.0.0",
"created_at": "2026-03-25",
"test_cases": [
{
"id": "TC-001",
"name": "代码安全漏洞检测 - SQL注入",
"category": "security",
"difficulty": "easy",
"type": "code",
"artifact": {
"language": "javascript",
"content": "// 不合格样本 - 存在SQL注入\nfunction getUser(userId) {\n const query = \"SELECT * FROM users WHERE id = \" + userId;\n return db.query(query);\n}"
},
"expected": {
"status": "RETRAIN_REQUIRED",
"min_score": 0,
"max_score": 49,
"redline_violations": ["SQL注入"],
"severity": "critical"
}
},
{
"id": "TC-002",
"name": "代码安全漏洞检测 - 硬编码密钥",
"category": "security",
"difficulty": "easy",
"type": "code",
"artifact": {
"language": "python",
"content": "# 不合格样本 - 硬编码密钥\nAPI_KEY = 'sk-live-1234567890abcdef'\n\ndef call_api():\n headers = {'Authorization': f'Bearer {API_KEY}'}\n return requests.get('https://api.example.com/data', headers=headers)"
},
"expected": {
"status": "RETRAIN_REQUIRED",
"min_score": 0,
"max_score": 49,
"redline_violations": ["硬编码密钥"],
"severity": "critical"
}
},
{
"id": "TC-003",
"name": "代码质量 - 优秀样本",
"category": "code_quality",
"difficulty": "medium",
"type": "code",
"artifact": {
"language": "javascript",
"content": "// 优秀样本 - 安全、规范、可维护\nconst db = require('./db');\nconst { validateUserId } = require('./validators');\n\n/**\n * 根据ID获取用户信息\n * @param {string} userId - 用户ID\n * @returns {Promise<Object|null>} 用户信息或null\n * @throws {ValidationError} 当userId格式无效时\n */\nasync function getUser(userId) {\n // 输入验证\n if (!validateUserId(userId)) {\n throw new ValidationError('Invalid user ID format');\n }\n \n // 使用参数化查询防止SQL注入\n const query = 'SELECT id, name, email FROM users WHERE id = ?';\n const result = await db.query(query, [userId]);\n \n return result[0] || null;\n}\n\nmodule.exports = { getUser };"
},
"expected": {
"status": "PASS",
"min_score": 90,
"max_score": 100,
"redline_violations": [],
"severity": "none"
}
},
{
"id": "TC-004",
"name": "代码质量 - 合格但需改进",
"category": "code_quality",
"difficulty": "medium",
"type": "code",
"artifact": {
"language": "python",
"content": "# 合格样本 - 功能正确但缺少文档\ndef calculate_discount(price, discount_rate):\n if price < 0 or discount_rate < 0 or discount_rate > 1:\n raise ValueError('Invalid input')\n return price * (1 - discount_rate)"
},
"expected": {
"status": "REWORK",
"min_score": 50,
"max_score": 89,
"redline_violations": [],
"severity": "minor"
}
},
{
"id": "TC-005",
"name": "文本内容 - 优秀样本",
"category": "text_quality",
"difficulty": "medium",
"type": "text",
"original_task": "写一篇关于AI安全的科普文章，要求：1. 结构清晰 2. 语言通俗易懂 3. 包含实际案例",
"artifact": {
"content": "# AI安全：守护智能时代的数字防线\n\n## 引言\n随着人工智能技术的快速发展，AI安全已成为全球关注的焦点。本文将带您了解AI安全的核心概念和实际案例。\n\n## 什么是AI安全？\nAI安全是指保护人工智能系统免受攻击、滥用和意外行为的措施。它包括：\n- **数据安全**：防止训练数据被污染\n- **模型安全**：防止模型被窃取或篡改\n- **应用安全**：防止AI被用于恶意目的\n\n## 真实案例\n### 案例一：数据投毒攻击\n2023年，某公司的推荐系统因训练数据被投毒，导致推荐结果严重偏差，造成数百万损失。\n\n### 案例二：对抗样本攻击\n研究人员发现，通过在停车标志上添加特定图案，可以欺骗自动驾驶汽车的识别系统。\n\n## 如何防范？\n1. 建立数据验证流程\n2. 实施模型安全测试\n3. 加强访问控制\n\n## 结语\nAI安全是一个持续演进的领域，需要技术人员、政策制定者和公众的共同努力。"
},
"expected": {
"status": "PASS",
"min_score": 90,
"max_score": 100,
"completeness": 95,
"correctness": 90
}
},
{
"id": "TC-006",
"name": "文本内容 - 不合格样本",
"category": "text_quality",
"difficulty": "easy",
"type": "text",
"original_task": "写一篇关于AI安全的科普文章，要求：1. 结构清晰 2. 语言通俗易懂 3. 包含实际案例",
"artifact": {
"content": "AI安全很重要。我们要注意AI安全。AI有很多安全问题。大家都要小心。"
},
"expected": {
"status": "RETRAIN_REQUIRED",
"min_score": 0,
"max_score": 49,
"completeness": 20,
"correctness": 30
}
},
{
"id": "TC-007",
"name": "设计稿评审 - 一致性检查",
"category": "design_review",
"difficulty": "hard",
"type": "design",
"artifact": {
"description": "登录页面设计稿分析",
"issues": [
"导航栏高度不一致：首页64px，登录页56px",
"按钮颜色不统一：主要按钮使用#1890ff和#1677ff两种蓝色",
"字体大小混乱：标题使用24px和28px两种规格"
]
},
"expected": {
"status": "REWORK",
"min_score": 50,
"max_score": 75,
"consistency_issues": 3
}
},
{
"id": "TC-008",
"name": "数据格式 - 优秀样本",
"category": "data_quality",
"difficulty": "easy",
"type": "data",
"artifact": {
"format": "json",
"content": "{\n \"users\": [\n {\n \"id\": \"u001\",\n \"name\": \"张三\",\n \"email\": \"zhangsan@example.com\",\n \"created_at\": \"2024-01-15T08:30:00Z\"\n },\n {\n \"id\": \"u002\",\n \"name\": \"李四\",\n \"email\": \"lisi@example.com\",\n \"created_at\": \"2024-01-16T09:00:00Z\"\n }\n ],\n \"total\": 2,\n \"page\": 1\n}"
},
"expected": {
"status": "PASS",
"min_score": 90,
"max_score": 100,
"format_valid": true,
"completeness": 100
}
},
{
"id": "TC-009",
"name": "数据格式 - 不合格样本",
"category": "data_quality",
"difficulty": "easy",
"type": "data",
"artifact": {
"format": "json",
"content": "{\n users: [\n {\n id: u001,\n name: 张三\n }\n ]\n}"
},
"expected": {
"status": "REWORK",
"min_score": 50,
"max_score": 70,
"format_valid": false,
"errors": ["缺少引号", "格式不规范"]
}
},
{
"id": "TC-010",
"name": "多模态混合产出 - 复杂场景",
"category": "mixed",
"difficulty": "hard",
"type": "mixed",
"artifact": {
"components": [
{"type": "code", "quality": "good"},
{"type": "doc", "quality": "poor"},
{"type": "config", "quality": "good"}
]
},
"expected": {
"status": "REWORK",
"min_score": 60,
"max_score": 80,
"cross_consistency_check": true
}
},
{
"id": "TC-011",
"name": "边界条件处理 - 优秀样本",
"category": "edge_cases",
"difficulty": "hard",
"type": "code",
"artifact": {
"language": "javascript",
"content": "function divide(a, b) {\n // 边界条件检查\n if (typeof a !== 'number' || typeof b !== 'number') {\n throw new TypeError('Arguments must be numbers');\n }\n if (b === 0) {\n throw new Error('Division by zero');\n }\n if (!isFinite(a) || !isFinite(b)) {\n throw new Error('Arguments must be finite numbers');\n }\n return a / b;\n}"
},
"expected": {
"status": "PASS",
"min_score": 95,
"max_score": 100,
"edge_cases_covered": ["类型检查", "除零检查", "无穷大检查"]
}
},
{
"id": "TC-012",
"name": "边界条件处理 - 不合格样本",
"category": "edge_cases",
"difficulty": "medium",
"type": "code",
"artifact": {
"language": "javascript",
"content": "function divide(a, b) {\n return a / b;\n}"
},
"expected": {
"status": "REWORK",
"min_score": 50,
"max_score": 70,
"missing_edge_cases": ["类型检查", "除零检查"]
}
}
]
}