{
  "test_suite": "QA Judge Capability Benchmark",
  "version": "1.0.0",
  "created_at": "2026-03-25",
  "test_cases": [
    {
      "id": "TC-001",
      "name": "代码安全漏洞检测 - SQL注入",
      "category": "security",
      "difficulty": "easy",
      "type": "code",
      "artifact": {
        "language": "javascript",
        "content": "// 不合格样本 - 存在SQL注入\nfunction getUser(userId) {\n  const query = \"SELECT * FROM users WHERE id = \" + userId;\n  return db.query(query);\n}"
      },
      "expected": {
        "status": "RETRAIN_REQUIRED",
        "min_score": 0,
        "max_score": 49,
        "redline_violations": ["SQL注入"],
        "severity": "critical"
      }
    },
    {
      "id": "TC-002",
      "name": "代码安全漏洞检测 - 硬编码密钥",
      "category": "security",
      "difficulty": "easy",
      "type": "code",
      "artifact": {
        "language": "python",
        "content": "# 不合格样本 - 硬编码密钥\nAPI_KEY = 'sk-live-1234567890abcdef'\n\ndef call_api():\n    headers = {'Authorization': f'Bearer {API_KEY}'}\n    return requests.get('https://api.example.com/data', headers=headers)"
      },
      "expected": {
        "status": "RETRAIN_REQUIRED",
        "min_score": 0,
        "max_score": 49,
        "redline_violations": ["硬编码密钥"],
        "severity": "critical"
      }
    },
    {
      "id": "TC-003",
      "name": "代码质量 - 优秀样本",
      "category": "code_quality",
      "difficulty": "medium",
      "type": "code",
      "artifact": {
        "language": "javascript",
        "content": "// 优秀样本 - 安全、规范、可维护\nconst db = require('./db');\nconst { validateUserId } = require('./validators');\n\n/**\n * 根据ID获取用户信息\n * @param {string} userId - 用户ID\n * @returns {Promise<Object|null>} 用户信息或null\n * @throws {ValidationError} 当userId格式无效时\n */\nasync function getUser(userId) {\n  // 输入验证\n  if (!validateUserId(userId)) {\n    throw new ValidationError('Invalid user ID format');\n  }\n  \n  // 使用参数化查询防止SQL注入\n  const query = 'SELECT id, name, email FROM users WHERE id = ?';\n  const result = await db.query(query, [userId]);\n  \n  return result[0] || null;\n}\n\nmodule.exports = { getUser };"
      },
      "expected": {
        "status": "PASS",
        "min_score": 90,
        "max_score": 100,
        "redline_violations": [],
        "severity": "none"
      }
    },
    {
      "id": "TC-004",
      "name": "代码质量 - 合格但需改进",
      "category": "code_quality",
      "difficulty": "medium",
      "type": "code",
      "artifact": {
        "language": "python",
        "content": "# 合格样本 - 功能正确但缺少文档\ndef calculate_discount(price, discount_rate):\n    if price < 0 or discount_rate < 0 or discount_rate > 1:\n        raise ValueError('Invalid input')\n    return price * (1 - discount_rate)"
      },
      "expected": {
        "status": "REWORK",
        "min_score": 50,
        "max_score": 89,
        "redline_violations": [],
        "severity": "minor"
      }
    },
    {
      "id": "TC-005",
      "name": "文本内容 - 优秀样本",
      "category": "text_quality",
      "difficulty": "medium",
      "type": "text",
      "original_task": "写一篇关于AI安全的科普文章，要求：1. 结构清晰 2. 语言通俗易懂 3. 包含实际案例",
      "artifact": {
        "content": "# AI安全：守护智能时代的数字防线\n\n## 引言\n随着人工智能技术的快速发展，AI安全已成为全球关注的焦点。本文将带您了解AI安全的核心概念和实际案例。\n\n## 什么是AI安全？\nAI安全是指保护人工智能系统免受攻击、滥用和意外行为的措施。它包括：\n- **数据安全**：防止训练数据被污染\n- **模型安全**：防止模型被窃取或篡改\n- **应用安全**：防止AI被用于恶意目的\n\n## 真实案例\n### 案例一：数据投毒攻击\n2023年，某公司的推荐系统因训练数据被投毒，导致推荐结果严重偏差，造成数百万损失。\n\n### 案例二：对抗样本攻击\n研究人员发现，通过在停车标志上添加特定图案，可以欺骗自动驾驶汽车的识别系统。\n\n## 如何防范？\n1. 建立数据验证流程\n2. 实施模型安全测试\n3. 加强访问控制\n\n## 结语\nAI安全是一个持续演进的领域，需要技术人员、政策制定者和公众的共同努力。"
      },
      "expected": {
        "status": "PASS",
        "min_score": 90,
        "max_score": 100,
        "completeness": 95,
        "correctness": 90
      }
    },
    {
      "id": "TC-006",
      "name": "文本内容 - 不合格样本",
      "category": "text_quality",
      "difficulty": "easy",
      "type": "text",
      "original_task": "写一篇关于AI安全的科普文章，要求：1. 结构清晰 2. 语言通俗易懂 3. 包含实际案例",
      "artifact": {
        "content": "AI安全很重要。我们要注意AI安全。AI有很多安全问题。大家都要小心。"
      },
      "expected": {
        "status": "RETRAIN_REQUIRED",
        "min_score": 0,
        "max_score": 49,
        "completeness": 20,
        "correctness": 30
      }
    },
    {
      "id": "TC-007",
      "name": "设计稿评审 - 一致性检查",
      "category": "design_review",
      "difficulty": "hard",
      "type": "design",
      "artifact": {
        "description": "登录页面设计稿分析",
        "issues": [
          "导航栏高度不一致：首页64px，登录页56px",
          "按钮颜色不统一：主要按钮使用#1890ff和#1677ff两种蓝色",
          "字体大小混乱：标题使用24px和28px两种规格"
        ]
      },
      "expected": {
        "status": "REWORK",
        "min_score": 50,
        "max_score": 75,
        "consistency_issues": 3
      }
    },
    {
      "id": "TC-008",
      "name": "数据格式 - 优秀样本",
      "category": "data_quality",
      "difficulty": "easy",
      "type": "data",
      "artifact": {
        "format": "json",
        "content": "{\n  \"users\": [\n    {\n      \"id\": \"u001\",\n      \"name\": \"张三\",\n      \"email\": \"zhangsan@example.com\",\n      \"created_at\": \"2024-01-15T08:30:00Z\"\n    },\n    {\n      \"id\": \"u002\",\n      \"name\": \"李四\",\n      \"email\": \"lisi@example.com\",\n      \"created_at\": \"2024-01-16T09:00:00Z\"\n    }\n  ],\n  \"total\": 2,\n  \"page\": 1\n}"
      },
      "expected": {
        "status": "PASS",
        "min_score": 90,
        "max_score": 100,
        "format_valid": true,
        "completeness": 100
      }
    },
    {
      "id": "TC-009",
      "name": "数据格式 - 不合格样本",
      "category": "data_quality",
      "difficulty": "easy",
      "type": "data",
      "artifact": {
        "format": "json",
        "content": "{\n  users: [\n    {\n      id: u001,\n      name: 张三\n    }\n  ]\n}"
      },
      "expected": {
        "status": "REWORK",
        "min_score": 50,
        "max_score": 70,
        "format_valid": false,
        "errors": ["缺少引号", "格式不规范"]
      }
    },
    {
      "id": "TC-010",
      "name": "多模态混合产出 - 复杂场景",
      "category": "mixed",
      "difficulty": "hard",
      "type": "mixed",
      "artifact": {
        "components": [
          {"type": "code", "quality": "good"},
          {"type": "doc", "quality": "poor"},
          {"type": "config", "quality": "good"}
        ]
      },
      "expected": {
        "status": "REWORK",
        "min_score": 60,
        "max_score": 80,
        "cross_consistency_check": true
      }
    },
    {
      "id": "TC-011",
      "name": "边界条件处理 - 优秀样本",
      "category": "edge_cases",
      "difficulty": "hard",
      "type": "code",
      "artifact": {
        "language": "javascript",
        "content": "function divide(a, b) {\n  // 边界条件检查\n  if (typeof a !== 'number' || typeof b !== 'number') {\n    throw new TypeError('Arguments must be numbers');\n  }\n  if (b === 0) {\n    throw new Error('Division by zero');\n  }\n  if (!isFinite(a) || !isFinite(b)) {\n    throw new Error('Arguments must be finite numbers');\n  }\n  return a / b;\n}"
      },
      "expected": {
        "status": "PASS",
        "min_score": 95,
        "max_score": 100,
        "edge_cases_covered": ["类型检查", "除零检查", "无穷大检查"]
      }
    },
    {
      "id": "TC-012",
      "name": "边界条件处理 - 不合格样本",
      "category": "edge_cases",
      "difficulty": "medium",
      "type": "code",
      "artifact": {
        "language": "javascript",
        "content": "function divide(a, b) {\n  return a / b;\n}"
      },
      "expected": {
        "status": "REWORK",
        "min_score": 50,
        "max_score": 70,
        "missing_edge_cases": ["类型检查", "除零检查"]
      }
    }
  ]
}