train_dataset.json 9.02 KB
{
  "version": "1.0.0",
  "agent": "youtube-crawler",
  "description": "YouTube 爬虫训练数据集 - 覆盖多地区多分类场景",
  "generated_at": "2026-04-01T00:00:00Z",
  "total_cases": 24,
  "cases": [
    {
      "id": "TRAIN_001",
      "dimension": "api_correctness",
      "type": "api_request",
      "description": "基础 API 调用 - US 地区",
      "input": {
        "method": "GET",
        "url": "https://www.googleapis.com/youtube/v3/videos",
        "params": {
          "part": "snippet,statistics,contentDetails",
          "chart": "mostPopular",
          "regionCode": "US",
          "maxResults": 50,
          "key": "${YOUTUBE_API_KEY}"
        }
      },
      "expected": {
        "status": 200,
        "items_count": 50,
        "required_params": ["chart=mostPopular", "part=snippet,statistics,contentDetails"]
      }
    },
    {
      "id": "TRAIN_002",
      "dimension": "api_correctness",
      "type": "api_request",
      "description": "API 调用 - JP 地区",
      "input": {
        "method": "GET",
        "url": "https://www.googleapis.com/youtube/v3/videos",
        "params": {
          "part": "snippet,statistics",
          "chart": "mostPopular",
          "regionCode": "JP",
          "maxResults": 50
        }
      },
      "expected": {
        "status": 200,
        "items_count": 50,
        "region": "JP"
      }
    },
    {
      "id": "TRAIN_003",
      "dimension": "api_correctness",
      "type": "field_validation",
      "description": "验证响应字段完整性",
      "input": {
        "api_response_sample": {
          "id": "dQw4w9WgXcQ",
          "snippet": { "title": "...", "channelTitle": "..." },
          "statistics": { "viewCount": "1234567890" }
        }
      },
      "expected": {
        "required_fields": ["id", "snippet.title", "snippet.channelTitle", "statistics.viewCount"],
        "all_present": true
      }
    },
    {
      "id": "TRAIN_004",
      "dimension": "multi_region",
      "type": "multi_region_request",
      "description": "批量获取 5 个地区数据",
      "input": {
        "regions": ["US", "JP", "KR", "GB", "DE"],
        "batch_size": 5
      },
      "expected": {
        "success_count": 5,
        "each_region_min_items": 10,
        "quota_cost": 5
      }
    },
    {
      "id": "TRAIN_005",
      "dimension": "multi_region",
      "type": "region_request",
      "description": "KR 地区热门视频",
      "input": {
        "regionCode": "KR",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "region": "KR",
        "items_count": 50
      }
    },
    {
      "id": "TRAIN_006",
      "dimension": "multi_region",
      "type": "region_request",
      "description": "FR 地区热门视频",
      "input": {
        "regionCode": "FR",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "region": "FR"
      }
    },
    {
      "id": "TRAIN_007",
      "dimension": "multi_region",
      "type": "region_request",
      "description": "IN 地区热门视频",
      "input": {
        "regionCode": "IN",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "region": "IN"
      }
    },
    {
      "id": "TRAIN_008",
      "dimension": "multi_category",
      "type": "category_request",
      "description": "Music 分类 (10) - US 地区",
      "input": {
        "regionCode": "US",
        "videoCategoryId": "10",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "categoryId": "10",
        "all_items_category": "10"
      }
    },
    {
      "id": "TRAIN_009",
      "dimension": "multi_category",
      "type": "category_request",
      "description": "Gaming 分类 (20) - JP 地区",
      "input": {
        "regionCode": "JP",
        "videoCategoryId": "20",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "categoryId": "20"
      }
    },
    {
      "id": "TRAIN_010",
      "dimension": "multi_category",
      "type": "category_request",
      "description": "Entertainment 分类 (24)",
      "input": {
        "regionCode": "US",
        "videoCategoryId": "24",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "categoryId": "24"
      }
    },
    {
      "id": "TRAIN_011",
      "dimension": "multi_category",
      "type": "category_request",
      "description": "Science & Technology 分类 (28)",
      "input": {
        "regionCode": "GB",
        "videoCategoryId": "28",
        "maxResults": 50
      },
      "expected": {
        "status": 200,
        "categoryId": "28"
      }
    },
    {
      "id": "TRAIN_012",
      "dimension": "multi_category",
      "type": "multi_category_request",
      "description": "批量获取 4 个分类",
      "input": {
        "regionCode": "US",
        "categories": ["10", "20", "24", "28"]
      },
      "expected": {
        "success_count": 4,
        "total_quota_cost": 4
      }
    },
    {
      "id": "TRAIN_013",
      "dimension": "quota_management",
      "type": "quota_tracking",
      "description": "配额使用监控",
      "input": {
        "requests_count": 100,
        "expected_quota_used": 100
      },
      "expected": {
        "quota_used": 100,
        "quota_remaining": 9900,
        "tracking_accuracy": 100
      }
    },
    {
      "id": "TRAIN_014",
      "dimension": "quota_management",
      "type": "cache_test",
      "description": "缓存机制验证",
      "input": {
        "same_request_count": 3,
        "cache_ttl": 900
      },
      "expected": {
        "actual_api_calls": 1,
        "cache_hits": 2,
        "quota_saved": 2
      }
    },
    {
      "id": "TRAIN_015",
      "dimension": "quota_management",
      "type": "optimization_test",
      "description": "maxResults 优化验证",
      "input": {
        "maxResults": 50,
        "items_per_request": 50
      },
      "expected": {
        "quota_efficiency": "optimal",
        "cost_per_video": 0.02
      }
    },
    {
      "id": "TRAIN_016",
      "dimension": "quota_management",
      "type": "fallback_test",
      "description": "配额耗尽降级到 yt-dlp",
      "input": {
        "quota_exhausted": true
      },
      "expected": {
        "fallback_triggered": true,
        "fallback_method": "yt-dlp",
        "data_still_available": true
      }
    },
    {
      "id": "TRAIN_017",
      "dimension": "error_handling",
      "type": "error_400",
      "description": "无效 regionCode 处理",
      "input": {
        "regionCode": "XX"
      },
      "expected": {
        "status": 400,
        "error_reason": "badRequest",
        "handled_gracefully": true
      }
    },
    {
      "id": "TRAIN_018",
      "dimension": "error_handling",
      "type": "error_403_key",
      "description": "无效 API Key 处理",
      "input": {
        "api_key": "invalid_key"
      },
      "expected": {
        "status": 403,
        "error_reason": "accessNotConfigured",
        "clear_error_message": true
      }
    },
    {
      "id": "TRAIN_019",
      "dimension": "error_handling",
      "type": "error_429",
      "description": "速率限制处理",
      "input": {
        "simulate_429": true
      },
      "expected": {
        "backoff_triggered": true,
        "retry_count": 3,
        "success_after_retry": true
      }
    },
    {
      "id": "TRAIN_020",
      "dimension": "error_handling",
      "type": "error_404",
      "description": "资源不存在处理",
      "input": {
        "video_id": "nonexistent123"
      },
      "expected": {
        "status": 404,
        "skipped_gracefully": true
      }
    },
    {
      "id": "TRAIN_021",
      "dimension": "data_export",
      "type": "json_export",
      "description": "JSON 格式导出验证",
      "input": {
        "format": "json",
        "videos_count": 50
      },
      "expected": {
        "file_valid": true,
        "required_fields": ["videoId", "title", "viewCount", "channelTitle"],
        "encoding": "UTF-8"
      }
    },
    {
      "id": "TRAIN_022",
      "dimension": "data_export",
      "type": "csv_export",
      "description": "CSV 格式导出验证",
      "input": {
        "format": "csv",
        "videos_count": 50
      },
      "expected": {
        "file_valid": true,
        "has_header": true,
        "delimiter": ",",
        "quoted_fields": true
      }
    },
    {
      "id": "TRAIN_023",
      "dimension": "data_export",
      "type": "field_mapping",
      "description": "字段映射正确性",
      "input": {
        "source": "api",
        "target_fields": ["videoId", "url", "title", "viewCount"]
      },
      "expected": {
        "mapping_correct": true,
        "url_format": "https://youtube.com/watch?v={videoId}"
      }
    },
    {
      "id": "TRAIN_024",
      "dimension": "data_export",
      "type": "ytdlp_fallback_export",
      "description": "yt-dlp 数据导出",
      "input": {
        "method": "yt-dlp",
        "format": "json"
      },
      "expected": {
        "export_success": true,
        "fields_compatible": true
      }
    }
  ]
}