""" YouTube Data API v3 - 验证过的 Python 代码片段 任务: T-mnevamap | Agent: Hunter | 日期: 2026-04-01 """ import os import csv import json import time import logging from datetime import datetime from urllib.parse import urlencode import requests logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s" ) logger = logging.getLogger(__name__) # ============================================================================= # 1. 基础 API 请求 (已验证) # ============================================================================= BASE_URL = "https://www.googleapis.com/youtube/v3/videos" def get_trending_videos( api_key: str, region_code: str = "US", category_id: str | None = None, max_results: int = 50, page_token: str | None = None, ) -> dict: """ 获取指定地区的 YouTube 热门视频。 配额成本: 1 unit / call """ params = { "part": "snippet,statistics,contentDetails", "chart": "mostPopular", "regionCode": region_code, "maxResults": min(max_results, 50), "key": api_key, } if category_id: params["videoCategoryId"] = category_id if page_token: params["pageToken"] = page_token response = requests.get(BASE_URL, params=params, timeout=30) response.raise_for_status() return response.json() # ============================================================================= # 2. 错误处理 (已验证: 429 / 403 / 404) # ============================================================================= def get_trending_videos_with_error_handling( api_key: str, region_code: str = "US", category_id: str | None = None, max_results: int = 50, max_retries: int = 3, ) -> dict: """ 带完整错误处理的热门视频获取。 处理逻辑: - 400: 参数错误 → 记录日志并跳过 - 403: API Key 无效 / 配额耗尽 → 直接抛出异常,停止执行 - 404: 资源不存在 → 记录日志并返回空结果 - 429: 速率限制 → 指数退避重试 - 5xx: 服务器错误 → 线性退避重试 """ params = { "part": "snippet,statistics,contentDetails", "chart": "mostPopular", "regionCode": region_code, "maxResults": min(max_results, 50), "key": api_key, } if category_id: params["videoCategoryId"] = category_id attempt = 0 backoff = 2 # 基础退避秒数 while attempt <= max_retries: try: response = requests.get(BASE_URL, params=params, timeout=30) if response.status_code == 200: return response.json() if response.status_code == 400: logger.error(f"[400 Bad Request] region={region_code}, msg={response.text}") return {"items": [], "error": "bad_request", "details": response.text} if response.status_code == 403: error_body = response.json() reason = error_body.get("error", {}).get("errors", [{}])[0].get("reason", "unknown") if reason == "quotaExceeded": logger.error("[403 Quota Exceeded] Daily quota (10000 units) exhausted. Stop.") raise RuntimeError("YouTube API quota exceeded. Wait for Pacific midnight reset.") logger.error(f"[403 Forbidden] reason={reason}, msg={response.text}") raise RuntimeError(f"YouTube API 403 Forbidden: {reason}") if response.status_code == 404: logger.warning(f"[404 Not Found] region={region_code}, msg={response.text}") return {"items": [], "error": "not_found", "details": response.text} if response.status_code == 429: attempt += 1 sleep_time = backoff * (2 ** (attempt - 1)) logger.warning(f"[429 Rate Limit] retry {attempt}/{max_retries} after {sleep_time}s") time.sleep(sleep_time) continue if 500 <= response.status_code < 600: attempt += 1 logger.warning(f"[{response.status_code} Server Error] retry {attempt}/{max_retries}") time.sleep(backoff) continue response.raise_for_status() except requests.exceptions.RequestException as exc: attempt += 1 logger.warning(f"[Network Error] {exc}, retry {attempt}/{max_retries}") time.sleep(backoff) logger.error(f"Max retries exceeded for region={region_code}") return {"items": [], "error": "max_retries_exceeded"} # ============================================================================= # 3. 数据提取 (已验证) # ============================================================================= def extract_video_info(item: dict, region_code: str, crawl_time: str) -> dict: """ 从 API 单条 item 中提取结构化视频信息。 注意: likeCount / commentCount 可能缺失 (视频关闭了点赞/评论)。 """ snippet = item.get("snippet", {}) stats = item.get("statistics", {}) content_details = item.get("contentDetails", {}) video_id = item.get("id", "") return { "video_id": video_id, "title": snippet.get("title", ""), "description": snippet.get("description", ""), "channel_id": snippet.get("channelId", ""), "channel_title": snippet.get("channelTitle", ""), "published_at": snippet.get("publishedAt", ""), "thumbnail_url": snippet.get("thumbnails", {}).get("high", {}).get("url", ""), "tags": snippet.get("tags", []), "category_id": snippet.get("categoryId", ""), "region_code": region_code, "view_count": int(stats.get("viewCount", 0)), "like_count": int(stats.get("likeCount", 0)) if "likeCount" in stats else None, "comment_count": int(stats.get("commentCount", 0)) if "commentCount" in stats else None, "duration": content_details.get("duration", ""), "source_url": f"https://www.youtube.com/watch?v={video_id}", "crawl_time": crawl_time, } # ============================================================================= # 4. 多地区批量采集 (已验证) # ============================================================================= def fetch_multi_region_trending( api_key: str, regions: list[str], category_id: str | None = None, max_results: int = 50, delay_seconds: float = 2.0, ) -> dict: """ 批量获取多地区热门视频,自动遵守频率限制。 Args: api_key: YouTube Data API Key regions: 地区代码列表,如 ["US", "JP", "KR"] category_id: 可选的视频分类 ID max_results: 每地区最大返回数 (1-50) delay_seconds: 请求间隔秒数 Returns: {region_code: [video_info, ...]} """ results = {} crawl_time = datetime.utcnow().isoformat() + "Z" for region in regions: logger.info(f"Fetching trending videos for region={region}") raw_data = get_trending_videos_with_error_handling( api_key=api_key, region_code=region, category_id=category_id, max_results=max_results, ) videos = [ extract_video_info(item, region, crawl_time) for item in raw_data.get("items", []) ] results[region] = videos logger.info(f"Region={region}: extracted {len(videos)} videos") if delay_seconds > 0: time.sleep(delay_seconds) return results # ============================================================================= # 5. 数据导出: JSON (已验证) # ============================================================================= def save_to_json(data: dict | list, filepath: str) -> str: """将数据保存为 JSON 文件。""" with open(filepath, "w", encoding="utf-8") as f: json.dump(data, f, ensure_ascii=False, indent=2) logger.info(f"Saved JSON to {filepath}") return filepath # ============================================================================= # 6. 数据导出: CSV (已验证) # ============================================================================= def save_to_csv(videos: list[dict], filepath: str) -> str: """将视频列表保存为 CSV 文件。""" if not videos: logger.warning("No videos to save to CSV") return filepath # 展开 tags 列表为逗号分隔字符串,避免 CSV 写入问题 flattened = [] for v in videos: row = dict(v) if isinstance(row.get("tags"), list): row["tags"] = "|".join(row["tags"]) flattened.append(row) keys = flattened[0].keys() with open(filepath, "w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=keys) writer.writeheader() writer.writerows(flattened) logger.info(f"Saved CSV to {filepath}") return filepath # ============================================================================= # 7. 获取分类列表 (辅助函数,已验证) # ============================================================================= def get_video_categories(api_key: str, region_code: str = "US") -> list[dict]: """ 获取指定地区可用的视频分类列表。 配额成本: 1 unit """ url = "https://www.googleapis.com/youtube/v3/videoCategories" params = { "part": "snippet", "regionCode": region_code, "key": api_key, } response = requests.get(url, params=params, timeout=30) response.raise_for_status() data = response.json() categories = [] for item in data.get("items", []): snippet = item.get("snippet", {}) categories.append({ "id": item.get("id"), "title": snippet.get("title"), "assignable": snippet.get("assignable", False), }) return categories # ============================================================================= # 8. 使用示例 (完整工作流) # ============================================================================= if __name__ == "__main__": API_KEY = os.getenv("YOUTUBE_API_KEY", "YOUR_API_KEY_HERE") if API_KEY == "YOUR_API_KEY_HERE": logger.warning("Please set YOUTUBE_API_KEY environment variable.") else: regions = ["US", "JP", "KR", "GB", "DE"] # 示例 1: 批量获取多地区热门视频 all_videos = fetch_multi_region_trending( api_key=API_KEY, regions=regions, category_id="10", # Music 分类 max_results=10, delay_seconds=2.0, ) # 示例 2: 保存为 JSON save_to_json(all_videos, "trending_videos.json") # 示例 3: 扁平化为 CSV (所有地区合并) flat_list = [] for region, videos in all_videos.items(): flat_list.extend(videos) save_to_csv(flat_list, "trending_videos.csv") # 示例 4: 获取地区分类列表 us_categories = get_video_categories(API_KEY, "US") logger.info(f"US has {len(us_categories)} video categories")