code_snippets.py 11 KB
"""
YouTube Data API v3 - 验证过的 Python 代码片段
任务: T-mnevamap | Agent: Hunter | 日期: 2026-04-01
"""

import os
import csv
import json
import time
import logging
from datetime import datetime
from urllib.parse import urlencode

import requests

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(message)s"
)
logger = logging.getLogger(__name__)


# =============================================================================
# 1. 基础 API 请求 (已验证)
# =============================================================================

BASE_URL = "https://www.googleapis.com/youtube/v3/videos"


def get_trending_videos(
    api_key: str,
    region_code: str = "US",
    category_id: str | None = None,
    max_results: int = 50,
    page_token: str | None = None,
) -> dict:
    """
    获取指定地区的 YouTube 热门视频。

    配额成本: 1 unit / call
    """
    params = {
        "part": "snippet,statistics,contentDetails",
        "chart": "mostPopular",
        "regionCode": region_code,
        "maxResults": min(max_results, 50),
        "key": api_key,
    }
    if category_id:
        params["videoCategoryId"] = category_id
    if page_token:
        params["pageToken"] = page_token

    response = requests.get(BASE_URL, params=params, timeout=30)
    response.raise_for_status()
    return response.json()


# =============================================================================
# 2. 错误处理 (已验证: 429 / 403 / 404)
# =============================================================================

def get_trending_videos_with_error_handling(
    api_key: str,
    region_code: str = "US",
    category_id: str | None = None,
    max_results: int = 50,
    max_retries: int = 3,
) -> dict:
    """
    带完整错误处理的热门视频获取。

    处理逻辑:
    - 400: 参数错误 → 记录日志并跳过
    - 403: API Key 无效 / 配额耗尽 → 直接抛出异常,停止执行
    - 404: 资源不存在 → 记录日志并返回空结果
    - 429: 速率限制 → 指数退避重试
    - 5xx: 服务器错误 → 线性退避重试
    """
    params = {
        "part": "snippet,statistics,contentDetails",
        "chart": "mostPopular",
        "regionCode": region_code,
        "maxResults": min(max_results, 50),
        "key": api_key,
    }
    if category_id:
        params["videoCategoryId"] = category_id

    attempt = 0
    backoff = 2  # 基础退避秒数

    while attempt <= max_retries:
        try:
            response = requests.get(BASE_URL, params=params, timeout=30)

            if response.status_code == 200:
                return response.json()

            if response.status_code == 400:
                logger.error(f"[400 Bad Request] region={region_code}, msg={response.text}")
                return {"items": [], "error": "bad_request", "details": response.text}

            if response.status_code == 403:
                error_body = response.json()
                reason = error_body.get("error", {}).get("errors", [{}])[0].get("reason", "unknown")
                if reason == "quotaExceeded":
                    logger.error("[403 Quota Exceeded] Daily quota (10000 units) exhausted. Stop.")
                    raise RuntimeError("YouTube API quota exceeded. Wait for Pacific midnight reset.")
                logger.error(f"[403 Forbidden] reason={reason}, msg={response.text}")
                raise RuntimeError(f"YouTube API 403 Forbidden: {reason}")

            if response.status_code == 404:
                logger.warning(f"[404 Not Found] region={region_code}, msg={response.text}")
                return {"items": [], "error": "not_found", "details": response.text}

            if response.status_code == 429:
                attempt += 1
                sleep_time = backoff * (2 ** (attempt - 1))
                logger.warning(f"[429 Rate Limit] retry {attempt}/{max_retries} after {sleep_time}s")
                time.sleep(sleep_time)
                continue

            if 500 <= response.status_code < 600:
                attempt += 1
                logger.warning(f"[{response.status_code} Server Error] retry {attempt}/{max_retries}")
                time.sleep(backoff)
                continue

            response.raise_for_status()

        except requests.exceptions.RequestException as exc:
            attempt += 1
            logger.warning(f"[Network Error] {exc}, retry {attempt}/{max_retries}")
            time.sleep(backoff)

    logger.error(f"Max retries exceeded for region={region_code}")
    return {"items": [], "error": "max_retries_exceeded"}


# =============================================================================
# 3. 数据提取 (已验证)
# =============================================================================

def extract_video_info(item: dict, region_code: str, crawl_time: str) -> dict:
    """
    从 API 单条 item 中提取结构化视频信息。

    注意: likeCount / commentCount 可能缺失 (视频关闭了点赞/评论)。
    """
    snippet = item.get("snippet", {})
    stats = item.get("statistics", {})
    content_details = item.get("contentDetails", {})
    video_id = item.get("id", "")

    return {
        "video_id": video_id,
        "title": snippet.get("title", ""),
        "description": snippet.get("description", ""),
        "channel_id": snippet.get("channelId", ""),
        "channel_title": snippet.get("channelTitle", ""),
        "published_at": snippet.get("publishedAt", ""),
        "thumbnail_url": snippet.get("thumbnails", {}).get("high", {}).get("url", ""),
        "tags": snippet.get("tags", []),
        "category_id": snippet.get("categoryId", ""),
        "region_code": region_code,
        "view_count": int(stats.get("viewCount", 0)),
        "like_count": int(stats.get("likeCount", 0)) if "likeCount" in stats else None,
        "comment_count": int(stats.get("commentCount", 0)) if "commentCount" in stats else None,
        "duration": content_details.get("duration", ""),
        "source_url": f"https://www.youtube.com/watch?v={video_id}",
        "crawl_time": crawl_time,
    }


# =============================================================================
# 4. 多地区批量采集 (已验证)
# =============================================================================

def fetch_multi_region_trending(
    api_key: str,
    regions: list[str],
    category_id: str | None = None,
    max_results: int = 50,
    delay_seconds: float = 2.0,
) -> dict:
    """
    批量获取多地区热门视频,自动遵守频率限制。

    Args:
        api_key: YouTube Data API Key
        regions: 地区代码列表,如 ["US", "JP", "KR"]
        category_id: 可选的视频分类 ID
        max_results: 每地区最大返回数 (1-50)
        delay_seconds: 请求间隔秒数

    Returns:
        {region_code: [video_info, ...]}
    """
    results = {}
    crawl_time = datetime.utcnow().isoformat() + "Z"

    for region in regions:
        logger.info(f"Fetching trending videos for region={region}")
        raw_data = get_trending_videos_with_error_handling(
            api_key=api_key,
            region_code=region,
            category_id=category_id,
            max_results=max_results,
        )

        videos = [
            extract_video_info(item, region, crawl_time)
            for item in raw_data.get("items", [])
        ]
        results[region] = videos
        logger.info(f"Region={region}: extracted {len(videos)} videos")

        if delay_seconds > 0:
            time.sleep(delay_seconds)

    return results


# =============================================================================
# 5. 数据导出: JSON (已验证)
# =============================================================================

def save_to_json(data: dict | list, filepath: str) -> str:
    """将数据保存为 JSON 文件。"""
    with open(filepath, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=2)
    logger.info(f"Saved JSON to {filepath}")
    return filepath


# =============================================================================
# 6. 数据导出: CSV (已验证)
# =============================================================================

def save_to_csv(videos: list[dict], filepath: str) -> str:
    """将视频列表保存为 CSV 文件。"""
    if not videos:
        logger.warning("No videos to save to CSV")
        return filepath

    # 展开 tags 列表为逗号分隔字符串,避免 CSV 写入问题
    flattened = []
    for v in videos:
        row = dict(v)
        if isinstance(row.get("tags"), list):
            row["tags"] = "|".join(row["tags"])
        flattened.append(row)

    keys = flattened[0].keys()
    with open(filepath, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=keys)
        writer.writeheader()
        writer.writerows(flattened)

    logger.info(f"Saved CSV to {filepath}")
    return filepath


# =============================================================================
# 7. 获取分类列表 (辅助函数,已验证)
# =============================================================================

def get_video_categories(api_key: str, region_code: str = "US") -> list[dict]:
    """
    获取指定地区可用的视频分类列表。
    配额成本: 1 unit
    """
    url = "https://www.googleapis.com/youtube/v3/videoCategories"
    params = {
        "part": "snippet",
        "regionCode": region_code,
        "key": api_key,
    }
    response = requests.get(url, params=params, timeout=30)
    response.raise_for_status()

    data = response.json()
    categories = []
    for item in data.get("items", []):
        snippet = item.get("snippet", {})
        categories.append({
            "id": item.get("id"),
            "title": snippet.get("title"),
            "assignable": snippet.get("assignable", False),
        })
    return categories


# =============================================================================
# 8. 使用示例 (完整工作流)
# =============================================================================

if __name__ == "__main__":
    API_KEY = os.getenv("YOUTUBE_API_KEY", "YOUR_API_KEY_HERE")

    if API_KEY == "YOUR_API_KEY_HERE":
        logger.warning("Please set YOUTUBE_API_KEY environment variable.")
    else:
        regions = ["US", "JP", "KR", "GB", "DE"]

        # 示例 1: 批量获取多地区热门视频
        all_videos = fetch_multi_region_trending(
            api_key=API_KEY,
            regions=regions,
            category_id="10",      # Music 分类
            max_results=10,
            delay_seconds=2.0,
        )

        # 示例 2: 保存为 JSON
        save_to_json(all_videos, "trending_videos.json")

        # 示例 3: 扁平化为 CSV (所有地区合并)
        flat_list = []
        for region, videos in all_videos.items():
            flat_list.extend(videos)
        save_to_csv(flat_list, "trending_videos.csv")

        # 示例 4: 获取地区分类列表
        us_categories = get_video_categories(API_KEY, "US")
        logger.info(f"US has {len(us_categories)} video categories")