crawl_config.json 6.24 KB
{
  "meta": {
    "name": "youtube-trending-crawler",
    "version": "1.0.0",
    "description": "YouTube Data API v3 trending video crawler configuration",
    "created_at": "2026-04-01T01:30:00Z",
    "created_by": "Hunter",
    "source_api": "YouTube Data API v3",
    "base_endpoint": "https://www.googleapis.com/youtube/v3/videos"
  },
  "auth": {
    "type": "api_key",
    "env_var": "YOUTUBE_API_KEY",
    "required": true,
    "description": "Google Cloud API Key with YouTube Data API v3 enabled"
  },
  "request": {
    "method": "GET",
    "headers": {
      "Accept": "application/json",
      "Accept-Encoding": "gzip",
      "User-Agent": "Mozilla/5.0 (compatible; YouTubeCrawler/1.0; +http://example.com/bot)"
    },
    "timeout_seconds": 30,
    "max_retries": 3,
    "retry_backoff_base_seconds": 2,
    "request_delay_seconds": 2.0,
    "respect_robots_txt": true
  },
  "parameters": {
    "default": {
      "part": "snippet,statistics,contentDetails",
      "chart": "mostPopular",
      "maxResults": 50
    },
    "optional": {
      "regionCode": "{region_code}",
      "videoCategoryId": "{category_id}",
      "pageToken": "{page_token}"
    }
  },
  "regions": {
    "primary": [
      { "code": "US", "name": "United States" },
      { "code": "JP", "name": "Japan" },
      { "code": "KR", "name": "South Korea" },
      { "code": "GB", "name": "United Kingdom" },
      { "code": "DE", "name": "Germany" },
      { "code": "FR", "name": "France" },
      { "code": "IN", "name": "India" },
      { "code": "BR", "name": "Brazil" },
      { "code": "RU", "name": "Russia" }
    ],
    "extended": [
      { "code": "AU", "name": "Australia" },
      { "code": "CA", "name": "Canada" },
      { "code": "MX", "name": "Mexico" },
      { "code": "AR", "name": "Argentina" },
      { "code": "CL", "name": "Chile" },
      { "code": "ES", "name": "Spain" },
      { "code": "IT", "name": "Italy" },
      { "code": "NL", "name": "Netherlands" },
      { "code": "SE", "name": "Sweden" },
      { "code": "TR", "name": "Turkey" },
      { "code": "ID", "name": "Indonesia" },
      { "code": "TH", "name": "Thailand" },
      { "code": "VN", "name": "Vietnam" },
      { "code": "PH", "name": "Philippines" },
      { "code": "EG", "name": "Egypt" },
      { "code": "ZA", "name": "South Africa" }
    ]
  },
  "categories": {
    "target": [
      { "id": "10", "name": "Music", "enabled": true },
      { "id": "20", "name": "Gaming", "enabled": true },
      { "id": "24", "name": "Entertainment", "enabled": true },
      { "id": "28", "name": "Science & Technology", "enabled": true }
    ],
    "extended": [
      { "id": "1", "name": "Film & Animation", "enabled": false },
      { "id": "2", "name": "Autos & Vehicles", "enabled": false },
      { "id": "15", "name": "Pets & Animals", "enabled": false },
      { "id": "17", "name": "Sports", "enabled": false },
      { "id": "19", "name": "Travel & Events", "enabled": false },
      { "id": "22", "name": "People & Blogs", "enabled": false },
      { "id": "23", "name": "Comedy", "enabled": false },
      { "id": "25", "name": "News & Politics", "enabled": false },
      { "id": "26", "name": "Howto & Style", "enabled": false },
      { "id": "27", "name": "Education", "enabled": false }
    ]
  },
  "quota": {
    "daily_limit": 10000,
    "cost_per_call": 1,
    "max_results_per_call": 50,
    "estimated_max_videos_per_day": 500000,
    "rate_limit": {
      "requests_per_second": 5,
      "requests_per_minute": 180,
      "note": "Google does not publish exact per-minute limits. Use conservative pacing."
    }
  },
  "parsing_rules": {
    "video_id": "$.items[*].id",
    "title": "$.items[*].snippet.title",
    "description": "$.items[*].snippet.description",
    "channel_id": "$.items[*].snippet.channelId",
    "channel_title": "$.items[*].snippet.channelTitle",
    "published_at": "$.items[*].snippet.publishedAt",
    "thumbnail_url": "$.items[*].snippet.thumbnails.high.url",
    "tags": "$.items[*].snippet.tags",
    "category_id": "$.items[*].snippet.categoryId",
    "view_count": "$.items[*].statistics.viewCount",
    "like_count": "$.items[*].statistics.likeCount",
    "comment_count": "$.items[*].statistics.commentCount",
    "duration": "$.items[*].contentDetails.duration",
    "next_page_token": "$.nextPageToken",
    "total_results": "$.pageInfo.totalResults"
  },
  "data_schema": {
    "required_fields": [
      "video_id",
      "title",
      "channel_id",
      "channel_title",
      "published_at",
      "view_count",
      "category_id",
      "source_url",
      "crawl_time"
    ],
    "optional_fields": [
      "description",
      "thumbnail_url",
      "tags",
      "like_count",
      "comment_count",
      "duration"
    ],
    "enrichment": {
      "source_url": "https://www.youtube.com/watch?v={video_id}",
      "crawl_time": "{iso8601_now}"
    }
  },
  "output": {
    "formats": ["json", "csv"],
    "default_encoding": "utf-8",
    "json_indent": 2,
    "csv_delimiter": ",",
    "include_headers": true
  },
  "error_handling": {
    "400": {
      "action": "skip_and_log",
      "description": "Bad Request - invalid parameters"
    },
    "403": {
      "action": "halt",
      "description": "Forbidden - quota exceeded or API key invalid"
    },
    "404": {
      "action": "skip_and_log",
      "description": "Not Found - resource does not exist"
    },
    "429": {
      "action": "exponential_backoff",
      "base_delay_seconds": 2,
      "max_delay_seconds": 60,
      "max_retries": 5,
      "description": "Rate limit exceeded - too many requests"
    },
    "500": {
      "action": "retry",
      "max_retries": 3,
      "description": "Internal Server Error"
    },
    "502": {
      "action": "retry",
      "max_retries": 3,
      "description": "Bad Gateway"
    },
    "503": {
      "action": "exponential_backoff",
      "base_delay_seconds": 5,
      "max_delay_seconds": 120,
      "max_retries": 5,
      "description": "Service Unavailable"
    }
  },
  "schedule": {
    "fetch_interval_hours": 4,
    "recommended_crontab": "0 */4 * * *",
    "timezone": "UTC"
  },
  "references": [
    "https://developers.google.com/youtube/v3/docs/videos/list",
    "https://developers.google.com/youtube/v3/docs/videoCategories/list",
    "https://developers.google.com/youtube/v3/getting-started#quota"
  ]
}