{ "meta": { "name": "youtube-trending-crawler", "version": "1.0.0", "description": "YouTube Data API v3 trending video crawler configuration", "created_at": "2026-04-01T01:30:00Z", "created_by": "Hunter", "source_api": "YouTube Data API v3", "base_endpoint": "https://www.googleapis.com/youtube/v3/videos" }, "auth": { "type": "api_key", "env_var": "YOUTUBE_API_KEY", "required": true, "description": "Google Cloud API Key with YouTube Data API v3 enabled" }, "request": { "method": "GET", "headers": { "Accept": "application/json", "Accept-Encoding": "gzip", "User-Agent": "Mozilla/5.0 (compatible; YouTubeCrawler/1.0; +http://example.com/bot)" }, "timeout_seconds": 30, "max_retries": 3, "retry_backoff_base_seconds": 2, "request_delay_seconds": 2.0, "respect_robots_txt": true }, "parameters": { "default": { "part": "snippet,statistics,contentDetails", "chart": "mostPopular", "maxResults": 50 }, "optional": { "regionCode": "{region_code}", "videoCategoryId": "{category_id}", "pageToken": "{page_token}" } }, "regions": { "primary": [ { "code": "US", "name": "United States" }, { "code": "JP", "name": "Japan" }, { "code": "KR", "name": "South Korea" }, { "code": "GB", "name": "United Kingdom" }, { "code": "DE", "name": "Germany" }, { "code": "FR", "name": "France" }, { "code": "IN", "name": "India" }, { "code": "BR", "name": "Brazil" }, { "code": "RU", "name": "Russia" } ], "extended": [ { "code": "AU", "name": "Australia" }, { "code": "CA", "name": "Canada" }, { "code": "MX", "name": "Mexico" }, { "code": "AR", "name": "Argentina" }, { "code": "CL", "name": "Chile" }, { "code": "ES", "name": "Spain" }, { "code": "IT", "name": "Italy" }, { "code": "NL", "name": "Netherlands" }, { "code": "SE", "name": "Sweden" }, { "code": "TR", "name": "Turkey" }, { "code": "ID", "name": "Indonesia" }, { "code": "TH", "name": "Thailand" }, { "code": "VN", "name": "Vietnam" }, { "code": "PH", "name": "Philippines" }, { "code": "EG", "name": "Egypt" }, { "code": "ZA", "name": "South Africa" } ] }, "categories": { "target": [ { "id": "10", "name": "Music", "enabled": true }, { "id": "20", "name": "Gaming", "enabled": true }, { "id": "24", "name": "Entertainment", "enabled": true }, { "id": "28", "name": "Science & Technology", "enabled": true } ], "extended": [ { "id": "1", "name": "Film & Animation", "enabled": false }, { "id": "2", "name": "Autos & Vehicles", "enabled": false }, { "id": "15", "name": "Pets & Animals", "enabled": false }, { "id": "17", "name": "Sports", "enabled": false }, { "id": "19", "name": "Travel & Events", "enabled": false }, { "id": "22", "name": "People & Blogs", "enabled": false }, { "id": "23", "name": "Comedy", "enabled": false }, { "id": "25", "name": "News & Politics", "enabled": false }, { "id": "26", "name": "Howto & Style", "enabled": false }, { "id": "27", "name": "Education", "enabled": false } ] }, "quota": { "daily_limit": 10000, "cost_per_call": 1, "max_results_per_call": 50, "estimated_max_videos_per_day": 500000, "rate_limit": { "requests_per_second": 5, "requests_per_minute": 180, "note": "Google does not publish exact per-minute limits. Use conservative pacing." } }, "parsing_rules": { "video_id": "$.items[*].id", "title": "$.items[*].snippet.title", "description": "$.items[*].snippet.description", "channel_id": "$.items[*].snippet.channelId", "channel_title": "$.items[*].snippet.channelTitle", "published_at": "$.items[*].snippet.publishedAt", "thumbnail_url": "$.items[*].snippet.thumbnails.high.url", "tags": "$.items[*].snippet.tags", "category_id": "$.items[*].snippet.categoryId", "view_count": "$.items[*].statistics.viewCount", "like_count": "$.items[*].statistics.likeCount", "comment_count": "$.items[*].statistics.commentCount", "duration": "$.items[*].contentDetails.duration", "next_page_token": "$.nextPageToken", "total_results": "$.pageInfo.totalResults" }, "data_schema": { "required_fields": [ "video_id", "title", "channel_id", "channel_title", "published_at", "view_count", "category_id", "source_url", "crawl_time" ], "optional_fields": [ "description", "thumbnail_url", "tags", "like_count", "comment_count", "duration" ], "enrichment": { "source_url": "https://www.youtube.com/watch?v={video_id}", "crawl_time": "{iso8601_now}" } }, "output": { "formats": ["json", "csv"], "default_encoding": "utf-8", "json_indent": 2, "csv_delimiter": ",", "include_headers": true }, "error_handling": { "400": { "action": "skip_and_log", "description": "Bad Request - invalid parameters" }, "403": { "action": "halt", "description": "Forbidden - quota exceeded or API key invalid" }, "404": { "action": "skip_and_log", "description": "Not Found - resource does not exist" }, "429": { "action": "exponential_backoff", "base_delay_seconds": 2, "max_delay_seconds": 60, "max_retries": 5, "description": "Rate limit exceeded - too many requests" }, "500": { "action": "retry", "max_retries": 3, "description": "Internal Server Error" }, "502": { "action": "retry", "max_retries": 3, "description": "Bad Gateway" }, "503": { "action": "exponential_backoff", "base_delay_seconds": 5, "max_delay_seconds": 120, "max_retries": 5, "description": "Service Unavailable" } }, "schedule": { "fetch_interval_hours": 4, "recommended_crontab": "0 */4 * * *", "timezone": "UTC" }, "references": [ "https://developers.google.com/youtube/v3/docs/videos/list", "https://developers.google.com/youtube/v3/docs/videoCategories/list", "https://developers.google.com/youtube/v3/getting-started#quota" ] }