crawl_config.json
1.52 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
{
"skill_name": "youtube-data-expert",
"skill_type": "data-extraction",
"target_platform": "youtube",
"api_version": "v3",
"api_base_url": "https://www.googleapis.com/youtube/v3",
"authentication": {
"type": "api_key",
"env_var": "YOUTUBE_API_KEY",
"required": true
},
"quota": {
"daily_limit": 10000,
"track_usage": true,
"warning_threshold": 8000
},
"endpoints": {
"videos": {
"path": "/videos",
"cost": 1,
"batch_size": 50,
"supported_parts": ["snippet", "contentDetails", "statistics", "status"]
},
"channels": {
"path": "/channels",
"cost": 1,
"batch_size": 50,
"supported_parts": ["snippet", "statistics", "contentDetails", "brandingSettings"]
},
"search": {
"path": "/search",
"cost": 100,
"batch_size": 50,
"supported_types": ["video", "channel", "playlist"]
},
"playlistItems": {
"path": "/playlistItems",
"cost": 1,
"batch_size": 50
}
},
"data_quality": {
"required_fields": ["id", "snippet.title", "snippet.publishedAt"],
"optional_fields": ["snippet.description", "snippet.tags", "statistics"],
"validation_rules": {
"duration_format": "ISO_8601",
"date_format": "ISO_8601",
"number_parsing": "string_to_int"
}
},
"rate_limiting": {
"max_concurrent": 5,
"retry_strategy": "exponential_backoff",
"max_retries": 3,
"base_delay_ms": 1000
},
"output_formats": ["json", "csv"],
"crawled_at": "2026-04-03T22:30:00Z"
}