test_youtube_crawler.py
14.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
"""
YouTube Trending Crawler Tests
测试用例验证代码可运行
"""
import os
import sys
import json
import pytest
from unittest.mock import Mock, patch, MagicMock
# 添加父目录到路径
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from youtube_crawler import YouTubeTrendingCrawler
class TestYouTubeTrendingCrawler:
"""YouTubeTrendingCrawler 测试类"""
def test_init_with_api_key(self):
"""测试使用参数传入 API Key"""
crawler = YouTubeTrendingCrawler(api_key="test_key")
assert crawler.api_key == "test_key"
assert crawler.quota_used == 0
def test_init_with_env_var(self):
"""测试从环境变量读取 API Key"""
os.environ["YOUTUBE_API_KEY"] = "env_test_key"
crawler = YouTubeTrendingCrawler()
assert crawler.api_key == "env_test_key"
def test_init_without_api_key(self):
"""测试未提供 API Key 时抛出异常"""
# 清除环境变量
if "YOUTUBE_API_KEY" in os.environ:
del os.environ["YOUTUBE_API_KEY"]
with pytest.raises(ValueError, match="API Key 未提供"):
YouTubeTrendingCrawler()
def test_video_categories(self):
"""测试视频分类映射"""
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[10] == "Music"
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[20] == "Gaming"
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[24] == "Entertainment"
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[28] == "Science & Technology"
def test_region_codes(self):
"""测试地区代码列表"""
assert "US" in YouTubeTrendingCrawler.REGION_CODES
assert "JP" in YouTubeTrendingCrawler.REGION_CODES
assert "KR" in YouTubeTrendingCrawler.REGION_CODES
class TestExtractVideoInfo:
"""测试 extract_video_info 方法"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
def test_extract_complete_data(self):
"""测试完整数据解析"""
api_item = {
"id": "test123",
"snippet": {
"title": "Test Video",
"description": "Test Description",
"channelTitle": "Test Channel",
"channelId": "UCtest",
"publishedAt": "2024-01-15T10:00:00Z",
"categoryId": "10",
"thumbnails": {
"default": {"url": "https://default.jpg"},
"medium": {"url": "https://medium.jpg"},
"high": {"url": "https://high.jpg"}
},
"tags": ["tag1", "tag2"],
"liveBroadcastContent": "none"
},
"statistics": {
"viewCount": "1000000",
"likeCount": "50000",
"commentCount": "2000"
},
"contentDetails": {
"duration": "PT3M45S"
}
}
result = self.crawler.extract_video_info(api_item)
assert result["videoId"] == "test123"
assert result["title"] == "Test Video"
assert result["url"] == "https://youtube.com/watch?v=test123"
assert result["channelTitle"] == "Test Channel"
assert result["viewCount"] == 1000000
assert result["likeCount"] == 50000
assert result["commentCount"] == 2000
assert result["categoryName"] == "Music"
assert result["duration"] == "PT3M45S"
def test_extract_minimal_data(self):
"""测试最小数据解析"""
api_item = {
"id": "min123",
"snippet": {
"title": "Minimal",
"channelTitle": "Channel"
},
"statistics": {}
}
result = self.crawler.extract_video_info(api_item)
assert result["videoId"] == "min123"
assert result["viewCount"] == 0
assert result["likeCount"] == 0
assert result["thumbnails"]["default"] == ""
def test_extract_invalid_data(self):
"""测试无效数据处理"""
result = self.crawler.extract_video_info({})
assert result is None
class TestQuotaStatus:
"""测试配额管理功能"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
def test_initial_quota(self):
"""测试初始配额状态"""
status = self.crawler.get_quota_status()
assert status["quota_used"] == 0
assert status["quota_limit"] == 10000
assert status["quota_remaining"] == 10000
assert status["usage_percentage"] == 0.0
def test_quota_after_usage(self):
"""测试使用后的配额状态"""
self.crawler.quota_used = 500
status = self.crawler.get_quota_status()
assert status["quota_used"] == 500
assert status["quota_remaining"] == 9500
assert status["usage_percentage"] == 5.0
class TestSaveFunctions:
"""测试数据保存功能"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
self.test_dir = os.path.dirname(os.path.abspath(__file__))
def test_save_to_json(self, tmp_path):
"""测试 JSON 保存"""
test_data = {"region": "US", "videos": [{"id": "1", "title": "Test"}]}
filepath = str(tmp_path / "test.json")
result = self.crawler.save_to_json(test_data, filepath=filepath)
assert os.path.exists(result)
with open(result, "r", encoding="utf-8") as f:
loaded = json.load(f)
assert loaded["region"] == "US"
def test_save_to_csv(self, tmp_path):
"""测试 CSV 保存"""
test_videos = [
{
"videoId": "vid1",
"title": "Video 1",
"channelTitle": "Channel 1",
"viewCount": 1000,
"thumbnails": {"high": "https://thumb.jpg"}
}
]
filepath = str(tmp_path / "test.csv")
result = self.crawler.save_to_csv(test_videos, filepath=filepath)
assert os.path.exists(result)
with open(result, "r", encoding="utf-8") as f:
content = f.read()
assert "videoId" in content
assert "vid1" in content
def test_save_empty_videos(self, tmp_path):
"""测试保存空视频列表"""
filepath = str(tmp_path / "empty.csv")
result = self.crawler.save_to_csv([], filepath=filepath)
assert os.path.exists(result)
class TestParameterValidation:
"""测试参数验证"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
@patch.object(YouTubeTrendingCrawler, '_make_request')
def test_max_results_validation(self, mock_request):
"""测试 max_results 参数验证"""
mock_request.return_value = {"items": []}
# 有效值
self.crawler.get_trending(max_results=1)
self.crawler.get_trending(max_results=50)
# 无效值
with pytest.raises(ValueError, match="max_results 必须在 1-50 之间"):
self.crawler.get_trending(max_results=0)
with pytest.raises(ValueError, match="max_results 必须在 1-50 之间"):
self.crawler.get_trending(max_results=51)
class TestErrorHandling:
"""测试错误处理"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
@patch('requests.Session.get')
def test_403_quota_exceeded(self, mock_get):
"""测试配额耗尽错误"""
mock_response = Mock()
mock_response.status_code = 403
mock_response.json.return_value = {
"error": {"errors": [{"reason": "quotaExceeded"}]}
}
mock_get.return_value = mock_response
with pytest.raises(ValueError, match="API 配额已耗尽"):
self.crawler._make_request("videos", {"part": "snippet"})
@patch('requests.Session.get')
def test_403_invalid_key(self, mock_get):
"""测试无效 API Key 错误"""
mock_response = Mock()
mock_response.status_code = 403
mock_response.json.return_value = {
"error": {"errors": [{"reason": "keyInvalid"}]}
}
mock_get.return_value = mock_response
with pytest.raises(ValueError, match="API Key 无效"):
self.crawler._make_request("videos", {"part": "snippet"})
@patch('requests.Session.get')
def test_timeout_error(self, mock_get):
"""测试超时错误"""
import requests
mock_get.side_effect = requests.exceptions.Timeout()
with pytest.raises(ValueError, match="请求超时"):
self.crawler._make_request("videos", {"part": "snippet"})
class TestMockAPI:
"""使用 Mock 测试 API 调用"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
@patch.object(YouTubeTrendingCrawler, '_make_request')
def test_get_trending_success(self, mock_request):
"""测试获取热门视频成功"""
mock_request.return_value = {
"items": [
{
"id": "video1",
"snippet": {
"title": "Video 1",
"channelTitle": "Channel 1",
"publishedAt": "2024-01-01T00:00:00Z",
"categoryId": "10",
"thumbnails": {"high": {"url": "https://thumb.jpg"}}
},
"statistics": {"viewCount": "1000"},
"contentDetails": {"duration": "PT1M"}
}
],
"nextPageToken": "token123"
}
result = self.crawler.get_trending(region_code="US", max_results=10)
assert result["region_code"] == "US"
assert result["total_results"] == 1
assert result["next_page_token"] == "token123"
assert len(result["videos"]) == 1
assert result["videos"][0]["title"] == "Video 1"
@patch.object(YouTubeTrendingCrawler, '_make_request')
def test_get_trending_with_category(self, mock_request):
"""测试带分类的获取"""
mock_request.return_value = {"items": []}
self.crawler.get_trending(region_code="JP", category_id=10, max_results=5)
# 验证请求参数
call_args = mock_request.call_args
assert call_args[0][0] == "videos"
params = call_args[0][1]
assert params["regionCode"] == "JP"
assert params["videoCategoryId"] == 10
assert params["maxResults"] == 5
class TestMultiRegion:
"""测试多地区批量获取"""
def setup_method(self):
self.crawler = YouTubeTrendingCrawler(api_key="test_key")
@patch.object(YouTubeTrendingCrawler, 'get_trending')
def test_multi_region_success(self, mock_get_trending):
"""测试多地区获取成功"""
mock_get_trending.return_value = {
"total_results": 5,
"videos": [{"id": "v1"}]
}
results = self.crawler.get_multi_region_trending(
regions=["US", "JP"],
max_results=5
)
assert "US" in results
assert "JP" in results
assert results["US"]["total_results"] == 5
@patch.object(YouTubeTrendingCrawler, 'get_trending')
def test_multi_region_partial_failure(self, mock_get_trending):
"""测试部分失败处理"""
def side_effect(*args, **kwargs):
if kwargs.get("region_code") == "US":
return {"total_results": 5, "videos": []}
raise ValueError("API Error")
mock_get_trending.side_effect = side_effect
results = self.crawler.get_multi_region_trending(
regions=["US", "XX"],
max_results=5
)
assert results["US"]["total_results"] == 5
assert "error" in results["XX"]
def run_basic_tests():
"""运行基础测试 (无需 pytest)"""
print("=" * 60)
print("YouTube Trending Crawler - 基础测试")
print("=" * 60)
# 测试 1: 初始化
print("\n1. 测试初始化...")
try:
os.environ["YOUTUBE_API_KEY"] = "test_key"
crawler = YouTubeTrendingCrawler()
print(" ✅ 初始化成功")
except Exception as e:
print(f" ❌ 初始化失败: {e}")
# 测试 2: 分类映射
print("\n2. 测试分类映射...")
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[10] == "Music"
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[20] == "Gaming"
assert YouTubeTrendingCrawler.VIDEO_CATEGORIES[28] == "Science & Technology"
print(" ✅ 分类映射正确")
# 测试 3: 数据提取
print("\n3. 测试数据提取...")
api_item = {
"id": "test123",
"snippet": {
"title": "Test Video",
"channelTitle": "Test Channel",
"publishedAt": "2024-01-15T10:00:00Z",
"categoryId": "10",
"thumbnails": {"high": {"url": "https://thumb.jpg"}}
},
"statistics": {"viewCount": "1000000"},
"contentDetails": {"duration": "PT3M45S"}
}
result = crawler.extract_video_info(api_item)
assert result["videoId"] == "test123"
assert result["title"] == "Test Video"
assert result["viewCount"] == 1000000
print(" ✅ 数据提取正确")
# 测试 4: 配额状态
print("\n4. 测试配额状态...")
status = crawler.get_quota_status()
assert status["quota_limit"] == 10000
print(f" ✅ 配额限制: {status['quota_limit']}")
# 测试 5: 参数验证
print("\n5. 测试参数验证...")
try:
crawler.get_trending(max_results=0)
print(" ❌ 应该抛出 max_results 无效的错误")
except ValueError as e:
if "max_results" in str(e):
print(" ✅ 参数验证正确")
else:
print(f" ❌ 错误信息不匹配: {e}")
print("\n" + "=" * 60)
print("✅ 基础测试通过!")
print("=" * 60)
if __name__ == "__main__":
# 运行基础测试
run_basic_tests()
# 如果安装了 pytest,运行完整测试
try:
import pytest
print("\n运行 pytest 完整测试...")
pytest.main([__file__, "-v"])
except ImportError:
print("\n未安装 pytest,跳过完整测试")
print("安装命令: pip install pytest")