crawl_config.json
2.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
{
"skill_name": "satire_news_scraper",
"version": "1.0.0",
"description": "讽刺新闻网站抓取与短剧素材提取",
"sources": [
{
"name": "The Onion",
"url": "https://www.theonion.com",
"rss": "https://www.theonion.com/rss",
"type": "satire_news",
"style": "absurdist_realism",
"priority": 1
},
{
"name": "ClickHole",
"url": "https://www.clickhole.com",
"rss": "https://www.clickhole.com/rss",
"type": "clickbait_satire",
"style": "viral_mockery",
"priority": 2
},
{
"name": "Babylon Bee",
"url": "https://babylonbee.com",
"rss": "https://babylonbee.com/feed",
"type": "political_satire",
"style": "conservative_perspective",
"priority": 3
}
],
"crawl": {
"mode": "rss_primary",
"schedule": "0 9 * * *",
"batch_size": 20,
"rate_limit": {
"requests_per_minute": 10,
"delay_seconds": 6
}
},
"rss_parsing": {
"fields": [
"title",
"description",
"link",
"pubDate",
"category",
"content:encoded"
],
"content_cleaning": {
"remove_html": true,
"max_length": 2000,
"preserve_paragraphs": true
}
},
"absurd_detection": {
"keywords": {
"hyperbole": ["million", "billion", "infinity", "100%", "everyone", "nobody"],
"absurd_actions": ["declares", "announces", "elects", "appoints", "files"],
"impossible_subjects": ["dog", "cat", "baby", "local man", "area woman"]
},
"title_patterns": [
"^[A-Z][a-z]+ (Man|Woman|Dog|Cat)",
"Study (Finds|Shows|Reveals)",
"Breaking:",
"^(Area|Local) (Man|Woman)"
]
},
"output_schema": {
"headline": "string",
"absurd_premise": "string",
"satire_angle": "string",
"dramatic_twist": "string",
"adaptation_notes": {
"genre": "string",
"tone": "string",
"visual_elements": ["string"],
"dialogue_style": "string"
},
"source": {
"site": "string",
"url": "string",
"published_at": "datetime"
}
}
}