{ "skill_name": "satire-news-scraper", "version": "1.0.0", "crawl_time": "2026-04-13T02:40:00Z", "sources": [ { "id": "the_onion", "name": "The Onion", "type": "web_scraper", "endpoint": "https://www.theonion.com/", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": "article", "fields": { "title": "h2 a", "url": "h2 a[href]", "time": "time[datetime]" }, "article_selector": "article", "article_fields": { "headline": "h1", "byline": ".author-name", "published_at": "time[datetime]", "body": ".js_post-content", "tags": ".tags a" } }, "rate_limit": { "requests_per_minute": 30, "sleep_ms": 2500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 5000 }, "priority": 1, "notes": "Deadpan AP-style satire. No front-page disclaimer. Headlines often stand alone." }, { "id": "clickhole", "name": "ClickHole", "type": "web_scraper", "endpoint": "https://clickhole.com/", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": "article", "fields": { "title": "h2 a", "url": "h2 a[href]", "category": ".cat-links a" }, "article_selector": "article", "article_fields": { "headline": "h1.entry-title", "published_at": "time[datetime]", "body": ".entry-content", "tags": ".tags a" } }, "rate_limit": { "requests_per_minute": 30, "sleep_ms": 2500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 5000 }, "priority": 2, "notes": "Absurdist listicles and quizzes. Satire of clickbait culture." }, { "id": "babylon_bee", "name": "The Babylon Bee", "type": "web_scraper", "endpoint": "https://babylonbee.com/news", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": ".article-preview", "fields": { "title": "h3 a", "url": "h3 a[href]", "summary": ".excerpt" }, "article_selector": "article", "article_fields": { "headline": "h1", "byline": ".author", "published_at": "time[datetime]", "body": ".article-body", "tags": ".tag" } }, "rate_limit": { "requests_per_minute": 30, "sleep_ms": 2500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 5000 }, "priority": 3, "notes": "Conservative-leaning Christian satire. Short news-brief format common." }, { "id": "reductress", "name": "Reductress", "type": "web_scraper", "endpoint": "https://reductress.com/", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": "article.post", "fields": { "title": ".entry-title a", "url": ".entry-title a[href]", "category": ".cat-links a" }, "article_selector": "article.post", "article_fields": { "headline": "h1.entry-title", "byline": ".author vcard a", "published_at": "time[datetime]", "body": ".entry-content", "tags": ".tags a" } }, "rate_limit": { "requests_per_minute": 30, "sleep_ms": 2500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 5000 }, "priority": 4, "notes": "Feminist satire of women's lifestyle media. Uses listicles and headlines as primary comedy vehicles." }, { "id": "the_hard_times", "name": "The Hard Times", "type": "web_scraper", "endpoint": "https://thehardtimes.net/", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": ".post", "fields": { "title": ".entry-title a", "url": ".entry-title a[href]", "category": ".cat-links a" }, "article_selector": "article.post", "article_fields": { "headline": "h1.entry-title", "byline": ".author a", "published_at": "time[datetime]", "body": ".entry-content", "tags": ".tags a" } }, "rate_limit": { "requests_per_minute": 30, "sleep_ms": 2500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 5000 }, "priority": 5, "notes": "Punk/music subculture satire. Heavy use of insider jargon and scene tropes." }, { "id": "the_shovel", "name": "The Shovel", "type": "web_scraper", "endpoint": "https://theshovel.com.au/", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": "article", "fields": { "title": "h2.entry-title a", "url": "h2.entry-title a[href]", "category": ".cat-links a" }, "article_selector": "article", "article_fields": { "headline": "h1.entry-title", "byline": ".author a", "published_at": "time[datetime]", "body": ".entry-content", "tags": ".tags a" } }, "rate_limit": { "requests_per_minute": 30, "sleep_ms": 2500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 5000 }, "priority": 6, "notes": "Australian politics and sport satire. Also produces Shovel TV video skits." }, { "id": "the_daily_mash", "name": "The Daily Mash", "type": "web_scraper", "endpoint": "https://thedailymash.co.uk/", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "en-GB,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive" }, "parse_rules": { "list_selector": "article", "fields": { "title": "h2 a", "url": "h2 a[href]", "category": ".category a" }, "article_selector": "article", "article_fields": { "headline": "h1", "byline": ".author", "published_at": "time[datetime]", "body": ".article-body", "tags": ".tags a" } }, "rate_limit": { "requests_per_minute": 20, "sleep_ms": 3500, "respect_robots_txt": true, "max_retries": 3, "retry_backoff_ms": 6000 }, "priority": 7, "notes": "British tabloid satire. Mid-size outfit; keep rate limits conservative." } ], "data_schema": { "required_fields": [ "article_id", "headline", "source", "source_url", "crawl_time" ], "optional_fields": [ "published_at", "author", "body_text", "tags", "images", "satirical_signals", "absurdity_score", "adaptation" ] }, "output_format": { "file": "seed_data.json", "encoder": "json", "append_timestamp": true } }