crawl_config.json 9.31 KB
{
  "skill_name": "satire-news-scraper",
  "version": "1.0.0",
  "crawl_time": "2026-04-13T02:40:00Z",
  "sources": [
    {
      "id": "the_onion",
      "name": "The Onion",
      "type": "web_scraper",
      "endpoint": "https://www.theonion.com/",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": "article",
        "fields": {
          "title": "h2 a",
          "url": "h2 a[href]",
          "time": "time[datetime]"
        },
        "article_selector": "article",
        "article_fields": {
          "headline": "h1",
          "byline": ".author-name",
          "published_at": "time[datetime]",
          "body": ".js_post-content",
          "tags": ".tags a"
        }
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "sleep_ms": 2500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 5000
      },
      "priority": 1,
      "notes": "Deadpan AP-style satire. No front-page disclaimer. Headlines often stand alone."
    },
    {
      "id": "clickhole",
      "name": "ClickHole",
      "type": "web_scraper",
      "endpoint": "https://clickhole.com/",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": "article",
        "fields": {
          "title": "h2 a",
          "url": "h2 a[href]",
          "category": ".cat-links a"
        },
        "article_selector": "article",
        "article_fields": {
          "headline": "h1.entry-title",
          "published_at": "time[datetime]",
          "body": ".entry-content",
          "tags": ".tags a"
        }
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "sleep_ms": 2500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 5000
      },
      "priority": 2,
      "notes": "Absurdist listicles and quizzes. Satire of clickbait culture."
    },
    {
      "id": "babylon_bee",
      "name": "The Babylon Bee",
      "type": "web_scraper",
      "endpoint": "https://babylonbee.com/news",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": ".article-preview",
        "fields": {
          "title": "h3 a",
          "url": "h3 a[href]",
          "summary": ".excerpt"
        },
        "article_selector": "article",
        "article_fields": {
          "headline": "h1",
          "byline": ".author",
          "published_at": "time[datetime]",
          "body": ".article-body",
          "tags": ".tag"
        }
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "sleep_ms": 2500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 5000
      },
      "priority": 3,
      "notes": "Conservative-leaning Christian satire. Short news-brief format common."
    },
    {
      "id": "reductress",
      "name": "Reductress",
      "type": "web_scraper",
      "endpoint": "https://reductress.com/",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": "article.post",
        "fields": {
          "title": ".entry-title a",
          "url": ".entry-title a[href]",
          "category": ".cat-links a"
        },
        "article_selector": "article.post",
        "article_fields": {
          "headline": "h1.entry-title",
          "byline": ".author vcard a",
          "published_at": "time[datetime]",
          "body": ".entry-content",
          "tags": ".tags a"
        }
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "sleep_ms": 2500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 5000
      },
      "priority": 4,
      "notes": "Feminist satire of women's lifestyle media. Uses listicles and headlines as primary comedy vehicles."
    },
    {
      "id": "the_hard_times",
      "name": "The Hard Times",
      "type": "web_scraper",
      "endpoint": "https://thehardtimes.net/",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": ".post",
        "fields": {
          "title": ".entry-title a",
          "url": ".entry-title a[href]",
          "category": ".cat-links a"
        },
        "article_selector": "article.post",
        "article_fields": {
          "headline": "h1.entry-title",
          "byline": ".author a",
          "published_at": "time[datetime]",
          "body": ".entry-content",
          "tags": ".tags a"
        }
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "sleep_ms": 2500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 5000
      },
      "priority": 5,
      "notes": "Punk/music subculture satire. Heavy use of insider jargon and scene tropes."
    },
    {
      "id": "the_shovel",
      "name": "The Shovel",
      "type": "web_scraper",
      "endpoint": "https://theshovel.com.au/",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": "article",
        "fields": {
          "title": "h2.entry-title a",
          "url": "h2.entry-title a[href]",
          "category": ".cat-links a"
        },
        "article_selector": "article",
        "article_fields": {
          "headline": "h1.entry-title",
          "byline": ".author a",
          "published_at": "time[datetime]",
          "body": ".entry-content",
          "tags": ".tags a"
        }
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "sleep_ms": 2500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 5000
      },
      "priority": 6,
      "notes": "Australian politics and sport satire. Also produces Shovel TV video skits."
    },
    {
      "id": "the_daily_mash",
      "name": "The Daily Mash",
      "type": "web_scraper",
      "endpoint": "https://thedailymash.co.uk/",
      "method": "GET",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; SatireBot/1.0; +https://example.com/bot-info)",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-GB,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "DNT": "1",
        "Connection": "keep-alive"
      },
      "parse_rules": {
        "list_selector": "article",
        "fields": {
          "title": "h2 a",
          "url": "h2 a[href]",
          "category": ".category a"
        },
        "article_selector": "article",
        "article_fields": {
          "headline": "h1",
          "byline": ".author",
          "published_at": "time[datetime]",
          "body": ".article-body",
          "tags": ".tags a"
        }
      },
      "rate_limit": {
        "requests_per_minute": 20,
        "sleep_ms": 3500,
        "respect_robots_txt": true,
        "max_retries": 3,
        "retry_backoff_ms": 6000
      },
      "priority": 7,
      "notes": "British tabloid satire. Mid-size outfit; keep rate limits conservative."
    }
  ],
  "data_schema": {
    "required_fields": [
      "article_id",
      "headline",
      "source",
      "source_url",
      "crawl_time"
    ],
    "optional_fields": [
      "published_at",
      "author",
      "body_text",
      "tags",
      "images",
      "satirical_signals",
      "absurdity_score",
      "adaptation"
    ]
  },
  "output_format": {
    "file": "seed_data.json",
    "encoder": "json",
    "append_timestamp": true
  }
}