{
  "version": "1.0.0",
  "agent": "reddit-scraper",
  "updated_at": "2026-04-13T11:30:00+08:00",
  "sources": [
    {
      "id": "reddit_json_hot",
      "name": "Reddit Public JSON API - 帖子列表",
      "platform": "reddit",
      "category": "subreddit_posts",
      "url": "https://www.reddit.com/r/{subreddit}/hot.json",
      "method": "GET",
      "priority": 1,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json",
        "Accept-Language": "en-US,en;q=0.9",
        "Connection": "keep-alive"
      },
      "params": {
        "limit": 25,
        "after": null
      },
      "parser": {
        "type": "json",
        "root_path": "$.data.children",
        "item_selector": "*",
        "fields": {
          "post_id": { "path": "data.id", "type": "string", "required": true },
          "title": { "path": "data.title", "type": "string", "required": true },
          "author": { "path": "data.author", "type": "string", "required": false },
          "subreddit": { "path": "data.subreddit", "type": "string", "required": true },
          "permalink": { "path": "data.permalink", "type": "string", "required": true },
          "score": { "path": "data.score", "type": "integer", "required": true },
          "num_comments": { "path": "data.num_comments", "type": "integer", "required": false, "default": 0 },
          "upvote_ratio": { "path": "data.upvote_ratio", "type": "float", "required": false },
          "created_utc": { "path": "data.created_utc", "type": "float", "required": false },
          "selftext": { "path": "data.selftext", "type": "string", "required": false, "default": "" },
          "post_type": { "path": "data.post_hint", "type": "string", "required": false },
          "is_nsfw": { "path": "data.over_18", "type": "boolean", "required": false, "default": false },
          "is_spoiler": { "path": "data.spoiler", "type": "boolean", "required": false, "default": false },
          "awards": { "path": "data.total_awards_received", "type": "integer", "required": false, "default": 0 },
          "flair": { "path": "data.link_flair_text", "type": "string", "required": false }
        },
        "filters": {
          "valid_title": "item.title != null && item.title.length > 0",
          "not_nsfw": "item.is_nsfw != true"
        }
      },
      "notes": "公开 JSON API，无需 OAuth。支持 sort=hot/new/top/rising/controversial。"
    },
    {
      "id": "reddit_json_comments",
      "name": "Reddit Public JSON API - 帖子详情+评论",
      "platform": "reddit",
      "category": "comments",
      "url": "https://www.reddit.com/r/{subreddit}/comments/{post_id}.json",
      "method": "GET",
      "priority": 2,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json",
        "Accept-Language": "en-US,en;q=0.9"
      },
      "params": {
        "limit": 100,
        "sort": "top"
      },
      "parser": {
        "type": "json",
        "root_path": "$[1].data.children",
        "item_selector": "*",
        "fields": {
          "comment_id": { "path": "data.id", "type": "string", "required": true },
          "author": { "path": "data.author", "type": "string", "required": false },
          "body": { "path": "data.body", "type": "string", "required": true },
          "score": { "path": "data.score", "type": "integer", "required": false, "default": 0 },
          "parent_id": { "path": "data.parent_id", "type": "string", "required": false },
          "created_utc": { "path": "data.created_utc", "type": "float", "required": false }
        }
      },
      "notes": "返回数组 [post, comments]。more 节点需递归展开。"
    },
    {
      "id": "reddit_json_search",
      "name": "Reddit Public JSON API - 搜索",
      "platform": "reddit",
      "category": "search",
      "url": "https://www.reddit.com/r/{subreddit}/search.json",
      "method": "GET",
      "priority": 3,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json"
      },
      "params": {
        "q": "",
        "restrict_sr": "1",
        "sort": "relevance",
        "t": "week",
        "limit": 25
      },
      "parser": {
        "type": "json",
        "root_path": "$.data.children",
        "item_selector": "*",
        "fields": {
          "post_id": { "path": "data.id", "type": "string", "required": true },
          "title": { "path": "data.title", "type": "string", "required": true },
          "score": { "path": "data.score", "type": "integer", "required": true },
          "subreddit": { "path": "data.subreddit", "type": "string", "required": true }
        }
      },
      "notes": "restricted_sr=1 限制在指定 subreddit 内搜索。"
    },
    {
      "id": "reddit_rss",
      "name": "Reddit RSS Feed - 帖子流",
      "platform": "reddit",
      "category": "rss",
      "url": "https://www.reddit.com/r/{subreddit}/.rss",
      "method": "GET",
      "priority": 4,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/rss+xml,application/xml;q=0.9,*/*;q=0.8"
      },
      "parser": {
        "type": "xml",
        "root_path": "//channel/item",
        "item_selector": "*",
        "fields": {
          "title": { "path": "title/text()", "type": "string", "required": true },
          "link": { "path": "link/text()", "type": "string", "required": true },
          "pubDate": { "path": "pubDate/text()", "type": "string", "required": false },
          "author": { "path": "author/text()", "type": "string", "required": false }
        }
      },
      "notes": "RSS 稳定性高但字段较少，适合长期监控的降级方案。"
    },
    {
      "id": "old_reddit_html",
      "name": "Old Reddit HTML - 无 JS 页面解析",
      "platform": "reddit",
      "category": "html_fallback",
      "url": "https://old.reddit.com/r/{subreddit}/",
      "method": "GET",
      "priority": 5,
      "enabled": true,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
      },
      "parser": {
        "type": "html",
        "item_selector": "div.thing",
        "fields": {
          "post_id": { "selector": "[data-fullname]", "type": "string", "required": true, "attr": "data-fullname" },
          "title": { "selector": "a.title", "type": "string", "required": true, "attr": "text" },
          "permalink": { "selector": "a.title", "type": "string", "required": false, "attr": "href" },
          "score": { "selector": "div.score.unvoted", "type": "string", "required": false, "attr": "text" },
          "subreddit": { "selector": "a.subreddit", "type": "string", "required": false, "attr": "text" }
        }
      },
      "notes": "当 JSON API 被封时的最终降级方案，页面轻量、无需 JS。"
    },
    {
      "id": "pushshift_pullpush",
      "name": "PullPush API (Pushshift 替代)",
      "platform": "reddit",
      "category": "historical",
      "url": "https://api.pullpush.io/reddit/search/submission/",
      "method": "GET",
      "priority": 6,
      "enabled": false,
      "headers": {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept": "application/json"
      },
      "params": {
        "subreddit": "AmITheAsshole",
        "size": 25,
        "sort": "desc",
        "sort_type": "created_utc"
      },
      "parser": {
        "type": "json",
        "root_path": "$.data",
        "item_selector": "*",
        "fields": {
          "post_id": { "path": "id", "type": "string", "required": true },
          "title": { "path": "title", "type": "string", "required": true },
          "author": { "path": "author", "type": "string", "required": false },
          "score": { "path": "score", "type": "integer", "required": false },
          "created_utc": { "path": "created_utc", "type": "float", "required": false }
        }
      },
      "notes": "Pushshift 的社区替代接口，历史数据可用，当前新数据收录不稳定，默认关闭。"
    }
  ],
  "rate_limit": {
    "strategy": "adaptive",
    "min_delay_seconds": 2,
    "max_delay_seconds": 5,
    "requests_per_minute": 20,
    "cooldown_minutes": 1,
    "retry_on_rate_limit": {
      "enabled": true,
      "max_retries": 3,
      "backoff_seconds": 15
    }
  },
  "user_agent_pool": [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 Mobile/15E148 Safari/604.1",
    "Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15"
  ],
  "target_subreddits": [
    { "name": "AmITheAsshole", "priority": 1, "min_score": 2000, "content_type": "moral_dilemma" },
    { "name": "relationship_advice", "priority": 2, "min_score": 1500, "content_type": "trust_issue" },
    { "name": "funny", "priority": 3, "min_score": 5000, "content_type": "observational_humor" },
    { "name": "tifu", "priority": 4, "min_score": 3000, "content_type": "social_conflict" }
  ],
  "validation": {
    "expected_content_type": "application/json",
    "fallback_content_types": ["text/html", "application/rss+xml"],
    "success_indicator": {
      "path": "$.kind",
      "value": "Listing"
    },
    "block_indicators": [
      "whoa there, pardner",
      "blocked due to a network policy"
    ]
  },
  "output": {
    "schema": "reddit_scraper_v1",
    "required_fields": [
      "post_id",
      "source.platform",
      "source.subreddit",
      "content.title",
      "engagement.score",
      "source.source_url",
      "source.crawl_time"
    ],
    "include_raw_data": true
  },
  "compliance": {
    "robots_txt": "https://www.reddit.com/robots.txt",
    "crawl_delay_seconds": 2,
    "exclude_private_data": true,
    "exclude_nsfw": true,
    "data_retention_days": 90
  }
}