宜博 / lemonboss

{
  "name": "reddit-scraper-training",
  "version": "1.0.0",
  "description": "Reddit 热门板块抓取配置 - 用于训练短剧内容识别模型",
  "created": "2026-03-25",
  "created_by": "Hunter Agent",

  "target_subreddits": [
    {
      "name": "AmITheAsshole",
      "alias": "AITA",
      "priority": 1,
      "content_type": "moral_conflict",
      "sort_methods": ["top", "hot", "controversial"],
      "time_ranges": ["month", "year", "all"],
      "min_upvotes": 1000,
      "min_comments": 200,
      "target_post_count": 100
    },
    {
      "name": "relationship_advice",
      "alias": "RA",
      "priority": 2,
      "content_type": "relationship_drama",
      "sort_methods": ["top", "hot"],
      "time_ranges": ["month", "year"],
      "min_upvotes": 800,
      "min_comments": 150,
      "target_post_count": 100
    },
    {
      "name": "tifu",
      "alias": "TIFU",
      "priority": 1,
      "content_type": "embarrassing_comedy",
      "sort_methods": ["top", "hot"],
      "time_ranges": ["month", "year", "all"],
      "min_upvotes": 2000,
      "min_comments": 100,
      "target_post_count": 100
    },
    {
      "name": "funny",
      "alias": null,
      "priority": 3,
      "content_type": "general_humor",
      "sort_methods": ["top"],
      "time_ranges": ["month", "all"],
      "min_upvotes": 5000,
      "min_comments": 50,
      "target_post_count": 50
    }
  ],

  "endpoints": {
    "json_api": {
      "base_url": "https://www.reddit.com/r/{subreddit}/{sort}.json",
      "method": "GET",
      "auth_required": false,
      "rate_limit": {
        "requests_per_minute": 30,
        "requests_per_hour": 500,
        "note": "无认证情况下保守估计，建议添加随机延迟"
      },
      "query_params": {
        "limit": {
          "type": "integer",
          "default": 25,
          "max": 100,
          "description": "每页返回帖子数量"
        },
        "t": {
          "type": "string",
          "options": ["hour", "day", "week", "month", "year", "all"],
          "default": "all",
          "description": "时间范围（仅用于 top/controversial）"
        },
        "after": {
          "type": "string",
          "description": "分页参数，上一页最后一个帖子的 fullname"
        },
        "before": {
          "type": "string",
          "description": "分页参数，获取此 ID 之前的内容"
        }
      }
    },
    "old_reddit": {
      "base_url": "https://old.reddit.com/r/{subreddit}/",
      "method": "GET",
      "auth_required": false,
      "rate_limit": {
        "requests_per_minute": 20,
        "note": "HTML 抓取，频率需更低"
      },
      "query_params": {
        "sort": {
          "type": "string",
          "options": ["top", "hot", "new"],
          "default": "top"
        },
        "t": {
          "type": "string",
          "options": ["hour", "day", "week", "month", "year", "all"]
        },
        "limit": {
          "type": "integer",
          "max": 500,
          "description": "Old Reddit 支持最高 500"
        },
        "count": {
          "type": "integer",
          "description": "分页计数"
        },
        "after": {
          "type": "string",
          "description": "分页 ID"
        }
      }
    },
    "official_api": {
      "base_url": "https://oauth.reddit.com/r/{subreddit}/{sort}",
      "method": "GET",
      "auth_required": true,
      "auth_type": "OAuth 2.0",
      "rate_limit": {
        "requests_per_minute": 100,
        "note": "免费 tier，商业使用需付费"
      },
      "setup": {
        "register_url": "https://www.reddit.com/prefs/apps",
        "pricing": "$0.24 per 1,000 calls (commercial)"
      }
    }
  },

  "field_mapping": {
    "post": {
      "id": "data.id",
      "title": "data.title",
      "author": "data.author",
      "selftext": "data.selftext",
      "url": "data.url",
      "permalink": "data.permalink",
      "subreddit": "data.subreddit",
      "created_utc": "data.created_utc",
      "score": "data.score",
      "ups": "data.ups",
      "downs": "data.downs",
      "upvote_ratio": "data.upvote_ratio",
      "num_comments": "data.num_comments",
      "num_crossposts": "data.num_crossposts",
      "is_self": "data.is_self",
      "is_video": "data.is_video",
      "over_18": "data.over_18",
      "spoiler": "data.spoiler",
      "stickied": "data.stickied",
      "edited": "data.edited",
      "link_flair_text": "data.link_flair_text",
      "link_flair_css_class": "data.link_flair_css_class"
    },
    "comment": {
      "id": "data.id",
      "author": "data.author",
      "body": "data.body",
      "created_utc": "data.created_utc",
      "score": "data.score",
      "parent_id": "data.parent_id",
      "depth": "depth",
      "is_submitter": "data.is_submitter",
      "stickied": "data.stickied",
      "edited": "data.edited"
    }
  },

  "content_filters": {
    "high_drama_indicators": {
      "aita": {
        "flair_priority": ["YTA", "ESH", "NTA", "INFO", "NAH"],
        "keywords": ["divorce", "cheating", "wedding", "family", "money", "inheritance", "baby", "pregnant"],
        "min_controversy_score": 0.3
      },
      "relationship_advice": {
        "keywords": ["break up", "cheating", "lying", "trust", "ex", "inappropriate", "boundaries"],
        "min_length": 500
      },
      "tifu": {
        "keywords": ["accidentally", "mistake", "embarrassing", "awkward", "cringe"],
        "min_upvote_ratio": 0.85
      }
    },
    "short_drama_score": {
      "weights": {
        "conflict_clarity": 0.20,
        "moral_ambiguity": 0.20,
        "emotional_tension": 0.20,
        "twist_potential": 0.20,
        "visualizability": 0.20
      },
      "thresholds": {
        "excellent": 8.0,
        "good": 6.5,
        "average": 5.0,
        "poor": 3.0
      }
    },
    "exclusion_rules": [
      {
        "field": "stickied",
        "value": true,
        "reason": "排除置顶公告帖"
      },
      {
        "field": "is_self",
        "value": false,
        "reason": "仅保留文本帖（非链接帖）",
        "subreddits": ["AmITheAsshole", "relationship_advice", "tifu"]
      },
      {
        "field": "selftext",
        "operator": "length_lt",
        "value": 100,
        "reason": "排除过短内容"
      },
      {
        "field": "selftext",
        "operator": "contains",
        "value": ["[removed]", "[deleted]"],
        "reason": "排除被删除内容"
      },
      {
        "field": "over_18",
        "value": true,
        "action": "flag",
        "reason": "标记 NSFW 内容"
      }
    ]
  },

  "anti_detection": {
    "headers": {
      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
      "Accept": "application/json, text/plain, */*",
      "Accept-Language": "en-US,en;q=0.9",
      "Accept-Encoding": "gzip, deflate, br",
      "DNT": "1",
      "Connection": "keep-alive"
    },
    "user_agent_rotation": [
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
      "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0"
    ],
    "delays": {
      "between_requests_min": 2,
      "between_requests_max": 5,
      "between_pages": 3,
      "retry_delay": 60
    },
    "proxy_recommendations": {
      "residential": ["Bright Data", "Oxylabs", "Smartproxy"],
      "note": "住宅代理优于数据中心代理"
    }
  },

  "output_schema": {
    "post": {
      "id": "string",
      "title": "string",
      "content": "string",
      "author": "string",
      "subreddit": "string",
      "created_at": "timestamp",
      "score": "integer",
      "upvote_ratio": "float (0-1)",
      "num_comments": "integer",
      "permalink": "string",
      "flair": "string",
      "is_nsfw": "boolean",
      "drama_score": "float (0-10)",
      "adaptation_potential": "string (high/medium/low)",
      "conflict_type": "string",
      "extracted_at": "timestamp"
    },
    "comment_summary": {
      "top_comments": "array of top 5 comments",
      "consensus": "string",
      "controversy_level": "string"
    }
  },

  "error_handling": {
    "429_too_many_requests": {
      "action": "exponential_backoff",
      "max_retries": 5,
      "base_delay": 60
    },
    "403_forbidden": {
      "action": "rotate_user_agent",
      "fallback": "use_proxy"
    },
    "404_not_found": {
      "action": "skip_and_log"
    },
    "503_service_unavailable": {
      "action": "retry_with_delay",
      "delay": 300
    }
  },

  "schedule": {
    "frequency": "daily",
    "best_time": "06:00 UTC",
    "reason": "Reddit 流量较低时段，反爬限制较松"
  }
}