宜博 / lemonboss

#!/usr/bin/env python3
"""
Reddit 数据抓取脚本
任务: T-mn5xqqfz
目标: 抓取 ≥400 条 Reddit 帖子数据
"""

import requests
import json
import time
import os
from datetime import datetime

# 配置
OUTPUT_DIR = "../../workspace/hunter/reddit-scraper-v2"
RAW_DATA_FILE = os.path.join(OUTPUT_DIR, "raw_reddit_data.json")
CRAWL_LOG_FILE = os.path.join(OUTPUT_DIR, "crawl_log.md")
STATS_FILE = os.path.join(OUTPUT_DIR, "data_stats.md")
TOP_POSTS_FILE = os.path.join(OUTPUT_DIR, "top_posts_sample.md")

# Reddit API 配置 - 增强反爬
USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/121.0",
]
REQUEST_DELAY_MIN = 3
REQUEST_DELAY_MAX = 6

# 使用 Reddit JSON API 会话
session = requests.Session()

# 抓取目标配置
SUBREDDITS = [
    {"name": "AmITheAsshole", "sort": "top", "time": "month", "target": 100},
    {"name": "relationship_advice", "sort": "top", "time": "month", "target": 100},
    {"name": "funny", "sort": "top", "time": "week", "target": 100},
    {"name": "jokes", "sort": "top", "time": "week", "target": 50},
    {"name": "StandUpComedy", "sort": "hot", "time": None, "target": 50},
]

REQUIRED_FIELDS = [
    "id", "title", "author", "selftext", "score", "ups", "num_comments",
    "upvote_ratio", "created_utc", "permalink", "link_flair_text"
]


def fetch_subreddit_posts(subreddit_config):
    """抓取单个 subreddit 的帖子"""
    import random

    name = subreddit_config["name"]
    sort = subreddit_config["sort"]
    time_param = subreddit_config.get("time")
    target = subreddit_config["target"]

    print(f"\n📡 开始抓取 r/{name} (目标: {target} 条)")

    all_posts = []
    after = None
    page = 1
    max_pages = 20

    while len(all_posts) < target and page <= max_pages:
        # 构建 URL (使用 old.reddit.com 可能限制更少)
        url = f"https://old.reddit.com/r/{name}/{sort}/.json?limit=100"
        if time_param:
            url += f"&t={time_param}"
        if after:
            url += f"&after={after}"

        # 随机选择 User-Agent
        headers = {"User-Agent": random.choice(USER_AGENTS)}

        try:
            print(f"  📄 第 {page} 页... ", end="", flush=True)
            response = session.get(url, headers=headers, timeout=30)

            if response.status_code == 403:
                print(f"❌ 403 Forbidden - 可能需要认证或代理")
                log_error(name, "HTTP 403 Forbidden", url)
                break
            elif response.status_code != 200:
                print(f"❌ HTTP {response.status_code}")
                log_error(name, f"HTTP {response.status_code}", url)
                break

            data = response.json()
            posts = data["data"]["children"]

            if not posts:
                print("⚠️ 无更多数据")
                break

            # 提取并验证字段
            for post in posts:
                post_data = post["data"]
                extracted = extract_fields(post_data)
                if extracted:
                    extracted["source_subreddit"] = name
                    extracted["crawled_at"] = datetime.now().isoformat()
                    all_posts.append(extracted)

            print(f"✅ 已获取 {len(all_posts)}/{target} 条")

            # 获取下一页的 after 参数
            after = data["data"].get("after")
            if not after:
                print("  ℹ️ 已到达最后一页")
                break

            page += 1
            # 随机延迟
            delay = random.uniform(REQUEST_DELAY_MIN, REQUEST_DELAY_MAX)
            time.sleep(delay)

        except Exception as e:
            print(f"❌ 错误: {e}")
            log_error(name, str(e), url)
            break

    print(f"  🎯 r/{name} 完成: {len(all_posts)} 条")
    return all_posts


def extract_fields(post_data):
    """提取需要的字段"""
    try:
        return {
            "id": post_data.get("id"),
            "title": post_data.get("title"),
            "author": post_data.get("author"),
            "selftext": post_data.get("selftext", ""),
            "score": post_data.get("score"),
            "ups": post_data.get("ups"),
            "num_comments": post_data.get("num_comments"),
            "upvote_ratio": post_data.get("upvote_ratio"),
            "created_utc": post_data.get("created_utc"),
            "permalink": post_data.get("permalink"),
            "link_flair_text": post_data.get("link_flair_text"),
            "url": post_data.get("url"),
            "is_self": post_data.get("is_self"),
            "over_18": post_data.get("over_18"),
        }
    except Exception as e:
        print(f"  ⚠️ 字段提取失败: {e}")
        return None


def log_error(subreddit, error, url):
    """记录错误日志"""
    with open(CRAWL_LOG_FILE, "a", encoding="utf-8") as f:
        f.write(f"\n## {datetime.now().isoformat()}\n")
        f.write(f"- Subreddit: r/{subreddit}\n")
        f.write(f"- Error: {error}\n")
        f.write(f"- URL: {url}\n")


def generate_stats(all_posts):
    """生成数据统计报告"""
    stats = {}

    for post in all_posts:
        sub = post["source_subreddit"]
        if sub not in stats:
            stats[sub] = {
                "count": 0,
                "total_score": 0,
                "total_comments": 0,
                "avg_upvote_ratio": 0,
                "upvote_ratios": []
            }
        stats[sub]["count"] += 1
        stats[sub]["total_score"] += post.get("score", 0) or 0
        stats[sub]["total_comments"] += post.get("num_comments", 0) or 0
        if post.get("upvote_ratio"):
            stats[sub]["upvote_ratios"].append(post["upvote_ratio"])

    # 计算平均值
    for sub in stats:
        count = stats[sub]["count"]
        stats[sub]["avg_score"] = stats[sub]["total_score"] / count if count > 0 else 0
        stats[sub]["avg_comments"] = stats[sub]["total_comments"] / count if count > 0 else 0
        ratios = stats[sub]["upvote_ratios"]
        stats[sub]["avg_upvote_ratio"] = sum(ratios) / len(ratios) if ratios else 0

    return stats


def write_stats_report(stats, total_posts):
    """写入统计报告"""
    with open(STATS_FILE, "w", encoding="utf-8") as f:
        f.write("# Reddit 数据抓取统计报告\n\n")
        f.write(f"生成时间: {datetime.now().isoformat()}\n\n")
        f.write(f"**总抓取量: {total_posts} 条**\n\n")

        f.write("## 各版块统计\n\n")
        f.write("| 版块 | 数量 | 平均点赞 | 平均评论 | 平均赞同率 |\n")
        f.write("|------|------|----------|----------|------------|\n")

        for sub, data in sorted(stats.items(), key=lambda x: -x[1]["count"]):
            f.write(f"| r/{sub} | {data['count']} | {data['avg_score']:.0f} | "
                   f"{data['avg_comments']:.0f} | {data['avg_upvote_ratio']:.2f} |\n")

        # 时间分布
        f.write("\n## 数据质量检查\n\n")
        f.write(f"- 总帖子数: {total_posts}\n")
        f.write(f"- 验收标准: ≥400 条\n")
        f.write(f"- 状态: {'✅ 通过' if total_posts >= 400 else '❌ 未达标'}\n")


def write_top_posts_sample(all_posts):
    """写入各版块前10高赞帖子摘要"""
    # 按版块分组并排序
    by_subreddit = {}
    for post in all_posts:
        sub = post["source_subreddit"]
        if sub not in by_subreddit:
            by_subreddit[sub] = []
        by_subreddit[sub].append(post)

    with open(TOP_POSTS_FILE, "w", encoding="utf-8") as f:
        f.write("# 各版块前10高赞帖子\n\n")

        for sub, posts in sorted(by_subreddit.items()):
            # 按 score 排序
            top_posts = sorted(posts, key=lambda x: -(x.get("score") or 0))[:10]

            f.write(f"## r/{sub}\n\n")
            for i, post in enumerate(top_posts, 1):
                f.write(f"### {i}. {post['title'][:100]}{'...' if len(post['title']) > 100 else ''}\n\n")
                f.write(f"- **ID**: {post['id']}\n")
                f.write(f"- **作者**: u/{post['author']}\n")
                f.write(f"- **点赞**: {post['score']}\n")
                f.write(f"- **评论**: {post['num_comments']}\n")
                f.write(f"- **赞同率**: {post['upvote_ratio']}\n")
                f.write(f"- **链接**: https://reddit.com{post['permalink']}\n")
                if post.get('link_flair_text'):
                    f.write(f"- **标签**: {post['link_flair_text']}\n")

                # 内容摘要（前200字符）
                content = post.get('selftext', '')
                if content:
                    summary = content[:200].replace('\n', ' ')
                    f.write(f"- **摘要**: {summary}{'...' if len(content) > 200 else ''}\n")

                f.write("\n")


def main():
    """主函数"""
    print("=" * 60)
    print("🕵️ Hunter Agent - Reddit 数据抓取任务")
    print("任务ID: T-mn5xqqfz")
    print("=" * 60)

    start_time = datetime.now()

    # 初始化日志
    with open(CRAWL_LOG_FILE, "w", encoding="utf-8") as f:
        f.write("# Reddit 数据抓取日志\n\n")
        f.write(f"开始时间: {start_time.isoformat()}\n\n")

    # 抓取所有版块
    all_posts = []
    for config in SUBREDDITS:
        posts = fetch_subreddit_posts(config)
        all_posts.extend(posts)
        print(f"  累计: {len(all_posts)} 条")

    # 保存原始数据
    print("\n💾 保存原始数据...")
    with open(RAW_DATA_FILE, "w", encoding="utf-8") as f:
        json.dump(all_posts, f, ensure_ascii=False, indent=2)
    print(f"  ✅ 已保存: {RAW_DATA_FILE}")

    # 生成统计报告
    print("\n📊 生成统计报告...")
    stats = generate_stats(all_posts)
    write_stats_report(stats, len(all_posts))
    print(f"  ✅ 统计报告: {STATS_FILE}")

    # 生成 Top 帖子摘要
    print("\n📝 生成 Top 帖子摘要...")
    write_top_posts_sample(all_posts)
    print(f"  ✅ Top帖子: {TOP_POSTS_FILE}")

    # 完成日志
    end_time = datetime.now()
    duration = (end_time - start_time).total_seconds()

    with open(CRAWL_LOG_FILE, "a", encoding="utf-8") as f:
        f.write(f"\n## 完成\n")
        f.write(f"- 结束时间: {end_time.isoformat()}\n")
        f.write(f"- 耗时: {duration:.1f} 秒\n")
        f.write(f"- 总抓取: {len(all_posts)} 条\n")
        f.write(f"- 状态: {'成功' if len(all_posts) >= 400 else '未达标'}\n")

    print("\n" + "=" * 60)
    print(f"✅ 任务完成!")
    print(f"📦 总抓取: {len(all_posts)} 条")
    print(f"⏱️  耗时: {duration:.1f} 秒")
    print(f"📁 输出目录: {OUTPUT_DIR}")
    print("=" * 60)

    return len(all_posts)


if __name__ == "__main__":
    count = main()
    exit(0 if count >= 400 else 1)