宜博 / lemonboss

const https = require('https');
const fs = require('fs');
const path = require('path');

const OUTPUT_DIR = '/Users/yibo/Documents/LemonAI/lemonboss/workspace/hunter/reddit-scraper-v2';
const RATE_LIMIT_MS = 2000; // 2秒间隔

// 抓取配置
const SUBREDDITS = [
  { name: 'AmITheAsshole', sort: 'top', time: 'month', target: 100 },
  { name: 'relationship_advice', sort: 'top', time: 'month', target: 100 },
  { name: 'funny', sort: 'top', time: 'week', target: 100 },
  { name: 'jokes', sort: 'top', time: 'week', target: 50 },
  { name: 'StandUpComedy', sort: 'hot', time: null, target: 50 }
];

const crawlLog = [];
const allPosts = [];
const stats = {};

function log(message) {
  const timestamp = new Date().toISOString();
  const logEntry = `[${timestamp}] ${message}`;
  console.log(logEntry);
  crawlLog.push(logEntry);
}

function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms));
}

function fetchJSON(url) {
  return new Promise((resolve, reject) => {
    const options = {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'application/json, text/plain, */*',
        'Accept-Language': 'en-US,en;q=0.9',
        'Accept-Encoding': 'gzip, deflate, br',
        'DNT': '1',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'none',
        'Sec-Fetch-User': '?1',
        'Cache-Control': 'max-age=0'
      },
      timeout: 30000
    };

    const req = https.get(url, options, (res) => {
      // 处理重定向
      if (res.statusCode === 301 || res.statusCode === 302) {
        const redirectUrl = res.headers.location;
        log(`  Redirecting to: ${redirectUrl}`);
        fetchJSON(redirectUrl).then(resolve).catch(reject);
        return;
      }

      if (res.statusCode !== 200) {
        reject(new Error(`HTTP ${res.statusCode}`));
        return;
      }

      let data = '';
      res.on('data', chunk => data += chunk);
      res.on('end', () => {
        try {
          const json = JSON.parse(data);
          resolve(json);
        } catch (e) {
          reject(new Error(`JSON parse error: ${e.message}, data: ${data.substring(0, 200)}`));
        }
      });
    });

    req.on('error', reject);
    req.on('timeout', () => {
      req.destroy();
      reject(new Error('Request timeout'));
    });
  });
}

async function fetchSubreddit(subreddit, sort, time, target) {
  log(`Starting: r/${subreddit} (${sort}${time ? '/' + time : ''}) - target: ${target}`);

  const posts = [];
  let after = null;
  let attempts = 0;
  const maxAttempts = 10;

  while (posts.length < target && attempts < maxAttempts) {
    attempts++;

    let url = `https://www.reddit.com/r/${subreddit}/${sort}.json?limit=100`;
    if (time) url += `&t=${time}`;
    if (after) url += `&after=${after}`;

    try {
      log(`  Fetching: ${url.replace(/after=[^&]+/, 'after=...')}`);
      const data = await fetchJSON(url);

      if (!data.data || !data.data.children) {
        log(`  Warning: Invalid response structure`);
        break;
      }

      const children = data.data.children;
      if (children.length === 0) {
        log(`  No more posts available`);
        break;
      }

      for (const child of children) {
        const post = child.data;
        posts.push({
          id: post.id,
          title: post.title,
          author: post.author,
          selftext: post.selftext || '',
          score: post.score,
          upvotes: post.ups,
          num_comments: post.num_comments,
          upvote_ratio: post.upvote_ratio,
          created_utc: post.created_utc,
          permalink: post.permalink,
          link_flair_text: post.link_flair_text || null,
          subreddit: post.subreddit,
          url: post.url
        });

        if (posts.length >= target) break;
      }

      after = data.data.after;
      log(`  Progress: ${posts.length}/${target} posts (after: ${after ? after.slice(0, 20) + '...' : 'null'})`);

      if (!after) {
        log(`  No more pages available`);
        break;
      }

      // 速率限制
      await sleep(RATE_LIMIT_MS);

    } catch (error) {
      log(`  Error: ${error.message}`);
      await sleep(RATE_LIMIT_MS * 2); // 出错时等待更久
    }
  }

  log(`Completed: r/${subreddit} - fetched ${posts.length} posts`);
  return posts;
}

function calculateStats(posts, subreddit) {
  const scores = posts.map(p => p.score);
  const comments = posts.map(p => p.num_comments);
  const upvoteRatios = posts.filter(p => p.upvote_ratio).map(p => p.upvote_ratio);

  const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
  const avgComments = comments.reduce((a, b) => a + b, 0) / comments.length;
  const avgUpvoteRatio = upvoteRatios.length > 0
    ? upvoteRatios.reduce((a, b) => a + b, 0) / upvoteRatios.length
    : 0;

  // 时间分布
  const now = Date.now() / 1000;
  const timeRanges = {
    '24h': posts.filter(p => now - p.created_utc < 86400).length,
    '7d': posts.filter(p => now - p.created_utc < 86400 * 7).length,
    '30d': posts.filter(p => now - p.created_utc < 86400 * 30).length,
    'older': posts.filter(p => now - p.created_utc >= 86400 * 30).length
  };

  return {
    subreddit,
    count: posts.length,
    avgScore: Math.round(avgScore),
    avgComments: Math.round(avgComments),
    avgUpvoteRatio: avgUpvoteRatio.toFixed(3),
    maxScore: Math.max(...scores),
    maxComments: Math.max(...comments),
    timeDistribution: timeRanges
  };
}

async function main() {
  log('=== Reddit Scraper v2 - Starting Batch Crawl ===');
  log(`Target: ${SUBREDDITS.reduce((sum, s) => sum + s.target, 0)} posts total`);
  log(`Rate limit: ${RATE_LIMIT_MS}ms between requests`);
  log('');

  for (const config of SUBREDDITS) {
    const posts = await fetchSubreddit(config.name, config.sort, config.time, config.target);
    allPosts.push(...posts);
    stats[config.name] = calculateStats(posts, config.name);
    log('');
    await sleep(RATE_LIMIT_MS);
  }

  // 保存原始数据
  const rawDataPath = path.join(OUTPUT_DIR, 'raw_reddit_data.json');
  fs.writeFileSync(rawDataPath, JSON.stringify(allPosts, null, 2));
  log(`Saved raw data: ${rawDataPath} (${allPosts.length} posts)`);

  // 生成统计报告
  const statsMd = generateStatsMarkdown();
  const statsPath = path.join(OUTPUT_DIR, 'data_stats.md');
  fs.writeFileSync(statsPath, statsMd);
  log(`Saved stats report: ${statsPath}`);

  // 保存抓取日志
  const logPath = path.join(OUTPUT_DIR, 'crawl_log.md');
  fs.writeFileSync(logPath, crawlLog.join('\n'));
  log(`Saved crawl log: ${logPath}`);

  // 生成Top帖子摘要
  const topPostsMd = generateTopPostsMarkdown();
  const topPostsPath = path.join(OUTPUT_DIR, 'top_posts_sample.md');
  fs.writeFileSync(topPostsPath, topPostsMd);
  log(`Saved top posts sample: ${topPostsPath}`);

  log('');
  log('=== Crawl Complete ===');
  log(`Total posts: ${allPosts.length}`);
  log(`Fields completeness check:`);

  const fieldChecks = ['id', 'title', 'author', 'score', 'num_comments', 'upvote_ratio', 'created_utc'];
  for (const field of fieldChecks) {
    const present = allPosts.filter(p => p[field] !== undefined && p[field] !== null).length;
    const pct = ((present / allPosts.length) * 100).toFixed(1);
    log(`  ${field}: ${present}/${allPosts.length} (${pct}%)`);
  }
}

function generateStatsMarkdown() {
  const total = Object.values(stats).reduce((sum, s) => sum + s.count, 0);

  let md = `# Reddit Data Statistics Report\n\n`;
  md += `Generated: ${new Date().toISOString()}\n\n`;
  md += `## Summary\n\n`;
  md += `- **Total Posts**: ${total}\n`;
  md += `- **Subreddits**: ${Object.keys(stats).length}\n\n`;

  md += `## Per-Subreddit Breakdown\n\n`;
  md += `| Subreddit | Posts | Avg Score | Avg Comments | Avg Upvote Ratio | Max Score | Max Comments |\n`;
  md += `|-----------|-------|-----------|--------------|------------------|-----------|--------------|\n`;

  for (const [name, s] of Object.entries(stats)) {
    md += `| r/${name} | ${s.count} | ${s.avgScore.toLocaleString()} | ${s.avgComments.toLocaleString()} | ${s.avgUpvoteRatio} | ${s.maxScore.toLocaleString()} | ${s.maxComments.toLocaleString()} |\n`;
  }

  md += `\n## Time Distribution\n\n`;
  md += `| Subreddit | Last 24h | Last 7d | Last 30d | Older |\n`;
  md += `|-----------|----------|---------|----------|-------|\n`;

  for (const [name, s] of Object.entries(stats)) {
    const t = s.timeDistribution;
    md += `| r/${name} | ${t['24h']} | ${t['7d']} | ${t['30d']} | ${t['older']} |\n`;
  }

  md += `\n## Field Completeness\n\n`;
  const fieldChecks = ['id', 'title', 'author', 'selftext', 'score', 'upvotes', 'num_comments', 'upvote_ratio', 'created_utc', 'permalink'];
  for (const field of fieldChecks) {
    const present = allPosts.filter(p => p[field] !== undefined && p[field] !== null).length;
    const pct = ((present / allPosts.length) * 100).toFixed(1);
    md += `- **${field}**: ${present}/${allPosts.length} (${pct}%)\n`;
  }

  return md;
}

function generateTopPostsMarkdown() {
  let md = `# Top Posts Sample by Subreddit\n\n`;
  md += `Generated: ${new Date().toISOString()}\n\n`;

  for (const config of SUBREDDITS) {
    const subPosts = allPosts.filter(p => p.subreddit.toLowerCase() === config.name.toLowerCase());
    const top10 = subPosts.sort((a, b) => b.score - a.score).slice(0, 10);

    md += `## r/${config.name}\n\n`;
    md += `Total fetched: ${subPosts.length} posts | Showing top 10 by score\n\n`;

    for (let i = 0; i < top10.length; i++) {
      const p = top10[i];
      const date = new Date(p.created_utc * 1000).toISOString().split('T')[0];
      md += `### ${i + 1}. ${p.title.substring(0, 100)}${p.title.length > 100 ? '...' : ''}\n\n`;
      md += `- **Score**: ${p.score.toLocaleString()} | **Comments**: ${p.num_comments.toLocaleString()} | **Ratio**: ${p.upvote_ratio || 'N/A'}\n`;
      md += `- **Author**: u/${p.author} | **Date**: ${date}\n`;
      md += `- **Link**: https://reddit.com${p.permalink}\n`;
      if (p.link_flair_text) {
        md += `- **Flair**: ${p.link_flair_text}\n`;
      }
      if (p.selftext) {
        const preview = p.selftext.substring(0, 300).replace(/\n/g, ' ');
        md += `- **Preview**: ${preview}${p.selftext.length > 300 ? '...' : ''}\n`;
      }
      md += `\n`;
    }

    md += `---\n\n`;
  }

  return md;
}

main().catch(err => {
  log(`Fatal error: ${err.message}`);
  process.exit(1);
});