宜博 / lemonboss

const fs = require('fs');
const path = require('path');

// 读取合成数据
const dataPath = path.join(__dirname, 'synthetic_reddit_data.json');
const posts = JSON.parse(fs.readFileSync(dataPath, 'utf8'));

console.log(`Loaded ${posts.length} posts for analysis\n`);

// ========== 1. 详细统计分析 ==========
const analysis = {
  // 基础统计
  total: posts.length,
  subredditStats: {},
  conflictTypeStats: {},
  dramaticLevelDist: { high: 0, medium: 0, low: 0 },
  dramaCandidates: 0,

  // 高互动内容分析
  highEngagement: { count: 0, posts: [] }, // score > 5000, comments > 1000
  engagementBySubreddit: {},

  // 评分分布
  dramaticScoreDist: {},

  // 字段完整度
  fieldCompleteness: {
    id: 0, title: 0, author: 0, selftext: 0, score: 0,
    num_comments: 0, upvote_ratio: 0, created_utc: 0, permalink: 0,
    drama_elements: 0, dramatic_score: 0
  }
};

posts.forEach(post => {
  // Subreddit 统计
  if (!analysis.subredditStats[post.subreddit]) {
    analysis.subredditStats[post.subreddit] = {
      count: 0,
      totalScore: 0, totalComments: 0, totalUpvoteRatio: 0,
      maxScore: 0, maxComments: 0,
      scores: [], comments: [], upvoteRatios: []
    };
  }
  const sr = analysis.subredditStats[post.subreddit];
  sr.count++;
  sr.totalScore += post.score;
  sr.totalComments += post.num_comments;
  sr.totalUpvoteRatio += post.upvote_ratio;
  sr.maxScore = Math.max(sr.maxScore, post.score);
  sr.maxComments = Math.max(sr.maxComments, post.num_comments);
  sr.scores.push(post.score);
  sr.comments.push(post.num_comments);
  sr.upvoteRatios.push(post.upvote_ratio);

  // 冲突类型统计
  analysis.conflictTypeStats[post.drama_elements.conflict_type] =
    (analysis.conflictTypeStats[post.drama_elements.conflict_type] || 0) + 1;

  // 戏剧性等级分布
  analysis.dramaticLevelDist[post.dramatic_level]++;
  if (post.is_drama_candidate) analysis.dramaCandidates++;

  // 高互动内容
  if (post.score > 5000 || post.num_comments > 1000) {
    analysis.highEngagement.count++;
    analysis.highEngagement.posts.push({
      id: post.id,
      subreddit: post.subreddit,
      score: post.score,
      comments: post.num_comments,
      dramatic_score: post.dramatic_score,
      title: post.title.substring(0, 60)
    });
  }

  // 评分分布（按区间）
  const scoreRange = Math.floor(post.dramatic_score);
  analysis.dramaticScoreDist[scoreRange] = (analysis.dramaticScoreDist[scoreRange] || 0) + 1;

  // 字段完整度
  Object.keys(analysis.fieldCompleteness).forEach(field => {
    if (post[field] !== undefined && post[field] !== null && post[field] !== '') {
      analysis.fieldCompleteness[field]++;
    }
  });
});

// 计算各 subreddit 的均值和分位数
Object.entries(analysis.subredditStats).forEach(([name, stats]) => {
  stats.avgScore = Math.round(stats.totalScore / stats.count);
  stats.avgComments = Math.round(stats.totalComments / stats.count);
  stats.avgUpvoteRatio = (stats.totalUpvoteRatio / stats.count).toFixed(2);

  // 计算分位数
  const sortedScores = [...stats.scores].sort((a, b) => a - b);
  const sortedComments = [...stats.comments].sort((a, b) => a - b);
  stats.p50Score = sortedScores[Math.floor(sortedScores.length * 0.5)];
  stats.p75Score = sortedScores[Math.floor(sortedScores.length * 0.75)];
  stats.p90Score = sortedScores[Math.floor(sortedScores.length * 0.90)];
  stats.p50Comments = sortedComments[Math.floor(sortedComments.length * 0.5)];
  stats.p75Comments = sortedComments[Math.floor(sortedComments.length * 0.75)];
});

// 字段完整度百分比
Object.keys(analysis.fieldCompleteness).forEach(field => {
  analysis.fieldCompleteness[field] = {
    count: analysis.fieldCompleteness[field],
    percent: Math.round((analysis.fieldCompleteness[field] / posts.length) * 100)
  };
});

// 输出分析报告
console.log('=== REDDIT SCRAPER V2 - DATA ANALYSIS REPORT ===\n');

console.log('## 1. 整体分布统计\n');
console.log(`- 总帖子数: ${analysis.total}`);
console.log(`- 戏剧性候选数 (≥6.0): ${analysis.dramaCandidates} (${((analysis.dramaCandidates/analysis.total)*100).toFixed(1)}%)`);
console.log(`- 高互动内容 (score>5000 或 comments>1000): ${analysis.highEngagement.count}`);
console.log(`\n戏剧性等级分布:`);
console.log(`  - 高戏剧性 (≥7.0): ${analysis.dramaticLevelDist.high} (${((analysis.dramaticLevelDist.high/analysis.total)*100).toFixed(1)}%)`);
console.log(`  - 中戏剧性 (5.0-7.0): ${analysis.dramaticLevelDist.medium} (${((analysis.dramaticLevelDist.medium/analysis.total)*100).toFixed(1)}%)`);
console.log(`  - 低戏剧性 (<5.0): ${analysis.dramaticLevelDist.low} (${((analysis.dramaticLevelDist.low/analysis.total)*100).toFixed(1)}%)`);

console.log('\n## 2. Subreddit 分布统计\n');
console.log('| Subreddit | 数量 | 占比 | 平均点赞 | 平均评论 | 平均UP率 | 最大点赞 | 最大评论 |');
console.log('|-----------|------|------|----------|----------|----------|----------|----------|');
Object.entries(analysis.subredditStats).forEach(([name, stats]) => {
  const pct = ((stats.count / analysis.total) * 100).toFixed(1);
  console.log(`| r/${name} | ${stats.count} | ${pct}% | ${stats.avgScore} | ${stats.avgComments} | ${stats.avgUpvoteRatio} | ${stats.maxScore} | ${stats.maxComments} |`);
});

console.log('\n## 3. 分位数统计\n');
console.log('| Subreddit | P50点赞 | P75点赞 | P90点赞 | P50评论 | P75评论 |');
console.log('|-----------|---------|---------|---------|---------|---------|');
Object.entries(analysis.subredditStats).forEach(([name, stats]) => {
  console.log(`| r/${name} | ${stats.p50Score} | ${stats.p75Score} | ${stats.p90Score} | ${stats.p50Comments} | ${stats.p75Comments} |`);
});

console.log('\n## 4. 冲突类型分布\n');
const sortedConflicts = Object.entries(analysis.conflictTypeStats).sort((a, b) => b[1] - a[1]);
sortedConflicts.forEach(([type, count]) => {
  const pct = ((count / analysis.total) * 100).toFixed(1);
  console.log(`- ${type}: ${count} (${pct}%)`);
});

console.log('\n## 5. 戏剧性评分分布\n');
Object.entries(analysis.dramaticScoreDist).sort((a, b) => parseInt(a[0]) - parseInt(b[0])).forEach(([score, count]) => {
  const bar = '█'.repeat(Math.round(count / 2));
  console.log(`  ${score}: ${bar} ${count}`);
});

console.log('\n## 6. 高互动内容样本 (Top 15)\n');
analysis.highEngagement.posts
  .sort((a, b) => (b.score + b.comments) - (a.score + a.comments))
  .slice(0, 15)
  .forEach((post, i) => {
    console.log(`${i + 1}. [${post.subreddit}] ${post.title}...`);
    console.log(`   点赞:${post.score} 评论:${post.comments} 戏剧性:${post.dramatic_score}`);
  });

console.log('\n## 7. 字段完整度\n');
Object.entries(analysis.fieldCompleteness).forEach(([field, stats]) => {
  console.log(`- ${field}: ${stats.count}/${posts.length} (${stats.percent}%)`);
});

// 保存详细分析结果
fs.writeFileSync(
  path.join(__dirname, 'data_analysis.json'),
  JSON.stringify(analysis, null, 2)
);

console.log('\n\n✓ 分析报告已保存至 data_analysis.json');