宜博 / lemonboss

#!/usr/bin/env node
/**
 * 热点监控爬虫 - RSS 采集脚本
 * 负责采集、清洗、分类、评分并输出结构化热点数据
 */

const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');

// 配置
const CONFIG_PATH = '/Users/yibo/Documents/LemonAI/lemonboss/data/workspace/wechatai/config/hotspot_sources.json';
const OUTPUT_DIR = '/Users/yibo/Documents/LemonAI/lemonboss/data/workspace/wechatai/hotspots';
const DATA_CONTRACT_VERSION = '1.0';

// AI 关键词分类配置
const CATEGORY_KEYWORDS = {
  llm: {
    name: '大模型',
    keywords: ['GPT', '大模型', 'LLM', 'Claude', 'Gemini', 'Llama', '大语言模型', 'Transformer', 'GPT-5', 'GPT-4', 'GPT-3', '多模态']
  },
  ai_app: {
    name: 'AI应用',
    keywords: ['AI应用', 'AI工具', 'ChatGPT', 'Agent', 'AI产品', '生成式AI', 'AIGC', 'Copilot', 'AI助手', '智能体', 'AI Agent', 'Vibe Coding']
  },
  ai_hardware: {
    name: 'AI硬件',
    keywords: ['AI芯片', 'GPU', 'NPU', 'TPU', 'AI手机', 'AI PC', '机器人', '具身智能', '人形机器人', '算力', '英伟达', 'NVIDIA']
  },
  industry: {
    name: '行业动态',
    keywords: ['OpenAI', 'Google', 'Microsoft', 'Meta', '字节', '阿里', '腾讯', '百度', 'Anthropic', 'DeepSeek', 'xAI', 'Mistral', 'AI投资', '融资', '收购', 'IPO', '财报', '裁员', '招聘']
  },
  research: {
    name: '论文/研究',
    keywords: ['论文', 'arXiv', '研究', 'Paper', 'SOTA', 'benchmark', '数据集', '算法', '模型架构', 'CVPR', 'ICML', 'NeurIPS', 'ACL']
  }
};

// 数据源可信度评分
const SOURCE_SCORES = {
  'jiqizhixin': 85,
  'qbitai': 85,
  'openai_blog': 95,
  '36kr': 75,
  'hackernews': 80
};

/**
 * 生成唯一 ID
 */
function generateId(prefix = 'hs') {
  return `${prefix}-${Date.now()}-${Math.random().toString(36).substr(2, 6)}`;
}

/**
 * 计算热度评分
 */
function calculateHotScore(item, sourceId) {
  // 基础分数：数据源可信度 (0-100)
  const sourceScore = SOURCE_SCORES[sourceId] || 50;

  // 时效性分数：越新越高 (0-100)
  const now = new Date();
  const published = new Date(item.published_at);
  const hoursDiff = (now - published) / (1000 * 60 * 60);
  let timelinessScore = Math.max(0, 100 - hoursDiff * 5); // 每小时衰减 5 分

  // 内容质量分数：基于标题长度和内容丰富度 (0-100)
  let contentScore = 50;
  if (item.title) {
    contentScore += Math.min(30, item.title.length / 2);
  }
  if (item.summary) {
    contentScore += Math.min(20, item.summary.length / 50);
  }

  // 关键词热度加分
  let keywordBonus = 0;
  const text = `${item.title || ''} ${item.summary || ''}`.toLowerCase();
  if (/gpt-?5|claude|openai/i.test(text)) keywordBonus += 10;
  if (/agent|智能体/i.test(text)) keywordBonus += 5;
  if (/融资|收购|ipo/i.test(text)) keywordBonus += 5;

  // 加权计算总分
  const finalScore = Math.min(100, Math.round(
    sourceScore * 0.35 +
    timelinessScore * 0.30 +
    Math.min(100, contentScore) * 0.20 +
    keywordBonus
  ));

  return finalScore;
}

/**
 * 分类 AI 内容
 */
function classifyContent(title, summary = '') {
  const text = `${title} ${summary}`.toLowerCase();
  const categories = [];

  for (const [catId, config] of Object.entries(CATEGORY_KEYWORDS)) {
    const matched = config.keywords.some(kw => text.includes(kw.toLowerCase()));
    if (matched) {
      categories.push(catId);
    }
  }

  // 默认分类
  if (categories.length === 0) {
    categories.push('industry');
  }

  return categories;
}

/**
 * 提取关键词
 */
function extractKeywords(title, summary = '') {
  const text = `${title} ${summary}`;
  const keywords = new Set();

  // 提取所有分类关键词
  for (const config of Object.values(CATEGORY_KEYWORDS)) {
    for (const kw of config.keywords) {
      if (text.toLowerCase().includes(kw.toLowerCase())) {
        keywords.add(kw);
      }
    }
  }

  // 提取公司/产品名（首字母大写的单词）
  const properNouns = text.match(/[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*/g) || [];
  for (const noun of properNouns.slice(0, 5)) {
    if (noun.length > 2 && !keywords.has(noun)) {
      keywords.add(noun);
    }
  }

  return Array.from(keywords).slice(0, 10);
}

/**
 * 解析 RSS 内容
 */
function parseRSSContent(content, sourceId, sourceName) {
  const items = [];
  const now = new Date().toISOString();

  // 简单的 RSS item 提取（基于正则）
  const itemRegex = /<item>[\s\S]*?<\/item>/gi;
  const itemsMatch = content.match(itemRegex) || [];

  for (const itemXml of itemsMatch.slice(0, 20)) { // 限制每个源 20 条
    try {
      // 提取标题
      const titleMatch = itemXml.match(/<title>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/title>/i);
      const title = titleMatch ? titleMatch[1].replace(/<!\[CDATA\[|\]\]>/g, '').trim() : '';

      // 提取链接
      const linkMatch = itemXml.match(/<link>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/link>/i);
      const url = linkMatch ? linkMatch[1].replace(/<!\[CDATA\[|\]\]>/g, '').trim() : '';

      // 提取描述/摘要
      const descMatch = itemXml.match(/<description>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/description>/i);
      let summary = descMatch ? descMatch[1].replace(/<!\[CDATA\[|\]\]>/g, '').trim() : '';
      // 去除 HTML 标签
      summary = summary.replace(/<[^>]+>/g, '').substring(0, 500);

      // 提取发布时间
      const pubDateMatch = itemXml.match(/<pubDate>([\s\S]*?)<\/pubDate>/i);
      let published_at = now;
      if (pubDateMatch) {
        const date = new Date(pubDateMatch[1]);
        if (!isNaN(date)) published_at = date.toISOString();
      }

      // 提取作者
      const authorMatch = itemXml.match(/<author>(?:<!\[CDATA\[)?([\s\S]*?)(?:\]\]>)?<\/author>/i);
      const author = authorMatch ? authorMatch[1].replace(/<!\[CDATA\[|\]\]>/g, '').trim() : sourceName;

      if (title && url) {
        const categories = classifyContent(title, summary);
        const hot_score = calculateHotScore({ title, summary, published_at }, sourceId);

        items.push({
          id: generateId(),
          title,
          summary: summary || title,
          url,
          source: sourceName,
          author,
          published_at,
          categories,
          hot_score,
          keywords: extractKeywords(title, summary),
          fetched_at: now
        });
      }
    } catch (e) {
      console.error(`解析 item 失败: ${e.message}`);
    }
  }

  return items;
}

/**
 * 直接获取 RSS XML 并解析
 */
async function fetchRSSDirectly(rssUrl, sourceId, sourceName) {
  try {
    const result = execSync(`curl -sL "${rssUrl}" --max-time 30 2>/dev/null`, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });

    if (!result.includes('<rss') && !result.includes('<feed')) {
      console.log(`${sourceName}: 直接获取失败，返回内容不是 RSS`);
      return [];
    }

    return parseRSSContent(result, sourceId, sourceName);
  } catch (e) {
    console.error(`${sourceName} 直接获取失败: ${e.message}`);
    return [];
  }
}

/**
 * 通过 jina.ai 获取并解析 RSS（作为 fallback）
 */
async function fetchViaJina(rssUrl, sourceId, sourceName) {
  try {
    const jinaUrl = `https://r.jina.ai/http://${rssUrl.replace(/^https?:\/\//, '')}`;
    const result = execSync(`curl -sL "${jinaUrl}" --max-time 30 2>/dev/null`, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });

    // 如果返回的是 HTML 重定向页面，说明失败了
    if (result.includes('You are being') || result.includes('redirected') || result.trim().length < 100) {
      return [];
    }

    const items = [];
    const now = new Date().toISOString();
    let baseTime = new Date();

    // 提取发布时间（在 Markdown 头部）
    const timeMatch = result.match(/Published Time:\s*(.+)/);
    if (timeMatch) {
      const t = new Date(timeMatch[1]);
      if (!isNaN(t)) baseTime = t;
    }

    // 标准格式: [title](url)date - 用于中文 RSS 源
    const linkRegex = /\[([^\]]+)\]\((https?:\/\/[^\)]+)\)(?:\s*([A-Z][a-z]{2},?\s+\d{1,2}[a-z]*\s*\d{4}[\s\d:]*\s*[A-Z]{3,4})?)?/g;
    let match;
    let count = 0;

    while ((match = linkRegex.exec(result)) !== null && count < 20) {
      const title = match[1].trim();
      const url = match[2].trim();
      const dateStr = match[3];

      // 过滤掉标题为空的条目（这是之前的问题根源）
      if (!title || title === url || title.startsWith('http')) continue;

      const itemDate = dateStr ? new Date(dateStr) : baseTime;
      const published_at = !isNaN(itemDate) ? itemDate.toISOString() : baseTime.toISOString();

      const categories = classifyContent(title);
      const hot_score = calculateHotScore({ title, published_at }, sourceId);

      items.push({
        id: generateId(),
        title,
        summary: title, // fallback 模式下摘要用标题
        url,
        source: sourceName,
        author: sourceName,
        published_at,
        categories,
        hot_score,
        keywords: extractKeywords(title),
        fetched_at: now
      });
      count++;
    }

    return items;
  } catch (e) {
    console.error(`${sourceName} jina.ai 采集失败: ${e.message}`);
    return [];
  }
}

/**
 * 主采集函数
 */
async function crawl() {
  console.log('🤖 热点监控爬虫启动...');
  console.log(`⏰ 采集时间: ${new Date().toISOString()}`);

  // 读取配置
  const config = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf-8'));

  // 采集所有源
  const allItems = [];
  const sourceStats = {};

  for (const source of config.sources.rss) {
    if (source.priority === 'P0' || source.priority === 'P1') {
      console.log(`\n📡 正在采集: ${source.name} (${source.url})`);

      let items = [];
      let usedFallback = false;

      try {
        // 首先尝试直接获取 RSS XML
        items = await fetchRSSDirectly(source.url, source.id, source.name);

        // 如果直接获取失败，尝试 jina.ai fallback
        if (items.length === 0) {
          console.log(`   🔄 直接获取失败，尝试 jina.ai fallback...`);
          items = await fetchViaJina(source.url, source.id, source.name);
          usedFallback = true;
        }

        allItems.push(...items);
        sourceStats[source.id] = {
          name: source.name,
          count: items.length,
          success: items.length > 0,
          fallback: usedFallback
        };
        console.log(`   ✅ 成功采集 ${items.length} 条${usedFallback ? ' (via fallback)' : ''}`);
      } catch (e) {
        sourceStats[source.id] = {
          name: source.name,
          count: 0,
          success: false,
          error: e.message
        };
        console.log(`   ❌ 采集失败: ${e.message}`);
      }
    }
  }

  // 数据去重（基于 URL）
  const seenUrls = new Set();
  const uniqueItems = [];
  for (const item of allItems) {
    if (!seenUrls.has(item.url)) {
      seenUrls.add(item.url);
      uniqueItems.push(item);
    }
  }

  // 按热度排序
  uniqueItems.sort((a, b) => b.hot_score - a.hot_score);

  // 生成输出
  const today = new Date().toISOString().split('T')[0];
  const outputDir = path.join(OUTPUT_DIR, today);

  if (!fs.existsSync(outputDir)) {
    fs.mkdirSync(outputDir, { recursive: true });
  }

  const output = {
    meta: {
      version: DATA_CONTRACT_VERSION,
      created_at: new Date().toISOString(),
      created_by: 'wechatai:crawler',
      data_contract: 'raw_feeds_v1'
    },
    stats: {
      total_count: uniqueItems.length,
      sources_count: Object.keys(sourceStats).length,
      categories_count: Object.keys(CATEGORY_KEYWORDS).length,
      fetch_duration_ms: Date.now() - new Date(allItems[0]?.fetched_at || Date.now()).getTime()
    },
    sources: sourceStats,
    hotspots: uniqueItems
  };

  // 写入文件
  const outputPath = path.join(outputDir, 'raw_feeds.json');
  fs.writeFileSync(outputPath, JSON.stringify(output, null, 2), 'utf-8');

  console.log(`\n✅ 采集完成!`);
  console.log(`📊 总计: ${uniqueItems.length} 条 (去重前: ${allItems.length} 条)`);
  console.log(`📁 输出: ${outputPath}`);
  console.log('\n📈 数据源统计:');
  for (const [id, stat] of Object.entries(sourceStats)) {
    const status = stat.success ? '✅' : '❌';
    console.log(`   ${status} ${stat.name}: ${stat.count} 条`);
  }

  return output;
}

// 执行采集
if (require.main === module) {
  crawl().catch(console.error);
}

module.exports = { crawl, parseRSSContent, calculateHotScore, classifyContent, fetchRSSDirectly, fetchViaJina };