宜博 / lemonboss

{
  "_schema": "crawl_config_v1",
  "name": "热点分析师数据采集配置",
  "description": "为LemonV公司热点分析师Agent训练准备的爬虫配置，覆盖抖音、微博、小红书、快手四大平台",
  "version": "1.0.0",
  "created_at": "2026-03-26",
  "author": "Hunter Agent",

  "global_settings": {
    "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "request_timeout": 15,
    "retry_times": 3,
    "retry_delay": 2,
    "concurrent_requests": 8,
    "download_delay": 2,
    "respect_robots_txt": true,
    "data_retention_days": 90
  },

  "proxy_settings": {
    "enabled": false,
    "type": "rotating",
    "pool_size": 10,
    "providers": [
      "亿牛云爬虫代理",
      "Scrapy-Proxy-Pool"
    ]
  },

  "storage": {
    "type": "hybrid",
    "config": {
      "hot_data": {
        "engine": "redis",
        "host": "localhost",
        "port": 6379,
        "db": 0,
        "ttl": 86400
      },
      "content_data": {
        "engine": "mongodb",
        "host": "localhost",
        "port": 27017,
        "database": "trend_hunter",
        "collections": {
          "weibo_hot": "微博热搜",
          "douyin_hot": "抖音热点",
          "xiaohongshu_notes": "小红书笔记",
          "kuaishou_hot": "快手热榜"
        }
      },
      "timeseries_data": {
        "engine": "influxdb",
        "host": "localhost",
        "port": 8086,
        "database": "trend_metrics"
      }
    }
  },

  "sources": [
    {
      "id": "weibo_hot_search",
      "name": "微博热搜榜",
      "platform": "weibo",
      "type": "hot_list",
      "enabled": true,
      "priority": 1,
      "update_interval": 300,

      "endpoints": [
        {
          "name": "移动端热榜",
          "url": "https://m.weibo.cn/api/container/getIndex?containerid=106003type=25&t=3&disable_hot=1&filter_type=realtimehot",
          "method": "GET",
          "headers": {
            "Referer": "https://m.weibo.cn",
            "Accept": "application/json"
          }
        },
        {
          "name": "PC端热榜",
          "url": "https://s.weibo.com/top/summary?Refer=top_hot",
          "method": "GET",
          "parser": "html"
        }
      ],

      "api_alternatives": [
        {
          "provider": "天聚数行",
          "url": "https://apis.tianapi.com/weibohot/index",
          "method": "GET",
          "params": {
            "key": "YOUR_API_KEY"
          },
          "update_frequency": 1800,
          "quota": "100次/天(免费)"
        },
        {
          "provider": "接口盒子",
          "url": "https://cn.apihz.cn/api/xinwen/weibo2.php",
          "method": "GET",
          "params": {
            "id": "YOUR_ID",
            "key": "YOUR_KEY"
          }
        }
      ],

      "fields": {
        "rank": "排名",
        "desc": "热搜词",
        "desc_extr": "热度值",
        "scheme": "话题链接",
        "timestamp": "采集时间",
        "category": "分类(新/爆/热)"
      },

      "anti_detection": {
        "cookie_rotation": true,
        "ip_rotation": true,
        "request_interval": "5-10秒随机",
        "mobile_first": true
      }
    },

    {
      "id": "douyin_trend",
      "name": "抖音热点",
      "platform": "douyin",
      "type": "hot_list",
      "enabled": true,
      "priority": 1,
      "update_interval": 600,

      "endpoints": [
        {
          "name": "巨量算数热点",
          "url": "https://trendinsight.oceanengine.com",
          "note": "需登录获取API权限",
          "method": "API"
        }
      ],

      "third_party_apis": [
        {
          "provider": "数说聚合",
          "url": "https://hermes.datastory.com.cn/api/douyin/hot",
          "auth": "企业级API Key"
        }
      ],

      "fields": {
        "title": "热点标题",
        "hot_value": "热度值",
        "category": "分类",
        "publish_time": "发布时间",
        "view_count": "播放量",
        "increase_rate": "增长率"
      },

      "note": "抖音反爬严格，建议使用官方API或第三方数据服务"
    },

    {
      "id": "xiaohongshu_trend",
      "name": "小红书热门话题",
      "platform": "xiaohongshu",
      "type": "topic_list",
      "enabled": true,
      "priority": 2,
      "update_interval": 1800,

      "endpoints": [
        {
          "name": "搜索发现页",
          "url": "https://www.xiaohongshu.com/search_result",
          "note": "需登录态Cookie",
          "method": "GET"
        }
      ],

      "third_party_apis": [
        {
          "provider": "新红",
          "url": "https://xh.newrank.cn",
          "type": "商业平台"
        },
        {
          "provider": "千瓜",
          "url": "https://www.qiangua.com",
          "type": "商业平台"
        }
      ],

      "fields": {
        "topic_name": "话题名称",
        "view_count": "浏览量",
        "note_count": "笔记数",
        "participation": "参与人数",
        "hot_ratio": "热度比值(浏览/笔记)",
        "category": "分类"
      },

      "keywords_to_track": [
        "短剧", "爆款", "热点", "趋势", "选题",
        "剧情", "反转", "爽剧", "甜宠", "逆袭"
      ]
    },

    {
      "id": "kuaishou_hot",
      "name": "快手热榜",
      "platform": "kuaishou",
      "type": "hot_list",
      "enabled": true,
      "priority": 2,
      "update_interval": 600,

      "endpoints": [
        {
          "name": "快手热榜API",
          "url": "https://api.jxmb.net/api/hotnews/kuaishou",
          "method": "GET",
          "params": {
            "appid": "YOUR_APPID",
            "appsecret": "YOUR_APPSECRET"
          }
        }
      ],

      "api_details": {
        "provider": "聚合数据",
        "update_frequency": "600秒",
        "free_quota": "注册送100次",
        "paid_quota": "10,000次/天(年付会员)",
        "documentation": "https://api.jxmb.net/user/doc?id=61"
      },

      "fields": {
        "rank": "排行",
        "name": "热点标题",
        "img": "缩略图",
        "date": "热点时间",
        "viewnum": "关注人数",
        "url": "详情链接"
      }
    },

    {
      "id": "toutiao_hot",
      "name": "今日头条热点",
      "platform": "toutiao",
      "type": "hot_list",
      "enabled": true,
      "priority": 3,
      "update_interval": 300,

      "endpoints": [
        {
          "name": "热点API",
          "url": "https://www.toutiao.com/hot/event/hot-board/?origin=toutiao_pc",
          "method": "GET",
          "parser": "json"
        }
      ],

      "fields": {
        "title": "标题",
        "label": "标签",
        "url": "链接",
        "hot_value": "热度值",
        "category": "分类"
      }
    },

    {
      "id": "zhihu_hot",
      "name": "知乎热榜",
      "platform": "zhihu",
      "type": "hot_list",
      "enabled": true,
      "priority": 3,
      "update_interval": 600,

      "endpoints": [
        {
          "name": "热榜API",
          "url": "https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total",
          "method": "GET",
          "headers": {
            "User-Agent": "Mozilla/5.0"
          }
        }
      ],

      "fields": {
        "title": "问题标题",
        "excerpt": "摘要",
        "detail_text": "热度文本",
        "url": "链接"
      }
    },

    {
      "id": "bilibili_hot",
      "name": "B站热门",
      "platform": "bilibili",
      "type": "hot_list",
      "enabled": false,
      "priority": 4,
      "update_interval": 3600,

      "endpoints": [
        {
          "name": "热门视频",
          "url": "https://api.bilibili.com/x/web-interface/ranking/v2",
          "method": "GET"
        }
      ],

      "fields": {
        "title": "标题",
        "owner": "UP主",
        "stat": "统计数据",
        "short_link": "短链接"
      }
    }
  ],

  "data_processing": {
    "deduplication": {
      "method": "bloom_filter",
      "false_positive_rate": 0.001,
      "expected_items": 1000000
    },
    "cleaning_rules": [
      {
        "field": "title",
        "action": "strip_whitespace",
        "description": "去除标题前后空格"
      },
      {
        "field": "hot_value",
        "action": "normalize_number",
        "description": "统一数值格式"
      },
      {
        "field": "content",
        "action": "remove_html_tags",
        "description": "去除HTML标签"
      }
    ],
    "sentiment_analysis": {
      "enabled": true,
      "engine": "jieba+textblob",
      "fields": ["title", "content", "comments"]
    }
  },

  "alerting": {
    "enabled": true,
    "rules": [
      {
        "name": "热点爆发预警",
        "condition": "growth_rate > 300% AND hot_value > 1000000",
        "action": "send_notification",
        "channels": ["webhook", "email"]
      },
      {
        "name": "数据采集异常",
        "condition": "failed_requests > 10 in 5 minutes",
        "action": "send_alert",
        "channels": ["email"]
      }
    ]
  },

  "scheduling": {
    "jobs": [
      {
        "name": "微博热搜采集",
        "source": "weibo_hot_search",
        "cron": "*/5 * * * *",
        "enabled": true
      },
      {
        "name": "抖音热点采集",
        "source": "douyin_trend",
        "cron": "*/10 * * * *",
        "enabled": true
      },
      {
        "name": "小红书话题采集",
        "source": "xiaohongshu_trend",
        "cron": "0 */2 * * *",
        "enabled": true
      },
      {
        "name": "快手热榜采集",
        "source": "kuaishou_hot",
        "cron": "*/10 * * * *",
        "enabled": true
      },
      {
        "name": "知乎热榜采集",
        "source": "zhihu_hot",
        "cron": "*/10 * * * *",
        "enabled": true
      }
    ]
  },

  "output_formats": {
    "raw_data": {
      "format": "json",
      "compression": "gzip",
      "retention": "30_days"
    },
    "analytics": {
      "format": "parquet",
      "destination": "s3://analytics-bucket/trend-hunter/"
    },
    "reports": {
      "format": "markdown",
      "schedule": "daily"
    }
  },

  "compliance": {
    "robots_txt": true,
    "rate_limiting": true,
    "user_agent_identification": true,
    "data_privacy": {
      "exclude_personal_info": true,
      "anonymize_user_data": true,
      "retention_policy": "90_days"
    },
    "legal_notice": "本配置仅供学习和研究使用，请遵守各平台的robots.txt和使用协议"
  },

  "monitoring": {
    "metrics": [
      "requests_per_minute",
      "success_rate",
      "average_response_time",
      "data_volume_per_hour",
      "unique_items_per_day"
    ],
    "dashboard": {
      "enabled": true,
      "tool": "grafana",
      "refresh_interval": "10s"
    }
  }
}