{ "_schema": "crawl_config_v1", "name": "热点分析师数据采集配置", "description": "为LemonV公司热点分析师Agent训练准备的爬虫配置，覆盖抖音、微博、小红书、快手四大平台", "version": "1.0.0", "created_at": "2026-03-26", "author": "Hunter Agent", "global_settings": { "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "request_timeout": 15, "retry_times": 3, "retry_delay": 2, "concurrent_requests": 8, "download_delay": 2, "respect_robots_txt": true, "data_retention_days": 90 }, "proxy_settings": { "enabled": false, "type": "rotating", "pool_size": 10, "providers": [ "亿牛云爬虫代理", "Scrapy-Proxy-Pool" ] }, "storage": { "type": "hybrid", "config": { "hot_data": { "engine": "redis", "host": "localhost", "port": 6379, "db": 0, "ttl": 86400 }, "content_data": { "engine": "mongodb", "host": "localhost", "port": 27017, "database": "trend_hunter", "collections": { "weibo_hot": "微博热搜", "douyin_hot": "抖音热点", "xiaohongshu_notes": "小红书笔记", "kuaishou_hot": "快手热榜" } }, "timeseries_data": { "engine": "influxdb", "host": "localhost", "port": 8086, "database": "trend_metrics" } } }, "sources": [ { "id": "weibo_hot_search", "name": "微博热搜榜", "platform": "weibo", "type": "hot_list", "enabled": true, "priority": 1, "update_interval": 300, "endpoints": [ { "name": "移动端热榜", "url": "https://m.weibo.cn/api/container/getIndex?containerid=106003type=25&t=3&disable_hot=1&filter_type=realtimehot", "method": "GET", "headers": { "Referer": "https://m.weibo.cn", "Accept": "application/json" } }, { "name": "PC端热榜", "url": "https://s.weibo.com/top/summary?Refer=top_hot", "method": "GET", "parser": "html" } ], "api_alternatives": [ { "provider": "天聚数行", "url": "https://apis.tianapi.com/weibohot/index", "method": "GET", "params": { "key": "YOUR_API_KEY" }, "update_frequency": 1800, "quota": "100次/天(免费)" }, { "provider": "接口盒子", "url": "https://cn.apihz.cn/api/xinwen/weibo2.php", "method": "GET", "params": { "id": "YOUR_ID", "key": "YOUR_KEY" } } ], "fields": { "rank": "排名", "desc": "热搜词", "desc_extr": "热度值", "scheme": "话题链接", "timestamp": "采集时间", "category": "分类(新/爆/热)" }, "anti_detection": { "cookie_rotation": true, "ip_rotation": true, "request_interval": "5-10秒随机", "mobile_first": true } }, { "id": "douyin_trend", "name": "抖音热点", "platform": "douyin", "type": "hot_list", "enabled": true, "priority": 1, "update_interval": 600, "endpoints": [ { "name": "巨量算数热点", "url": "https://trendinsight.oceanengine.com", "note": "需登录获取API权限", "method": "API" } ], "third_party_apis": [ { "provider": "数说聚合", "url": "https://hermes.datastory.com.cn/api/douyin/hot", "auth": "企业级API Key" } ], "fields": { "title": "热点标题", "hot_value": "热度值", "category": "分类", "publish_time": "发布时间", "view_count": "播放量", "increase_rate": "增长率" }, "note": "抖音反爬严格，建议使用官方API或第三方数据服务" }, { "id": "xiaohongshu_trend", "name": "小红书热门话题", "platform": "xiaohongshu", "type": "topic_list", "enabled": true, "priority": 2, "update_interval": 1800, "endpoints": [ { "name": "搜索发现页", "url": "https://www.xiaohongshu.com/search_result", "note": "需登录态Cookie", "method": "GET" } ], "third_party_apis": [ { "provider": "新红", "url": "https://xh.newrank.cn", "type": "商业平台" }, { "provider": "千瓜", "url": "https://www.qiangua.com", "type": "商业平台" } ], "fields": { "topic_name": "话题名称", "view_count": "浏览量", "note_count": "笔记数", "participation": "参与人数", "hot_ratio": "热度比值(浏览/笔记)", "category": "分类" }, "keywords_to_track": [ "短剧", "爆款", "热点", "趋势", "选题", "剧情", "反转", "爽剧", "甜宠", "逆袭" ] }, { "id": "kuaishou_hot", "name": "快手热榜", "platform": "kuaishou", "type": "hot_list", "enabled": true, "priority": 2, "update_interval": 600, "endpoints": [ { "name": "快手热榜API", "url": "https://api.jxmb.net/api/hotnews/kuaishou", "method": "GET", "params": { "appid": "YOUR_APPID", "appsecret": "YOUR_APPSECRET" } } ], "api_details": { "provider": "聚合数据", "update_frequency": "600秒", "free_quota": "注册送100次", "paid_quota": "10,000次/天(年付会员)", "documentation": "https://api.jxmb.net/user/doc?id=61" }, "fields": { "rank": "排行", "name": "热点标题", "img": "缩略图", "date": "热点时间", "viewnum": "关注人数", "url": "详情链接" } }, { "id": "toutiao_hot", "name": "今日头条热点", "platform": "toutiao", "type": "hot_list", "enabled": true, "priority": 3, "update_interval": 300, "endpoints": [ { "name": "热点API", "url": "https://www.toutiao.com/hot/event/hot-board/?origin=toutiao_pc", "method": "GET", "parser": "json" } ], "fields": { "title": "标题", "label": "标签", "url": "链接", "hot_value": "热度值", "category": "分类" } }, { "id": "zhihu_hot", "name": "知乎热榜", "platform": "zhihu", "type": "hot_list", "enabled": true, "priority": 3, "update_interval": 600, "endpoints": [ { "name": "热榜API", "url": "https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total", "method": "GET", "headers": { "User-Agent": "Mozilla/5.0" } } ], "fields": { "title": "问题标题", "excerpt": "摘要", "detail_text": "热度文本", "url": "链接" } }, { "id": "bilibili_hot", "name": "B站热门", "platform": "bilibili", "type": "hot_list", "enabled": false, "priority": 4, "update_interval": 3600, "endpoints": [ { "name": "热门视频", "url": "https://api.bilibili.com/x/web-interface/ranking/v2", "method": "GET" } ], "fields": { "title": "标题", "owner": "UP主", "stat": "统计数据", "short_link": "短链接" } } ], "data_processing": { "deduplication": { "method": "bloom_filter", "false_positive_rate": 0.001, "expected_items": 1000000 }, "cleaning_rules": [ { "field": "title", "action": "strip_whitespace", "description": "去除标题前后空格" }, { "field": "hot_value", "action": "normalize_number", "description": "统一数值格式" }, { "field": "content", "action": "remove_html_tags", "description": "去除HTML标签" } ], "sentiment_analysis": { "enabled": true, "engine": "jieba+textblob", "fields": ["title", "content", "comments"] } }, "alerting": { "enabled": true, "rules": [ { "name": "热点爆发预警", "condition": "growth_rate > 300% AND hot_value > 1000000", "action": "send_notification", "channels": ["webhook", "email"] }, { "name": "数据采集异常", "condition": "failed_requests > 10 in 5 minutes", "action": "send_alert", "channels": ["email"] } ] }, "scheduling": { "jobs": [ { "name": "微博热搜采集", "source": "weibo_hot_search", "cron": "*/5 * * * *", "enabled": true }, { "name": "抖音热点采集", "source": "douyin_trend", "cron": "*/10 * * * *", "enabled": true }, { "name": "小红书话题采集", "source": "xiaohongshu_trend", "cron": "0 */2 * * *", "enabled": true }, { "name": "快手热榜采集", "source": "kuaishou_hot", "cron": "*/10 * * * *", "enabled": true }, { "name": "知乎热榜采集", "source": "zhihu_hot", "cron": "*/10 * * * *", "enabled": true } ] }, "output_formats": { "raw_data": { "format": "json", "compression": "gzip", "retention": "30_days" }, "analytics": { "format": "parquet", "destination": "s3://analytics-bucket/trend-hunter/" }, "reports": { "format": "markdown", "schedule": "daily" } }, "compliance": { "robots_txt": true, "rate_limiting": true, "user_agent_identification": true, "data_privacy": { "exclude_personal_info": true, "anonymize_user_data": true, "retention_policy": "90_days" }, "legal_notice": "本配置仅供学习和研究使用，请遵守各平台的robots.txt和使用协议" }, "monitoring": { "metrics": [ "requests_per_minute", "success_rate", "average_response_time", "data_volume_per_hour", "unique_items_per_day" ], "dashboard": { "enabled": true, "tool": "grafana", "refresh_interval": "10s" } } }