{ "skill_name": "trend-scout-cn", "version": "1.0.0", "description": "中国社交媒体趋势监控爬取配置 — 覆盖微博热搜、B站排行、知乎热榜、抖音热点、小红书、百度指数", "crawl_time": "2026-04-13T02:40:00Z", "global_settings": { "retry_times": 3, "retry_delay_ms": 5000, "timeout_ms": 15000, "concurrent_limit": 3, "default_headers": { "Accept": "application/json, text/plain, */*", "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", "Accept-Encoding": "gzip, deflate, br", "Connection": "keep-alive" }, "respect_robots_txt": true, "storage_base": "data/trends" }, "sources": [ { "id": "weibo_hotsearch", "name": "微博实时热搜总榜", "platform": "weibo", "type": "api", "endpoint": "https://weibo.com/ajax/side/hotSearch", "method": "GET", "description": "微博公开热搜 API,返回实时热搜前 50 条,无需登录", "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "application/json, text/plain, */*", "Accept-Language": "zh-CN,zh;q=0.9", "Referer": "https://weibo.com/", "X-Requested-With": "XMLHttpRequest", "sec-ch-ua": "\"Not_A Brand\";v=\"8\", \"Chromium\";v=\"120\"", "sec-ch-ua-platform": "\"macOS\"" }, "parse_rules": { "format": "json", "list_path": "data.realtime", "field_mapping": { "keyword": "word", "heat_score": "num", "rank": "realpos", "label": "label_name", "is_topic": "topic_flag", "word_scheme": "word_scheme", "flag": "flag", "note": "note" }, "normalize": { "heat_score": "to_integer", "label": "decode_empty_to_null", "is_topic": "to_integer" } }, "rate_limit": { "requests_per_minute": 2, "min_interval_ms": 30000, "daily_limit": 2000, "backoff_on_429": true }, "priority": 1, "status": "active", "last_verified": "2026-04-13", "auth_required": false, "data_schema": "standard_trend" }, { "id": "weibo_entertainment", "name": "微博文娱榜", "platform": "weibo", "type": "api", "endpoint": "https://weibo.com/ajax/statuses/entertainment", "method": "GET", "description": "微博文娱垂直榜单,需 SUB Cookie,适用于影视、综艺、偶像相关内容", "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "application/json, text/plain, */*", "Referer": "https://weibo.com/", "X-Requested-With": "XMLHttpRequest" }, "cookie_required": ["SUB", "XSRF-TOKEN"], "parse_rules": { "format": "json", "list_path": "data.list", "field_mapping": { "keyword": "text", "heat_score": "attitudes_count", "rank": "realpos", "author": "user.screen_name" } }, "rate_limit": { "requests_per_minute": 1, "min_interval_ms": 60000, "daily_limit": 500, "backoff_on_429": true }, "priority": 2, "status": "active", "last_verified": "2026-04-13", "auth_required": true, "data_schema": "standard_trend" }, { "id": "bilibili_ranking", "name": "B站热门排行榜", "platform": "bilibili", "type": "api", "endpoint": "https://api.bilibili.com/x/web-interface/ranking/v2", "method": "GET", "params": { "rid": 0, "type": "all" }, "description": "B站官方热门排行榜 API,返回全站热门视频,字段包含播放量/点赞/弹幕等完整数据", "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "application/json, text/plain, */*", "Referer": "https://www.bilibili.com/", "Origin": "https://www.bilibili.com" }, "parse_rules": { "format": "json", "list_path": "data.list", "field_mapping": { "keyword": "title", "author": "owner.name", "rank": "index", "heat_score": "stat.view", "likes": "stat.like", "coins": "stat.coin", "favorites": "stat.favorite", "danmaku": "stat.danmaku", "shares": "stat.share", "category": "tname", "bvid": "bvid", "description": "desc", "duration": "duration", "pubdate": "pubdate" }, "normalize": { "heat_score": "to_integer", "rank": "to_integer" } }, "rate_limit": { "requests_per_minute": 12, "min_interval_ms": 5000, "daily_limit": 10000, "backoff_on_429": true }, "priority": 1, "status": "active", "last_verified": "2026-04-13", "auth_required": false, "data_schema": "bilibili_video" }, { "id": "bilibili_topic_hot", "name": "B站全站热门话题", "platform": "bilibili", "type": "api", "endpoint": "https://api.bilibili.com/x/web-interface/ranking/v2", "method": "GET", "params": { "rid": 0, "type": "hot" }, "description": "B站热门视频(按热度排序),全类型", "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Referer": "https://www.bilibili.com/" }, "parse_rules": { "format": "json", "list_path": "data.list", "field_mapping": { "keyword": "title", "author": "owner.name", "rank": "index", "heat_score": "stat.view", "likes": "stat.like", "category": "tname", "bvid": "bvid" } }, "rate_limit": { "requests_per_minute": 12, "min_interval_ms": 5000 }, "priority": 2, "status": "active", "auth_required": false, "data_schema": "bilibili_video" }, { "id": "zhihu_hot", "name": "知乎热榜", "platform": "zhihu", "type": "api", "endpoint": "https://api.zhihu.com/topstory/hot-lists/total", "method": "GET", "description": "知乎全站热榜,返回前50条热门问答,深度讨论与社会议题的重要来源", "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "application/json", "Accept-Language": "zh-CN,zh;q=0.9", "Referer": "https://www.zhihu.com/", "X-Requested-With": "XMLHttpRequest" }, "parse_rules": { "format": "json", "list_path": "data", "field_mapping": { "keyword": "target.title", "rank": "index", "heat_score": "target.heat_percent", "answer_count": "target.answer_count", "follower_count": "target.follower_count", "comment_count": "target.comment_count", "snippet": "target.excerpt", "topic": "target.topic.zh_column.name", "created": "target.created", "updated_time": "target.updated_time", "question_id": "target.id" }, "normalize": { "rank": "to_integer", "answer_count": "to_integer", "follower_count": "to_integer" } }, "rate_limit": { "requests_per_minute": 6, "min_interval_ms": 10000, "daily_limit": 5000, "backoff_on_429": true }, "priority": 1, "status": "active", "last_verified": "2026-04-13", "auth_required": false, "data_schema": "zhihu_question" }, { "id": "douyin_hot_list", "name": "抖音热点榜(渲染模式)", "platform": "douyin", "type": "web_scraper", "endpoint": "https://www.douyin.com/hot", "method": "GET", "description": "抖音全站热点榜,需 Playwright 渲染页面提取热搜词条,强反爬建议 Playwright headless 模式", "render_required": true, "render_engine": "playwright", "render_options": { "wait_for_selector": ".hot-list-item", "wait_timeout_ms": 15000, "headless": true, "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1" }, "headers": { "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Accept-Language": "zh-CN,zh;q=0.9" }, "parse_rules": { "format": "dom", "list_selector": ".hot-list-item", "item_selectors": { "rank": ".hot-index", "keyword": ".title", "heat_score": ".heat-index", "label": ".hot-label", "topic_tag": ".tag" }, "field_mapping": { "keyword": "textContent(.title)", "rank": "textContent(.hot-index)", "heat_score": "textContent(.heat-index)" } }, "rate_limit": { "requests_per_minute": 4, "min_interval_ms": 15000, "daily_limit": 200, "backoff_on_429": true }, "priority": 2, "status": "active", "last_verified": "2026-04-13", "auth_required": false, "data_schema": "standard_trend" }, { "id": "xiaohongshu_search", "name": "小红书搜索 - 关键词抓取", "platform": "xiaohongshu", "type": "web_scraper", "endpoint": "https://www.xiaohongshu.com/search_result", "method": "GET", "description": "小红书搜索结果,需 Playwright 登录态,采集生活方式/消费趋势/种草笔记", "render_required": true, "render_engine": "playwright", "render_options": { "wait_for_selector": ".note-item", "wait_timeout_ms": 20000, "headless": true, "login_required": true, "cookie_refresh_hours": 24 }, "params": { "keyword": "互动剧", "sort": "general", "source": "web_search" }, "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml", "Accept-Language": "zh-CN,zh;q=0.9", "Referer": "https://www.xiaohongshu.com/" }, "parse_rules": { "format": "dom", "list_selector": ".note-item, .__timeline-mainFeed > div", "item_selectors": { "title": ".title", "author": ".author-wrapper .name", "likes": ".like-wrapper .count", "favorites": ".collect-wrapper .count", "comments": ".comment-wrapper .count", "tags": ".tag-list" }, "field_mapping": { "keyword": "textContent(.title)", "author": "textContent(.author-wrapper .name)", "likes": "textContent(.like-wrapper .count)", "category": "textContent(.tag-list)" } }, "rate_limit": { "requests_per_minute": 6, "min_interval_ms": 10000, "daily_limit": 300, "backoff_on_429": true }, "priority": 3, "status": "active", "last_verified": "2026-04-13", "auth_required": true, "data_schema": "xiaohongshu_note" }, { "id": "baidu_index", "name": "百度指数 - 关键词趋势", "platform": "baidu", "type": "api", "endpoint": "https://index.baidu.com/api/SearchApi/vol", "method": "GET", "description": "百度搜索指数,需 BDUSS Cookie,适用于全网关键词趋势追踪", "cookie_required": ["BDUSS"], "params": { "word": "", "days": 30 }, "headers": { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "application/json", "Referer": "https://index.baidu.com/" }, "parse_rules": { "format": "json", "list_path": "data", "field_mapping": { "keyword": "word", "heat_score": "index", "date": "date" } }, "rate_limit": { "requests_per_minute": 2, "min_interval_ms": 30000, "daily_limit": 100 }, "priority": 3, "status": "active", "last_verified": "2026-04-13", "auth_required": true, "data_schema": "baidu_index" } ], "data_schema": { "standard_trend": { "required_fields": [ "trend_id", "platform", "board_type", "keyword", "rank", "heat_score", "source_url", "crawl_time" ], "optional_fields": [ "category", "label", "is_topic", "snippet", "author", "likes", "comments_count", "shares", "sentiment", "extra" ] }, "bilibili_video": { "required_fields": [ "trend_id", "platform", "board_type", "keyword", "rank", "heat_score", "author", "bvid", "source_url", "crawl_time" ], "optional_fields": [ "category", "likes", "coins", "favorites", "danmaku", "shares", "duration", "description", "extra" ] }, "zhihu_question": { "required_fields": [ "trend_id", "platform", "board_type", "keyword", "rank", "heat_score", "question_id", "source_url", "crawl_time" ], "optional_fields": [ "snippet", "topic", "answer_count", "follower_count", "comment_count", "created", "updated_time", "extra" ] }, "xiaohongshu_note": { "required_fields": [ "trend_id", "platform", "keyword", "author", "source_url", "crawl_time" ], "optional_fields": [ "rank", "heat_score", "likes", "favorites", "comments", "category", "tags", "extra" ] }, "baidu_index": { "required_fields": [ "trend_id", "platform", "keyword", "heat_score", "date", "source_url", "crawl_time" ], "optional_fields": [ "region", "extra" ] } }, "output_format": { "file_pattern": "{platform}_{source_id}_{YYYYMMDDHHmmss}.jsonl", "encoder": "jsonl", "append_timestamp": true, "max_file_size_mb": 10, "compress_after_hours": 24 } }