crawl_config.json 7.4 KB
{
  "config_version": "2.0",
  "skill_target": "interactive-video-selector",
  "crawl_policy": {
    "respect_robots_txt": true,
    "request_interval_seconds": 2,
    "max_retries": 3,
    "user_agent": "Mozilla/5.0 (compatible; LemonAI-Hunter/1.0; +https://lemonai.io/bot)"
  },
  "data_sources": [
    {
      "id": "youtube_trending_api",
      "name": "YouTube Data API v3 - 热门视频",
      "type": "api",
      "relevance": "获取带字幕的热门视频Metadata,作为互动剧选题的原始输入素材",
      "endpoints": [
        {
          "url": "https://www.googleapis.com/youtube/v3/videos",
          "method": "GET",
          "params": {
            "part": "snippet,contentDetails,statistics",
            "chart": "mostPopular",
            "regionCode": "US",
            "maxResults": 50,
            "videoCategoryId": "24"
          },
          "headers": {
            "Accept": "application/json"
          }
        },
        {
          "url": "https://www.googleapis.com/youtube/v3/captions",
          "method": "GET",
          "params": {
            "part": "snippet",
            "videoId": "{{videoId}}"
          },
          "note": "需配合 caption download endpoint 获取实际字幕文本"
        }
      ],
      "parse_rules": {
        "video_id": "items[].id",
        "title": "items[].snippet.title",
        "channel_title": "items[].snippet.channelTitle",
        "publish_time": "items[].snippet.publishedAt",
        "duration": "items[].contentDetails.duration",
        "view_count": "items[].statistics.viewCount",
        "like_count": "items[].statistics.likeCount",
        "tags": "items[].snippet.tags"
      },
      "rate_limit": {
        "requests_per_day": 10000,
        "note": "每日配额需通过 Google Cloud Console 申请"
      },
      "update_frequency": "hourly",
      "difficulty": "medium"
    },
    {
      "id": "tiktok_research_api",
      "name": "TikTok Research API - 短视频趋势",
      "type": "api",
      "relevance": "获取热门短视频、标签趋势、互动数据,捕捉短剧/互动剧热门题材和情绪热点",
      "endpoints": [
        {
          "url": "https://open-api.tiktok.com/research/video/query/",
          "method": "POST",
          "headers": {
            "Content-Type": "application/json",
            "Authorization": "Bearer {{access_token}}"
          },
          "body": {
            "query": {
              "and": [
                { "operation": "EQ", "field_name": "region_code", "field_values": ["US"] },
                { "operation": "IN", "field_name": "keyword", "field_values": ["short drama", "reelshort", "dramabox", "interactive story"] }
              ]
            },
            "start_date": "{{start_date}}",
            "end_date": "{{end_date}}",
            "max_count": 100
          }
        }
      ],
      "parse_rules": {
        "video_id": "data[].id",
        "title": "data[].title",
        "username": "data[].username",
        "like_count": "data[].like_count",
        "comment_count": "data[].comment_count",
        "share_count": "data[].share_count",
        "view_count": "data[].view_count",
        "hashtag_names": "data[].hashtag_names",
        "create_time": "data[].create_time"
      },
      "rate_limit": {
        "requests_per_day": 600,
        "note": "需申请 TikTok for Business 开发者账号"
      },
      "update_frequency": "daily",
      "difficulty": "high"
    },
    {
      "id": "douban_short_drama_topics",
      "name": "豆瓣 - 短剧/互动剧话题讨论",
      "type": "web_scraper",
      "relevance": "爬取豆瓣小组和话题中的短剧推荐、口碑分析、题材偏好讨论,补充用户侧的定性洞察",
      "endpoints": [
        {
          "url": "https://www.douban.com/group/search?cat=1013&q=短剧",
          "method": "GET",
          "headers": {
            "Accept": "text/html"
          }
        },
        {
          "url": "https://www.douban.com/group/search?cat=1013&q=互动剧",
          "method": "GET",
          "headers": {
            "Accept": "text/html"
          }
        }
      ],
      "parse_rules": {
        "post_title": "css:.title a",
        "post_url": "css:.title a@href",
        "post_summary": "css:.content",
        "reply_count": "css:.pl",
        "post_time": "css:.time"
      },
      "rate_limit": {
        "request_interval_seconds": 3,
        "note": "需遵守豆瓣 robots.txt,避免高频爬取"
      },
      "update_frequency": "daily",
      "difficulty": "medium"
    },
    {
      "id": "reddit_short_drama",
      "name": "Reddit r/ShortDrama & r/DramaBox - 海外用户讨论",
      "type": "api",
      "relevance": "获取海外短剧观众的真实反馈、爆款剧推荐、平台内容偏好",
      "endpoints": [
        {
          "url": "https://www.reddit.com/r/ShortDrama/hot.json",
          "method": "GET",
          "headers": {
            "Accept": "application/json",
            "User-Agent": "LemonAI-Hunter/1.0"
          }
        },
        {
          "url": "https://www.reddit.com/r/ReelShort/hot.json",
          "method": "GET",
          "headers": {
            "Accept": "application/json",
            "User-Agent": "LemonAI-Hunter/1.0"
          }
        }
      ],
      "parse_rules": {
        "post_title": "data.children[].data.title",
        "post_url": "data.children[].data.url",
        "permalink": "data.children[].data.permalink",
        "score": "data.children[].data.score",
        "num_comments": "data.children[].data.num_comments",
        "created_utc": "data.children[].data.created_utc",
        "selftext": "data.children[].data.selftext"
      },
      "rate_limit": {
        "requests_per_minute": 30,
        "note": "Reddit API 对未认证请求有限制"
      },
      "update_frequency": "daily",
      "difficulty": "low"
    },
    {
      "id": "zhihu_short_drama",
      "name": "知乎 - 短剧/互动剧选题与行业分析",
      "type": "web_scraper",
      "relevance": "获取中文互联网关于短剧爆款逻辑、互动剧发展前景、平台策略的专业分析",
      "endpoints": [
        {
          "url": "https://www.zhihu.com/search?type=content&q=短剧%20爆款",
          "method": "GET",
          "headers": {
            "Accept": "text/html",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
          }
        },
        {
          "url": "https://www.zhihu.com/search?type=content&q=互动剧%20选题",
          "method": "GET",
          "headers": {
            "Accept": "text/html",
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
          }
        }
      ],
      "parse_rules": {
        "question_title": "css:.ContentItem-title",
        "question_url": "css:.ContentItem-title a@href",
        "answer_excerpt": "css:.RichContent-inner",
        "vote_count": "css:.VoteButton--up"
      },
      "rate_limit": {
        "request_interval_seconds": 3,
        "note": "知乎反爬严格,建议配合登录态或降低频率"
      },
      "update_frequency": "weekly",
      "difficulty": "medium"
    }
  ],
  "output_schema": {
    "required_fields": [
      "source_url",
      "crawl_time",
      "title",
      "platform",
      "category_tag",
      "duration_seconds",
      "view_count",
      "subtitle_extract",
      "interactive_suitability_score",
      "ai_producibility_score",
      "grade",
      "recommendation"
    ]
  }
}