{
  "version": "1.0",
  "skill_name": "competitor-spy",
  "generated_at": "2026-04-09",
  "global_settings": {
    "request_interval_seconds": 2,
    "high_defense_interval_seconds": 5,
    "max_retries": 3,
    "timeout_seconds": 30,
    "user_agent_rotation": true,
    "respect_robots_txt": true,
    "proxy_recommended": true
  },
  "data_sources": [
    {
      "id": "crunchbase_companies",
      "name": "Crunchbase Organizations Search",
      "category": "funding_m&a",
      "base_url": "https://www.crunchbase.com",
      "api_endpoint": "https://api.crunchbase.com/api/v4/searches/organizations",
      "method": "api_reverse_engineered",
      "fields": [
        "identifier",
        "name",
        "short_description",
        "funding_total",
        "funding_round",
        "ipo_status",
        "num_employees_enum",
        "location_identifiers",
        "website_url",
        "linkedin_url",
        "created_at",
        "updated_at"
      ],
      "headers": {
        "Accept": "application/json",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
      },
      "schedule": "0 8 * * *",
      "anti_detection": {
        "note": "Crunchbase uses Cloudflare. Recommend Playwright with fingerprint rotation or proxy pool.",
        "suggested_tools": ["Playwright", "Crawlee", "Bright Data"]
      }
    },
    {
      "id": "g2_reviews",
      "name": "G2 Product Reviews",
      "category": "b2b_reviews",
      "base_url": "https://www.g2.com",
      "method": "browser_automation",
      "entry_url_template": "https://www.g2.com/products/{product_slug}/reviews",
      "fields": [
        "review_id",
        "reviewer_name",
        "reviewer_title",
        "company_size",
        "industry",
        "overall_score",
        "pros_text",
        "cons_text",
        "switching_from",
        "review_date",
        "verified_badge"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
      },
      "schedule": "0 9 * * 1",
      "anti_detection": {
        "note": "G2 heavily rate-limits scrapers. Use residential proxies and human-like scroll delays.",
        "delay_between_pages_ms": 5000,
        "suggested_tools": ["Playwright", "Browse AI"]
      }
    },
    {
      "id": "capterra_reviews",
      "name": "Capterra Product Reviews",
      "category": "b2b_reviews",
      "base_url": "https://www.capterra.com",
      "method": "browser_automation",
      "entry_url_template": "https://www.capterra.com/p/{product_id}/{product_slug}/reviews/",
      "fields": [
        "review_id",
        "reviewer_name",
        "company_size",
        "industry",
        "overall_rating",
        "ease_of_use",
        "customer_support",
        "value_for_money",
        "pros",
        "cons",
        "review_date"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
      },
      "schedule": "0 9 * * 3",
      "anti_detection": {
        "note": "Capterra uses basic bot detection. Rotate User-Agent and limit to 1 request per 3 seconds.",
        "delay_between_pages_ms": 3000,
        "suggested_tools": ["Playwright", "Scrapy"]
      }
    },
    {
      "id": "product_hunt_launches",
      "name": "Product Hunt Daily Launches",
      "category": "product_launches",
      "base_url": "https://www.producthunt.com",
      "api_endpoint": "https://www.producthunt.com/frontend/graphql",
      "method": "api_reverse_engineered",
      "fields": [
        "post_id",
        "name",
        "tagline",
        "description",
        "votes_count",
        "comments_count",
        "maker_names",
        "topics",
        "website_url",
        "launch_date"
      ],
      "headers": {
        "Accept": "application/json",
        "Content-Type": "application/json",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
      },
      "schedule": "0 10 * * *",
      "anti_detection": {
        "note": "Product Hunt GraphQL API is relatively open but requires correct headers and occasional re-auth tokens.",
        "suggested_tools": ["requests + custom headers", "Playwright"]
      }
    },
    {
      "id": "app_store_reviews_rss",
      "name": "Apple App Store Reviews (iTunes RSS)",
      "category": "app_reviews",
      "base_url": "https://itunes.apple.com",
      "api_endpoint": "https://itunes.apple.com/{country}/rss/customerreviews/id={app_id}/sortBy=mostRecent/json",
      "method": "rss_api",
      "fields": [
        "review_id",
        "author_name",
        "title",
        "content",
        "rating",
        "version",
        "review_date",
        "country"
      ],
      "headers": {
        "Accept": "application/json",
        "User-Agent": "iTunes/12.0"
      },
      "schedule": "0 */6 * * *",
      "anti_detection": {
        "note": "Official RSS endpoint. Low risk but rate-limited to ~20 req/min. No scraping needed.",
        "suggested_tools": ["requests", "Apify Apple App Store Scraper"]
      }
    },
    {
      "id": "google_play_reviews",
      "name": "Google Play Reviews (Scraping)",
      "category": "app_reviews",
      "base_url": "https://play.google.com",
      "method": "browser_automation",
      "entry_url_template": "https://play.google.com/store/apps/details?id={package_name}&hl={lang}&gl={country}",
      "fields": [
        "review_id",
        "author_name",
        "review_text",
        "star_rating",
        "review_date",
        "app_version",
        " helpful_count"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36"
      },
      "schedule": "0 */6 * * *",
      "anti_detection": {
        "note": "Google Play uses dynamic rendering. Requires headless browser with mobile viewport.",
        "suggested_tools": ["Playwright", "SerpApi"]
      }
    },
    {
      "id": "qimai_app_data",
      "name": "七麦数据 (Qimai) - Chinese App Intelligence",
      "category": "app_market_data",
      "base_url": "https://www.qimai.cn",
      "method": "browser_automation",
      "entry_url_template": "https://www.qimai.cn/app/rank/appid/{app_id}/country/cn",
      "fields": [
        "app_name",
        "rank_category",
        "rank_position",
        "download_estimate",
        "revenue_estimate",
        "rating",
        "review_count",
        "update_history"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Referer": "https://www.qimai.cn"
      },
      "schedule": "0 8,20 * * *",
      "anti_detection": {
        "note": "Qimai uses basic anti-bot and requires login for deep data. Use session cookies + proxy rotation.",
        "suggested_tools": ["Playwright with saved session", "Bright Data"]
      }
    },
    {
      "id": "similarweb_traffic",
      "name": "SimilarWeb Traffic Overview",
      "category": "web_traffic",
      "base_url": "https://www.similarweb.com",
      "method": "browser_automation",
      "entry_url_template": "https://www.similarweb.com/website/{domain}/",
      "fields": [
        "domain",
        "global_rank",
        "country_rank",
        "category_rank",
        "total_visits",
        "bounce_rate",
        "pages_per_visit",
        "avg_visit_duration",
        "top_countries",
        "traffic_sources"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
      },
      "schedule": "0 7 * * 1",
      "anti_detection": {
        "note": "SimilarWeb free tier is heavily gated. Enterprise API is the preferred route for scale.",
        "suggested_tools": ["Playwright", "SimilarWeb API"]
      }
    },
    {
      "id": "linkedin_jobs",
      "name": "LinkedIn Job Postings",
      "category": "hr_signals",
      "base_url": "https://www.linkedin.com",
      "method": "browser_automation",
      "entry_url_template": "https://www.linkedin.com/jobs/search?f_C={company_id}&keywords={keyword}",
      "fields": [
        "job_id",
        "title",
        "location",
        "posted_time",
        "company_name",
        "employment_type",
        "job_description_summary",
        "applicant_count"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
      },
      "schedule": "0 9 * * 1,4",
      "anti_detection": {
        "note": "LinkedIn aggressively blocks scrapers. Must use logged-in session with human-like delays (5-10s).",
        "delay_between_pages_ms": 8000,
        "suggested_tools": ["Playwright with real account", "Proxycurl API"]
      }
    },
    {
      "id": "owler_news",
      "name": "Owler Company News & Competitors",
      "category": "news_monitoring",
      "base_url": "https://www.owler.com",
      "method": "browser_automation",
      "entry_url_template": "https://www.owler.com/company/{company_slug}",
      "fields": [
        "company_name",
        "revenue_estimate",
        "employee_count",
        "ceo_name",
        "headquarters",
        "competitors",
        "recent_news_title",
        "recent_news_date",
        "recent_news_url"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
      },
      "schedule": "0 8 * * *",
      "anti_detection": {
        "note": "Owler has a free Community tier and moderate anti-bot. Proxies recommended for scale.",
        "suggested_tools": ["Playwright", "Browse AI"]
      }
    },
    {
      "id": "weibo_hot_search",
      "name": "微博热搜榜",
      "category": "social_trends",
      "base_url": "https://s.weibo.com",
      "method": "static_html",
      "entry_url": "https://s.weibo.com/top/summary?cate=realtimehot",
      "fields": [
        "rank",
        "keyword",
        "hot_value",
        "tag",
        "crawl_time"
      ],
      "headers": {
        "Accept": "text/html",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
      },
      "schedule": "0 */2 * * *",
      "anti_detection": {
        "note": "Static HTML, low defense. Rate limit to 1 req per 2 seconds.",
        "suggested_tools": ["requests + BeautifulSoup", "Scrapy"]
      }
    },
    {
      "id": "bilibili_ranking",
      "name": "B站排行榜 (全站综合热门)",
      "category": "content_trends",
      "base_url": "https://api.bilibili.com",
      "api_endpoint": "https://api.bilibili.com/x/web-interface/ranking/v2?rid=0&type=all",
      "method": "api_public",
      "fields": [
        "bvid",
        "title",
        "author",
        "view_count",
        "like_count",
        "coin_count",
        "share_count",
        "pub_date"
      ],
      "headers": {
        "Accept": "application/json",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Referer": "https://www.bilibili.com"
      },
      "schedule": "0 */4 * * *",
      "anti_detection": {
        "note": "Public API with Referer requirement. Low risk.",
        "suggested_tools": ["requests"]
      }
    },
    {
      "id": "news_api_google",
      "name": "NewsAPI / Google News (RSS)",
      "category": "news_monitoring",
      "base_url": "https://news.google.com",
      "method": "rss_feed",
      "entry_url_template": "https://news.google.com/rss/search?q={company_name}+when:1d",
      "fields": [
        "title",
        "link",
        "pubDate",
        "source_name",
        "description"
      ],
      "headers": {
        "Accept": "application/rss+xml",
        "User-Agent": "Mozilla/5.0"
      },
      "schedule": "0 8,14,20 * * *",
      "anti_detection": {
        "note": "RSS feed from Google News. Safe and stable but limited to ~100 results per query.",
        "suggested_tools": [ "feedparser", "requests" ]
      }
    }
  ],
  "output_schema": {
    "required_fields": [ "source_url", "crawl_time", "data_type" ],
    "recommended_fields": [ "company", "product", "metric_name", "metric_value", "raw_text", "language", "country" ]
  }
}