crawl_config.json 6.32 KB
{
  "version": "1.0",
  "agent": "trend-scout-global",
  "schedule": "*/10 * * * *",
  "timeout_ms": 30000,
  "sources": [
    {
      "id": "reddit_r_all_hot",
      "name": "Reddit r/all Hot",
      "platform": "reddit",
      "enabled": true,
      "method": "GET",
      "url": "https://www.reddit.com/r/all/hot.json?limit=100",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; TrendScoutBot/1.0)"
      },
      "rate_limit": {
        "requests_per_window": 100,
        "window_seconds": 600
      },
      "parse_rules": {
        "root_selector": "$.data.children[*].data",
        "fields": {
          "title": "title",
          "author": "author",
          "subreddit": "subreddit_name_prefixed",
          "score": "score",
          "upvote_ratio": "upvote_ratio",
          "num_comments": "num_comments",
          "url": "url",
          "permalink": "permalink",
          "created_utc": "created_utc",
          "thumbnail": "thumbnail",
          "is_video": "is_video"
        }
      },
      "category_inference": ["entertainment", "technology", "politics", "sports", "gaming", "memes"]
    },
    {
      "id": "reddit_r_all_rising",
      "name": "Reddit r/all Rising",
      "platform": "reddit",
      "enabled": true,
      "method": "GET",
      "url": "https://www.reddit.com/r/all/rising.json?limit=100",
      "headers": {
        "User-Agent": "Mozilla/5.0 (compatible; TrendScoutBot/1.0)"
      },
      "rate_limit": {
        "requests_per_window": 100,
        "window_seconds": 600
      },
      "parse_rules": {
        "root_selector": "$.data.children[*].data",
        "fields": {
          "title": "title",
          "author": "author",
          "subreddit": "subreddit_name_prefixed",
          "score": "score",
          "upvote_ratio": "upvote_ratio",
          "num_comments": "num_comments",
          "url": "url",
          "permalink": "permalink",
          "created_utc": "created_utc"
        }
      },
      "category_inference": ["entertainment", "technology", "politics", "sports", "gaming", "memes"]
    },
    {
      "id": "youtube_trending_invidious",
      "name": "YouTube Trending (Invidious)",
      "platform": "youtube",
      "enabled": true,
      "method": "GET",
      "url": "https://vid.puffyan.us/api/v1/trending",
      "headers": {
        "Accept": "application/json",
        "User-Agent": "Mozilla/5.0 (compatible; TrendScoutBot/1.0)"
      },
      "query_params": {
        "region": "US",
        "type": "default"
      },
      "rate_limit": {
        "requests_per_window": 60,
        "window_seconds": 600
      },
      "parse_rules": {
        "root_selector": "$[*]",
        "fields": {
          "title": "title",
          "video_id": "videoId",
          "author": "author",
          "author_id": "authorId",
          "published": "published",
          "view_count": "viewCount",
          "length_seconds": "lengthSeconds",
          "description": "description",
          "thumbnails": "videoThumbnails"
        }
      },
      "fallback_instances": [
        "https://y.com.sb/api/v1/trending",
        "https://iv.nboeck.de/api/v1/trending"
      ],
      "category_inference": ["entertainment", "music", "gaming", "movies", "technology"]
    },
    {
      "id": "google_trends_daily",
      "name": "Google Trends Daily",
      "platform": "google_trends",
      "enabled": true,
      "method": "pytrends",
      "keywords": ["breaking news", "viral", "meme", "tech", "celebrity"],
      "geo": "US",
      "timeframe": "now 1-d",
      "rate_limit": {
        "requests_per_window": 20,
        "window_seconds": 600,
        "sleep_on_429_seconds": 60
      },
      "parse_rules": {
        "root_selector": "trending_searches",
        "fields": {
          "title": "title",
          "formatted_traffic": "formattedTraffic",
          "related_queries": "relatedQueries"
        }
      },
      "category_inference": ["news", "entertainment", "technology", "politics", "memes"]
    },
    {
      "id": "newsapi_headlines",
      "name": "NewsAPI Top Headlines",
      "platform": "newsapi",
      "enabled": true,
      "method": "GET",
      "url": "https://newsapi.org/v2/top-headlines",
      "headers": {
        "Accept": "application/json"
      },
      "query_params": {
        "country": "us",
        "pageSize": 100
      },
      "api_key_env": "NEWSAPI_KEY",
      "rate_limit": {
        "requests_per_window": 100,
        "window_seconds": 86400
      },
      "parse_rules": {
        "root_selector": "$.articles[*]",
        "fields": {
          "title": "title",
          "description": "description",
          "url": "url",
          "author": "author",
          "source_name": "source.name",
          "published_at": "publishedAt",
          "url_to_image": "urlToImage"
        }
      },
      "category_inference": ["news", "politics", "technology", "business", "sports", "entertainment"]
    },
    {
      "id": "twitter_x_trends",
      "name": "Twitter/X Trends (twscrape)",
      "platform": "twitter",
      "enabled": false,
      "method": "twscrape_graphql",
      "categories": ["news", "sport"],
      "rate_limit": {
        "requests_per_window": 50,
        "window_seconds": 600
      },
      "parse_rules": {
        "root_selector": "trends",
        "fields": {
          "title": "name",
          "query": "query",
          "url": "url",
          "tweet_volume": "tweet_volume"
        }
      },
      "notes": "Requires authenticated account pool. Enable only when credentials are configured.",
      "category_inference": ["politics", "entertainment", "sports", "technology", "memes"]
    },
    {
      "id": "tiktok_creative_center",
      "name": "TikTok Creative Center Trends",
      "platform": "tiktok",
      "enabled": false,
      "method": "apify_actor",
      "actor_id": "doliz/tiktok-creative-center-scraper",
      "rate_limit": {
        "requests_per_window": 10,
        "window_seconds": 3600
      },
      "parse_rules": {
        "root_selector": "dataset_items",
        "fields": {
          "title": "hashtag",
          "trend_type": "trend_type",
          "video_count": "video_count",
          "view_count": "view_count",
          "region": "region"
        }
      },
      "notes": "Requires Apify API token. Enable only when token is available.",
      "category_inference": ["entertainment", "music", "gaming", "memes", "fashion"]
    }
  ]
}