{
  "project": "digester-benchmark-hunt",
  "version": "1.0.0",
  "crawl_time": "2026-04-10T00:00:00Z",
  "rate_limit": {
    "requests_per_second": 0.5,
    "delay_between_requests_ms": 2000,
    "max_retries": 3,
    "retry_backoff_ms": 1000
  },
  "output_schema": {
    "required_fields": [
      "id",
      "category",
      "title",
      "description",
      "source_url",
      "crawl_time",
      "tags"
    ],
    "optional_fields": [
      "reliability",
      "difficulty",
      "language"
    ]
  },
  "data_sources": [
    {
      "name": "arXiv API",
      "type": "api",
      "description": "学术论文搜索，重点爬取 benchmark design、dataset construction、evaluation methodology 相关论文",
      "base_url": "http://export.arxiv.org/api/query",
      "headers": {
        "User-Agent": "DigesterBot/1.0 (contact@example.com)"
      },
      "endpoints": [
        {
          "name": "benchmark_papers",
          "url": "http://export.arxiv.org/api/query?search_query=all:benchmark+design+AND+cat:cs.LG&start=0&max_results=50&sortBy=submittedDate&sortOrder=descending",
          "method": "GET"
        },
        {
          "name": "dataset_construction_papers",
          "url": "http://export.arxiv.org/api/query?search_query=all:dataset+construction+AND+cat:cs.CL&start=0&max_results=50&sortBy=submittedDate&sortOrder=descending",
          "method": "GET"
        },
        {
          "name": "evaluation_metrics_papers",
          "url": "http://export.arxiv.org/api/query?search_query=all:evaluation+metrics+AND+cat:cs.AI&start=0&max_results=50&sortBy=submittedDate&sortOrder=descending",
          "method": "GET"
        }
      ],
      "parsing_rules": {
        "root_selector": "feed.entry",
        "fields": {
          "id": "id",
          "title": "title",
          "summary": "summary",
          "published": "published",
          "authors": "author.name",
          "primary_category": "primary_category",
          "pdf_link": "link[rel='alternate'][type='text/html']",
          "source_url": "id"
        }
      },
      "respect_robots_txt": true,
      "frequency_limit": "2s"
    },
    {
      "name": "Papers With Code API",
      "type": "api",
      "description": "获取 SOTA benchmark 结果、数据集和评估指标信息",
      "base_url": "https://paperswithcode.com/api/v1",
      "headers": {
        "User-Agent": "DigesterBot/1.0",
        "Accept": "application/json"
      },
      "endpoints": [
        {
          "name": "benchmarks_list",
          "url": "https://paperswithcode.com/api/v1/benchmarks/?page=1&items_per_page=50",
          "method": "GET"
        },
        {
          "name": "areas_list",
          "url": "https://paperswithcode.com/api/v1/areas/?page=1&items_per_page=50",
          "method": "GET"
        },
        {
          "name": "datasets_list",
          "url": "https://paperswithcode.com/api/v1/datasets/?page=1&items_per_page=50",
          "method": "GET"
        }
      ],
      "parsing_rules": {
        "root_selector": "results",
        "fields": {
          "id": "id",
          "title": "name",
          "description": "description",
          "source_url": "url",
          "created_at": "created_at"
        }
      },
      "respect_robots_txt": true,
      "frequency_limit": "2s"
    },
    {
      "name": "Hugging Face Datasets API",
      "type": "api",
      "description": "开源数据集元数据，特别是 evaluation、benchmark、text-classification 类数据集",
      "base_url": "https://huggingface.co/api/datasets",
      "headers": {
        "User-Agent": "DigesterBot/1.0",
        "Accept": "application/json"
      },
      "endpoints": [
        {
          "name": "benchmark_datasets",
          "url": "https://huggingface.co/api/datasets?filter=task_categories:question-answering&limit=50&full=false&config=true",
          "method": "GET"
        },
        {
          "name": "evaluation_datasets",
          "url": "https://huggingface.co/api/datasets?filter=task_categories:text-classification&limit=50&full=false&config=true",
          "method": "GET"
        }
      ],
      "parsing_rules": {
        "root_selector": "",
        "fields": {
          "id": "id",
          "title": "id",
          "description": "description",
          "downloads": "downloads",
          "tags": "tags",
          "source_url": "id"
        }
      },
      "respect_robots_txt": true,
      "frequency_limit": "2s"
    },
    {
      "name": "Semantic Scholar API",
      "type": "api",
      "description": "学术研究搜索，获取 benchmark 相关论文的引用、摘要、领域分类",
      "base_url": "https://api.semanticscholar.org/graph/v1",
      "headers": {
        "User-Agent": "DigesterBot/1.0",
        "Accept": "application/json"
      },
      "endpoints": [
        {
          "name": "paper_search_benchmark",
          "url": "https://api.semanticscholar.org/graph/v1/paper/search?query=benchmark+design+machine+learning&fields=paperId,title,abstract,year,citationCount,referenceCount,fieldsOfStudy,publicationDate,journal,authors&limit=50",
          "method": "GET"
        },
        {
          "name": "paper_search_evaluation",
          "url": "https://api.semanticscholar.org/graph/v1/paper/search?query=data+annotation+quality+control&fields=paperId,title,abstract,year,citationCount,referenceCount,fieldsOfStudy,publicationDate,journal,authors&limit=50",
          "method": "GET"
        }
      ],
      "parsing_rules": {
        "root_selector": "data",
        "fields": {
          "id": "paperId",
          "title": "title",
          "description": "abstract",
          "year": "year",
          "citation_count": "citationCount",
          "fields_of_study": "fieldsOfStudy",
          "source_url": "paperId"
        }
      },
      "respect_robots_txt": true,
      "frequency_limit": "2s"
    },
    {
      "name": "GitHub API",
      "type": "api",
      "description": "搜索开源 benchmark 框架和 evaluation toolkit 仓库",
      "base_url": "https://api.github.com",
      "headers": {
        "User-Agent": "DigesterBot/1.0",
        "Accept": "application/vnd.github.v3+json"
      },
      "endpoints": [
        {
          "name": "llm_benchmark_repos",
          "url": "https://api.github.com/search/repositories?q=llm+benchmark+evaluation+language:python&sort=stars&order=desc&per_page=50",
          "method": "GET"
        },
        {
          "name": "dataset_annotation_repos",
          "url": "https://api.github.com/search/repositories?q=dataset+annotation+toolkit+language:python&sort=stars&order=desc&per_page=50",
          "method": "GET"
        }
      ],
      "parsing_rules": {
        "root_selector": "items",
        "fields": {
          "id": "id",
          "title": "full_name",
          "description": "description",
          "stars": "stargazers_count",
          "language": "language",
          "updated_at": "updated_at",
          "source_url": "html_url",
          "topics": "topics"
        }
      },
      "respect_robots_txt": true,
      "frequency_limit": "2s"
    },
    {
      "name": "Scale AI Labs Leaderboard",
      "type": "web",
      "description": "企业级 Agent benchmark 与评估方法公开页面",
      "base_url": "https://labs.scale.com",
      "headers": {
        "User-Agent": "DigesterBot/1.0"
      },
      "endpoints": [
        {
          "name": "scale_leaderboard",
          "url": "https://labs.scale.com/leaderboard/tool_use_enterprise",
          "method": "GET"
        }
      ],
      "parsing_rules": {
        "root_selector": "body",
        "fields": {
          "title": "title",
          "content": "meta[description]",
          "source_url": "url"
        }
      },
      "respect_robots_txt": true,
      "frequency_limit": "3s"
    }
  ]
}