宜博 / lemonboss

[
  {
    "id": "seed_001",
    "category": "benchmark_framework",
    "title": "HELM - Holistic Evaluation of Language Models",
    "description": "Stanford CRFM 开发的整体评估框架，覆盖16个核心场景和7大指标类别（准确率、校准、鲁棒性、公平性、偏见、毒性、效率）。",
    "source_url": "https://crfm.stanford.edu/helm/lite/latest/#/leaderboard",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["HELM", "LLM", "holistic-evaluation", "stanford"],
    "reliability": "A"
  },
  {
    "id": "seed_002",
    "category": "benchmark_framework",
    "title": "EleutherAI LM Evaluation Harness",
    "description": "学术界最广泛使用的开源 benchmark 框架，集成60+任务（MMLU、BigBench、HellaSwag等），支撑 Hugging Face Open LLM Leaderboard。",
    "source_url": "https://github.com/EleutherAI/lm-evaluation-harness",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["EleutherAI", "lm-evaluation-harness", "open-source", "python"],
    "reliability": "A"
  },
  {
    "id": "seed_003",
    "category": "benchmark_framework",
    "title": "DeepEval - Pytest for LLMs",
    "description": "现代开源 LLM 评估框架，支持 MMLU、BIG-Bench Hard、GSM8K、HumanEval 等 benchmark 的10行代码快速集成，并提供 CI/CD、Red Teaming 功能。",
    "source_url": "https://github.com/confident-ai/deepeval",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["DeepEval", "CI-CD", "LLM-evaluation", "red-teaming"],
    "reliability": "A"
  },
  {
    "id": "seed_004",
    "category": "benchmark_framework",
    "title": "OpenAI Evals Framework",
    "description": "OpenAI 开源的评估框架，采用 Registry 格式，支持自定义任务扩展，注重结构化的 benchmark 定义。",
    "source_url": "https://github.com/openai/evals",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["OpenAI", "evals", "registry", "custom-tasks"],
    "reliability": "A"
  },
  {
    "id": "seed_005",
    "category": "benchmark_framework",
    "title": "OpenCompass 评估体系",
    "description": "上海人工智能实验室推出的开源统一评估平台，覆盖大量中英文 benchmark，特别适用于中文大模型评测。",
    "source_url": "https://opencompass.org.cn/",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["OpenCompass", "InternLM", "chinese-LLM", "evaluation-platform"],
    "reliability": "A"
  },
  {
    "id": "seed_006",
    "category": "benchmark_framework",
    "title": "RAGAS - RAG Evaluation Framework",
    "description": "专为检索增强生成（RAG）设计的评估框架，核心指标包括 Faithfulness、Answer Relevancy、Context Relevancy、Context Recall。",
    "source_url": "https://github.com/explodinggradients/ragas",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["RAGAS", "RAG", "retrieval", "LLM-as-judge"],
    "reliability": "A"
  },
  {
    "id": "seed_007",
    "category": "benchmark_framework",
    "title": "BigBench / BigBench-Hard (BBH)",
    "description": "Google DeepMind 领衔的大规模协作 benchmark，BigBench 包含数百个多样化的任务；BBH 精选23个复杂推理任务，专门用于测试 LLM 的推理极限。",
    "source_url": "https://github.com/google/BIG-bench",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["BigBench", "BBH", "reasoning", "Google"],
    "reliability": "A"
  },
  {
    "id": "seed_008",
    "category": "benchmark_framework",
    "title": "LLM-KG-Bench 3.0",
    "description": "面向语义技术与知识图谱工程能力的 benchmark，评估 LLM 生成 RDF/SPARQL 的语法与语义正确性。",
    "source_url": "https://arxiv.org/html/2505.13098v1",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["LLM-KG-Bench", "knowledge-graph", "SPARQL", "semantic-web"],
    "reliability": "A"
  },
  {
    "id": "seed_009",
    "category": "benchmark_framework",
    "title": "SimBA - Simplifying Benchmark Analysis",
    "description": "基于性能矩阵的元分析框架，能从 HELM、MMLU、BigBenchLite 中发现最小代表性数据集子集，研究表明1.7%-28%的数据集可覆盖95%以上的方差。",
    "source_url": "https://arxiv.org/abs/2510.17998",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["SimBA", "meta-analysis", "dataset-reduction", "HELM"],
    "reliability": "A"
  },
  {
    "id": "seed_010",
    "category": "benchmark_framework",
    "title": "Giskard - ML Model Testing",
    "description": "开源的 ML 模型测试和 RAG 评估工具，提供偏见、稳健性、数据泄露等自动化检测。",
    "source_url": "https://github.com/Giskard-AI/giskard",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["Giskard", "ML-testing", "bias-detection", "robustness"],
    "reliability": "B"
  },
  {
    "id": "seed_011",
    "category": "benchmark_framework",
    "title": "MLflow Benchmarking Orchestrator",
    "description": "模块化端到端 ML benchmark 框架，支持 YAML 配置、并发流水线、执行图构建（支持 RAG 等复杂系统的 fork 和 cycle），并与 MLflow 集成记录实验。",
    "source_url": "https://robertbayer.github.io/publication/towards-modular-benchmark/towards-modular-benchmark.pdf",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["MLflow", "orchestrator", "modular-benchmark", "YAML"],
    "reliability": "A"
  },
  {
    "id": "seed_012",
    "category": "benchmark_framework",
    "title": "Arize Phoenix / Langfuse / TruLens",
    "description": "可观测性优先的 LLM 评估框架组合，用于生产环境的 trace、eval 和 LLM-as-a-judge 评分。",
    "source_url": "https://github.com/Arize-ai/phoenix",
    "crawl_time": "2026-04-10T08:00:00Z",
    "tags": ["Arize-Phoenix", "Langfuse", "TruLens", "observability"],
    "reliability": "B"
  },
  {
    "id": "seed_013",
    "category": "benchmark_paper",
    "title": "MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark",
    "description": "MMLU 的增强版本，将选项从4个增加到10个，问题更复杂，降低猜测敏感性和 prompt 敏感性。",
    "source_url": "https://arxiv.org/html/2406.01574v4",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["MMLU-Pro", "benchmark", "multi-task", "QA"],
    "reliability": "A"
  },
  {
    "id": "seed_014",
    "category": "benchmark_paper",
    "title": "Automating Benchmark Design (BeTaL)",
    "description": "提出使用 LLM-in-the-Loop 自动优化 benchmark 参数的方法，将 benchmark 创建视为可配置的优化问题，能自动调整难度（目标偏差5%-13%）。",
    "source_url": "https://arxiv.org/html/2510.25039v1",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["BeTaL", "automated-benchmark", "LLM-designer", "dynamic-benchmark"],
    "reliability": "A"
  },
  {
    "id": "seed_015",
    "category": "benchmark_paper",
    "title": "ReportBench: Evaluating Deep Research Agents",
    "description": "从678篇专家撰写的综述论文中自动构建 benchmark，通过解析 LaTeX 源文件提取引用和参考文献，用于评估 AI 生成研究报告的检索精度和事实一致性。",
    "source_url": "https://arxiv.org/pdf/2508.15804",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["ReportBench", "research-agents", "LaTeX-parsing", "factuality"],
    "reliability": "A"
  },
  {
    "id": "seed_016",
    "category": "benchmark_paper",
    "title": "TaxoBench-CS: Taxonomy Generation in Computer Science",
    "description": "从156篇 CS 综述论文的作者 curated 分类树构建 benchmark，平均74.4篇引用论文、3.1层深度。",
    "source_url": "https://arxiv.org/pdf/2509.19125",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["TaxoBench-CS", "taxonomy", "computer-science", "literature"],
    "reliability": "A"
  },
  {
    "id": "seed_017",
    "category": "benchmark_paper",
    "title": "Rethinking Domain-Specific LLM Benchmark Construction",
    "description": "关于如何为特定领域（法律、医学等）设计和评估 LLM benchmark 的方法论研究。",
    "source_url": "https://arxiv.org/html/2508.07353v1",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["domain-specific", "LLM", "benchmark-design", "methodology"],
    "reliability": "A"
  },
  {
    "id": "seed_018",
    "category": "benchmark_paper",
    "title": "DeepScholar-bench: Generative Research Synthesis",
    "description": "活的 benchmark，用于评估 Related Work 部分的生成质量，通过自动筛选 arXiv 论文评估知识综合、检索质量和可验证性。",
    "source_url": "https://arxiv.org/pdf/2508.20033",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["DeepScholar", "research-synthesis", "related-work", "generative"],
    "reliability": "A"
  },
  {
    "id": "seed_019",
    "category": "benchmark_paper",
    "title": "Evaluation Metrics and Statistical Tests for Machine Learning",
    "description": "Nature Scientific Reports 发表的论文，系统讨论 ML 评估指标和统计检验，强调单测试集 t 检验会低估方差，推荐使用非参数检验。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["statistics", "evaluation-metrics", "nature", "non-parametric"],
    "reliability": "A"
  },
  {
    "id": "seed_020",
    "category": "benchmark_paper",
    "title": "Vendi Score: A Diversity Metric for ML Datasets",
    "description": "提出基于成对相似度矩阵 von Neumann 熵的数据集多样性通用指标，适用于图像、文本、分子等多领域。",
    "source_url": "https://arxiv.org/pdf/2210.02410",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["Vendi-Score", "diversity", "dataset-quality", "entropy"],
    "reliability": "A"
  },
  {
    "id": "seed_021",
    "category": "benchmark_paper",
    "title": "MAAB: Multimodal AutoML Agent Benchmark",
    "description": "评估 agent 处理原始多模态数据的 benchmark，覆盖7种问题类型、5种数据模态，共25个来自 Kaggle/UCI/BEIR 的数据集。",
    "source_url": "https://openreview.net/pdf?id=9t7gfYUp0U",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["MAAB", "AutoML", "multimodal", "agent"],
    "reliability": "A"
  },
  {
    "id": "seed_022",
    "category": "benchmark_paper",
    "title": "Datasets, Metrics, Benchmarks and Future Research in Autonomous Driving",
    "description": "IEEE/CAA JAS 综述，全面回顾自动驾驶领域的数据集构建、评估指标和 benchmark 演进。",
    "source_url": "https://www.ieee-jas.net/en/article/doi/10.1109/JAS.2025.125957",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["autonomous-driving", "survey", "datasets", "metrics"],
    "reliability": "A"
  },
  {
    "id": "seed_023",
    "category": "benchmark_paper",
    "title": "Design Principles for Benchmark Evaluating Multi-agent AI Security Operations",
    "description": "明确讨论 multi-agent AI 系统安全运营 benchmark 的构建设计原则。",
    "source_url": "https://arxiv.org/abs/2603.28998",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["multi-agent", "security", "benchmark-design", "principles"],
    "reliability": "A"
  },
  {
    "id": "seed_024",
    "category": "benchmark_paper",
    "title": "PCMDE: Physics-Guided Benchmarking for Multimodal Synthetic Images",
    "description": "结合物理规则与多模态融合的合成图像评估框架，通过目标级检测、空间/关系规则和轻量 LLM 推理评估结构真实性。",
    "source_url": "https://arxiv.org/html/2511.15204v2",
    "crawl_time": "2026-04-10T08:05:00Z",
    "tags": ["PCMDE", "multimodal", "synthetic-images", "physics-guided"],
    "reliability": "A"
  },
  {
    "id": "seed_025",
    "category": "annotation_standard",
    "title": "Scale AI Quality Control Pipelines",
    "description": "Scale AI 提供三种质量流水线：Standard（一次审核）、Consensus（三次审核决出最终结果）、Collection（收集全部标注员响应）。",
    "source_url": "https://labelyourdata.com/articles/scale-ai-review",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["Scale-AI", "quality-pipeline", "consensus", "enterprise"],
    "reliability": "A"
  },
  {
    "id": "seed_026",
    "category": "annotation_standard",
    "title": "Appen AI Data Quality Framework",
    "description": "Appen 使用全球众包平台 ADAP，核心 QC 包括 Test Questions、Redundancy、Contributor Tiers、Inter-Rater Agreements 和 ISO 9001 合规。",
    "source_url": "https://www.appen.com/ai-data-quality",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["Appen", "ADAP", "crowd", "ISO-9001", "IRA"],
    "reliability": "A"
  },
  {
    "id": "seed_027",
    "category": "annotation_standard",
    "title": "Labelbox Ontology Management & Automated QC",
    "description": "Labelbox 通过本体管理（Ontology Management）确保标签一致性，提供内置 Benchmark & Consensus 自动化测试，支持 SDK/API 集成。",
    "source_url": "https://docs.labelbox.com/",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["Labelbox", "ontology", "automated-QC", "API-first"],
    "reliability": "A"
  },
  {
    "id": "seed_028",
    "category": "annotation_standard",
    "title": "YOLO Dataset & Annotation Best Practices Guide 2024",
    "description": "计算机视觉标注最佳实践：锁定 taxonomy、使用紧凑轴对齐框、标注遮挡物、维持类别平衡（每类≥200实例）、冻结数据集划分。",
    "source_url": "https://imgmarker.com/blog/yolo-dataset-annotation-best-practices",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["YOLO", "computer-vision", "bounding-box", "best-practices"],
    "reliability": "A"
  },
  {
    "id": "seed_029",
    "category": "annotation_standard",
    "title": "Medical Annotation Best Practices 2024",
    "description": "医疗数据标注标准化流程：统一图像尺寸、精确识别类别标签、匿名化 PII、加密通信、确保 HIPAA/GDPR 合规。",
    "source_url": "https://research.aimultiple.com/medical-annotation/",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["medical-annotation", "HIPAA", "GDPR", "privacy"],
    "reliability": "A"
  },
  {
    "id": "seed_030",
    "category": "annotation_standard",
    "title": "Label Your Data - Guide to Inhouse Dataset Labeling 2024",
    "description": "内部数据集标注指南：收集要求与 golden benchmark、培训标注员、试点标注、多级验证、随机抽样与交叉验证 QA。",
    "source_url": "https://labelyourdata.com/pdf/Guide%20to%20Inhouse%20Dataset%20Labeling.pdf",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["guide", "inhouse-labeling", "pilot", "multi-level-QA"],
    "reliability": "A"
  },
  {
    "id": "seed_031",
    "category": "annotation_standard",
    "title": "Data Annotation Best Practices for Successful ML",
    "description": "强调将数据标注视为严格的迭代 QA 流程，包含视觉示例、术语表、golden dataset 交叉引用和持续更新的 Living Guidelines。",
    "source_url": "https://keylabs.ai/blog/data-annotation-best-practices-for-successful-machine-learning/",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["best-practices", "QA", "living-guidelines", "glossary"],
    "reliability": "A"
  },
  {
    "id": "seed_032",
    "category": "annotation_standard",
    "title": "Appen Quality Assurance in AI Blog",
    "description": "Appen 的 AI 质量保证博客，介绍大型 benchmark 项目经验（50万+标注）以及 contributor level 和 inter-rater agreement 的实施细节。",
    "source_url": "https://www.appen.com/blog/quality-assurance-in-ai",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["Appen", "QA", "benchmark-project", "500K-annotations"],
    "reliability": "A"
  },
  {
    "id": "seed_033",
    "category": "annotation_standard",
    "title": "Human-in-the-Loop Data Annotation",
    "description": "人机协同标注方法：自动化处理简单样本，将低置信度和边界案例路由给专家，降低成本并提高数据集多样性。",
    "source_url": "https://xonique.dev/blog/data-annotation-why-it-matters-and-best-practices/",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["HITL", "human-in-the-loop", "active-learning", "diversity"],
    "reliability": "B"
  },
  {
    "id": "seed_034",
    "category": "annotation_standard",
    "title": "Computer Vision for Plant Phenotyping - Annotation Best Practices",
    "description": "生物学表型数据注释：在最少的图像数量中最大化样本多样性，强调训练数据中对象实例数量比原始图像数量更重要。",
    "source_url": "https://pdfs.semanticscholar.org/46da/25493f3771ca395349b42d86cf67c6aa8b99.pdf",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["plant-phenotyping", "diversity", "instance-count", "biology"],
    "reliability": "A"
  },
  {
    "id": "seed_035",
    "category": "annotation_standard",
    "title": "AI Standards for Data Annotation - Restackio",
    "description": "汇总 AI 驱动数据标注的行业标准，涵盖质量控制流程、一致性协议和自动化验证方法。",
    "source_url": "https://www.restack.io/p/ai-driven-data-labeling-answer-ai-standards-cat-ai",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["AI-standards", "data-labeling", "automation", "quality-control"],
    "reliability": "B"
  },
  {
    "id": "seed_036",
    "category": "annotation_standard",
    "title": "Datasaur - Krippendorff's Alpha IAA Calculation Guide",
    "description": "Datasaur 发布的 Krippendorff's Alpha 与 Cohen's Kappa 对比指南，解释何时使用何种 IAA 指标。",
    "source_url": "https://datasaur.ai/blog-posts/inter-annotator-agreement-krippendorff-cohen",
    "crawl_time": "2026-04-10T08:10:00Z",
    "tags": ["Datasaur", "Krippendorff-Alpha", "Cohen-Kappa", "IAA"],
    "reliability": "A"
  },
  {
    "id": "seed_037",
    "category": "scoring_rubric",
    "title": "LLM Output Quality Analytic Rubric Template",
    "description": "推荐的 LLM 输出分析型评分量表模板，包含准确性（30%）、相关性（25%）、连贯性（20%）、完整性（15%）、安全性（10%）五个维度。",
    "source_url": "https://openreview.net/pdf?id=rKOJ59LuyI",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["rubric", "analytic", "LLM-evaluation", "weighted-scoring"],
    "reliability": "A"
  },
  {
    "id": "seed_038",
    "category": "scoring_rubric",
    "title": "EduVisBench Pedagogical Visualization Rubric",
    "description": "教育可视化 benchmark 的5维评分标准：Context Visualization、Diagram Design、Text-Graphic Integration、Thought Guidance、Interactivity。",
    "source_url": "https://arxiv.org/pdf/2505.16832",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["EduVisBench", "visualization", "pedagogy", "GPT-4o-evaluation"],
    "reliability": "A"
  },
  {
    "id": "seed_039",
    "category": "scoring_rubric",
    "title": "ProfBench - Professional LLM Rubrics",
    "description": "专业领域 LLM 任务评估 rubric，采用领域特定维度，以任务重要性加权，评估提取召回率和报告生成质量。",
    "source_url": "https://www.emergentmind.com/articles/2510.18941",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["ProfBench", "professional-tasks", "rubric", "importance-weighted"],
    "reliability": "A"
  },
  {
    "id": "seed_040",
    "category": "scoring_rubric",
    "title": "Xilankapu Design Evaluation AHP Weights",
    "description": "使用层次分析法（AHP）确定的文化遗产设计评估权重：Innovation 0.30、Cultural Relevance 0.30、Artistic Quality 0.20、Practicality 0.20，CR=0.036。",
    "source_url": "http://apjcriweb.org/content/vol11no5/26.pdf",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["AHP", "cultural-heritage", "weight-allocation", "consistency-ratio"],
    "reliability": "A"
  },
  {
    "id": "seed_041",
    "category": "scoring_rubric",
    "title": "Teacher Hiring Criteria Point System",
    "description": "菲律宾教师 hiring 的7维评分体系示例：Content Knowledge、Learning Environment、Diversity、Curriculum Planning、Assessment、Community、Personal Growth。",
    "source_url": "https://www.education.gov.pg/syllabus/secondarysyllabus/teacherguide/Business%20Studies%20Grade%209%20-%20TG.pdf",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["hiring-rubric", "teacher-evaluation", "point-system", "7-dimensions"],
    "reliability": "B"
  },
  {
    "id": "seed_042",
    "category": "scoring_rubric",
    "title": "Rubric Best Practices - NCSU DELTA",
    "description": "北卡州立大学教学资源中心总结的 rubric 设计最佳实践，区分 Holistic、Analytic、Single-Point 三种量表类型及其适用场景。",
    "source_url": "https://teaching-resources.delta.ncsu.edu/rubric_best-practices-examples-templates/",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["NCSU", "rubric-types", "holistic", "analytic", "teaching"],
    "reliability": "A"
  },
  {
    "id": "seed_043",
    "category": "scoring_rubric",
    "title": "12-Step Rubric Development Process",
    "description": "PNG 教育部提出的12步 rubric 开发流程：从列出任务组件、确定 benchmark 目的、拆分次要任务、提取 KSAV 到分配和验证权重。",
    "source_url": "https://www.education.gov.pg/syllabus/secondarysyllabus/teacherguide/Business%20Studies%20Grade%209%20-%20TG.pdf",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["rubric-development", "12-step", "KSAV", "validation"],
    "reliability": "B"
  },
  {
    "id": "seed_044",
    "category": "scoring_rubric",
    "title": "Turnitin - Types of Rubrics and Grading Forms",
    "description": "Turnitin 系统的 rubric 类型说明：百分比加权（Differential Weighting）要求各维度权重之和严格等于100%。",
    "source_url": "https://guides.turnitin.com/hc/en-us/articles/35522022970765--New-Types-of-rubrics-and-grading-forms",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["Turnitin", "percentage-weighting", "grading-forms", "sum-100"],
    "reliability": "A"
  },
  {
    "id": "seed_045",
    "category": "scoring_rubric",
    "title": "OpenReview LLM Evaluation Rubric Instructions",
    "description": "OpenReview 对 LLM 评估的严格要求：维度必须与问题强耦合、至少一个维度得分≤8、评分需提供直接引用。",
    "source_url": "https://openreview.net/pdf?id=rKOJ59LuyI",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["OpenReview", "strict-rules", "evidence-based", "ceiling-prevention"],
    "reliability": "A"
  },
  {
    "id": "seed_046",
    "category": "scoring_rubric",
    "title": "Sotopia Evaluation Dimension Concepts",
    "description": "Sotopia 多智能体社交模拟平台的评估维度概念文档，定义评估维度的设计哲学。",
    "source_url": "https://docs.sotopia.world/concepts/evaluation_dimension",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["Sotopia", "multi-agent", "social-simulation", "evaluation-dimension"],
    "reliability": "B"
  },
  {
    "id": "seed_047",
    "category": "scoring_rubric",
    "title": "How to Weight Rubrics - University of Northern Iowa",
    "description": " rubric 权重分配系列教程的第四部分，系统讲解如何为分析型量表分配权重。",
    "source_url": "https://chfasoa.uni.edu/howtoweightrubrics.pdf",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["rubric-weighting", "University-of-Northern-Iowa", "analytic-rubric"],
    "reliability": "A"
  },
  {
    "id": "seed_048",
    "category": "scoring_rubric",
    "title": "Designing Scoring Rubrics for Your Classroom - Learner.org",
    "description": "系统介绍 Holistic、Analytic、Single-Point 三种 rubric 架构，推荐复杂任务使用 Analytic Rubric 以支持差分权重。",
    "source_url": "https://www.learner.org/wp-content/uploads/2019/02/s7_rubrics.pdf",
    "crawl_time": "2026-04-10T08:15:00Z",
    "tags": ["Learner.org", "scoring-rubrics", "classroom", "analytic"],
    "reliability": "A"
  },
  {
    "id": "seed_049",
    "category": "evaluation_metric",
    "title": "Microsoft LLM Evaluation Metrics Guide",
    "description": "微软 AI Playbook 中系统总结的 LLM 评估指标列表，涵盖 RAGAS 指标（Faithfulness、Answer Relevancy、Context Relevancy、Context Recall）等。",
    "source_url": "https://learn.microsoft.com/en-us/ai/playbook/technology-guidance/generative-ai/working-with-llms/evaluation/list-of-eval-metrics",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["Microsoft", "LLM-evaluation", "RAGAS", "metrics-guide"],
    "reliability": "A"
  },
  {
    "id": "seed_050",
    "category": "evaluation_metric",
    "title": "Standard Classification Metrics",
    "description": "标准分类评估指标：Confusion Matrix、Accuracy、Precision、Recall、F1-Score、F0.5-Score、AUC-ROC、Log Loss。",
    "source_url": "https://www.analyticsvidhya.com/blog/2019/08/11-important-model-evaluation-error-metrics/",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["classification", "F1", "AUC-ROC", "confusion-matrix"],
    "reliability": "A"
  },
  {
    "id": "seed_051",
    "category": "evaluation_metric",
    "title": "Standard Regression Metrics",
    "description": "标准回归评估指标：MAE、MSE、RMSE、RMSLE、R²、Adjusted R²。",
    "source_url": "https://www.geeksforgeeks.org/machine-learning/metrics-for-machine-learning-model/",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["regression", "MAE", "RMSE", "R-squared"],
    "reliability": "A"
  },
  {
    "id": "seed_052",
    "category": "evaluation_metric",
    "title": "RAGAS Metrics for Retrieval-Augmented Generation",
    "description": "RAG 系统专用的4大核心指标：Faithfulness（事实一致性）、Answer Relevancy（答案相关性）、Context Relevancy（上下文相关性）、Context Recall（上下文召回）。",
    "source_url": "https://learn.microsoft.com/en-us/ai/playbook/technology-guidance/generative-ai/working-with-llms/evaluation/list-of-eval-metrics",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["RAGAS", "RAG", "faithfulness", "relevancy", "recall"],
    "reliability": "A"
  },
  {
    "id": "seed_053",
    "category": "evaluation_metric",
    "title": "IoU for Object Detection and Segmentation",
    "description": "目标检测和图像分割中广泛使用的交并比（Intersection over Union）指标，用于衡量预测框与真实框的空间重叠度。",
    "source_url": "https://ar5iv.labs.arxiv.org/html/1912.10107",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["IoU", "object-detection", "segmentation", "computer-vision"],
    "reliability": "A"
  },
  {
    "id": "seed_054",
    "category": "evaluation_metric",
    "title": "Span-Level F1 for NER and Sequence Labeling",
    "description": "命名实体识别和序列标注任务中，使用跨度级别的 Precision 和 Recall 计算 F1，比 Kappa 更适合多标签和边界检测任务。",
    "source_url": "https://www.frontiersin.org/journals/artificial-intelligence/articles/10.3389/frai.2026.1766899/full",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["span-F1", "NER", "sequence-labeling", "precision-recall"],
    "reliability": "A"
  },
  {
    "id": "seed_055",
    "category": "evaluation_metric",
    "title": "DPP Benchmark Score Scaling",
    "description": "Data Preprocessing Benchmark 中采用的评分缩放方法：将方法影响缩放至 -2（最差）到 +2（最好），0 表示无影响，便于跨数据集比较。",
    "source_url": "https://d-nb.info/1247171701/34",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["DPP", "preprocessing", "score-scaling", "comparability"],
    "reliability": "A"
  },
  {
    "id": "seed_056",
    "category": "evaluation_metric",
    "title": "Maintainability Benchmarking Metrics",
    "description": "软件可维护性 benchmark 中使用的指标：Accuracy、Precision、Recall、F1、F0.5（对假阳性惩罚更重，适用于责任预测）。",
    "source_url": "https://arxiv.org/html/2408.10754v1",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["maintainability", "F0.5", "false-positive", "software-engineering"],
    "reliability": "A"
  },
  {
    "id": "seed_057",
    "category": "evaluation_metric",
    "title": "Cross-Validation as Robustness Check",
    "description": "将交叉验证作为模型选择和泛化能力的稳健性检查方法，配合非参数统计检验使用效果更佳。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["cross-validation", "robustness", "generalization", "model-selection"],
    "reliability": "A"
  },
  {
    "id": "seed_058",
    "category": "evaluation_metric",
    "title": "Expected Calibration Error (ECE)",
    "description": "HELM 框架中使用的校准度指标，衡量模型预测置信度与实际准确率之间的差异。",
    "source_url": "https://crfm.stanford.edu/helm/lite/latest/#/leaderboard",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["ECE", "calibration", "HELM", "confidence"],
    "reliability": "A"
  },
  {
    "id": "seed_059",
    "category": "evaluation_metric",
    "title": "BLEU and ROUGE for Generation Tasks",
    "description": "文本生成任务的经典 n-gram 重叠指标：BLEU 侧重于精确率，ROUGE 侧重于召回率。",
    "source_url": "https://www.analyticsvidhya.com/blog/2019/08/11-important-model-evaluation-error-metrics/",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["BLEU", "ROUGE", "text-generation", "n-gram"],
    "reliability": "A"
  },
  {
    "id": "seed_060",
    "category": "evaluation_metric",
    "title": "Gini Coefficient and Log Loss",
    "description": "分类任务中的概率评估指标：Log Loss 惩罚错误且自信的预测；Gini Coefficient 在 Kaggle 竞赛中常用于评估排序能力。",
    "source_url": "https://www.analyticsvidhya.com/blog/2019/08/11-important-model-evaluation-error-metrics/",
    "crawl_time": "2026-04-10T08:20:00Z",
    "tags": ["Log-Loss", "Gini", "probability", "ranking"],
    "reliability": "A"
  },
  {
    "id": "seed_061",
    "category": "quality_control_method",
    "title": "Gold Standard Benchmark Injection",
    "description": "将预先由专家确认正确答案的黄金任务随机插入标注队列，持续测量标注员准确率。Scale AI 和 Labelbox 均将其作为核心 QC 手段。",
    "source_url": "https://labelyourdata.com/articles/scale-ai-review",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["gold-standard", "benchmark-injection", "accuracy-monitoring"],
    "reliability": "A"
  },
  {
    "id": "seed_062",
    "category": "quality_control_method",
    "title": "Inter-Annotator Agreement (IAA) - Krippendorff's Alpha",
    "description": "最通用的标注一致性统计指标，支持任意数量标注员、缺失数据和多类型数据（名义/序数/区间/比率），接受阈值 α > 0.8。",
    "source_url": "https://datasaur.ai/blog-posts/inter-annotator-agreement-krippendorff-cohen",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["IAA", "Krippendorff-Alpha", "missing-data", "multi-rater"],
    "reliability": "A"
  },
  {
    "id": "seed_063",
    "category": "quality_control_method",
    "title": "Cohen's Kappa for Pairwise Agreement",
    "description": "纠正偶然一致性的两名标注员配对指标，适用于二分类和名义数据，不适用于序列标注或多标签任务。",
    "source_url": "https://datasaur.ai/blog-posts/inter-annotator-agreement-krippendorff-cohen",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["Cohen-Kappa", "pairwise", "chance-correction", "nominal"],
    "reliability": "A"
  },
  {
    "id": "seed_064",
    "category": "quality_control_method",
    "title": "Fleiss' Kappa for Multiple Raters",
    "description": "Cohen's Kappa 的多人扩展版本，适用于3名及以上标注员的名义数据一致性评估。",
    "source_url": "https://datasaur.ai/blog-posts/inter-annotator-agreement-krippendorff-cohen",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["Fleiss-Kappa", "multiple-raters", "nominal-data"],
    "reliability": "A"
  },
  {
    "id": "seed_065",
    "category": "quality_control_method",
    "title": "Consensus and Majority Voting",
    "description": "将同一任务分配给多名标注员，通过投票或一致性算法确定最终标签。Appen 和 Labelbox 都提供自动化 consensus 工具。",
    "source_url": "https://www.appen.com/ai-data-quality",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["consensus", "majority-voting", "multi-annotation", "Labelbox"],
    "reliability": "A"
  },
  {
    "id": "seed_066",
    "category": "quality_control_method",
    "title": "Rater Vitality (Vi) Metric",
    "description": "从 Krippendorff's Alpha 衍生的个体标注员影响力指标：Vi = Kα - Ki（去掉该标注员后的 Alpha）。正值表示提升共识，负值表示降低共识。",
    "source_url": "https://ar5iv.labs.arxiv.org/html/1912.10107",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["Rater-Vitality", "annotator-quality", "Krippendorff", "outlier-detection"],
    "reliability": "A"
  },
  {
    "id": "seed_067",
    "category": "quality_control_method",
    "title": "Class Recognition Difficulty Analysis",
    "description": "基于 K-alpha 的类内一致性分析，识别哪些类别 inherently difficult to identify，从而优化标注指南或拆分任务。",
    "source_url": "https://ar5iv.labs.arxiv.org/html/1912.10107",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["class-difficulty", "K-alpha", "category-analysis", "guideline-improvement"],
    "reliability": "A"
  },
  {
    "id": "seed_068",
    "category": "quality_control_method",
    "title": "Two-Layer Review Cycle",
    "description": "Scale AI 采用的双层审核机制：第一层为初级标注员，第二层为资深审核员，负责补充遗漏标注和修正错误。",
    "source_url": "https://labelyourdata.com/articles/scale-ai-review",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["two-layer-review", "Scale-AI", "senior-auditor", "error-correction"],
    "reliability": "A"
  },
  {
    "id": "seed_069",
    "category": "quality_control_method",
    "title": "Continuous QA with Error Trend Analysis",
    "description": "Appen 的 AI 辅助 QA 框架，通过 Model Mate 和质量报告框架进行标签级报告和错误趋势分析，实现贯穿训练数据开发生命周期的持续 QA。",
    "source_url": "https://www.appen.com/blog/quality-assurance-in-ai",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["continuous-QA", "error-trend", "Appen", "Model-Mate"],
    "reliability": "A"
  },
  {
    "id": "seed_070",
    "category": "quality_control_method",
    "title": "Pilot Annotation and Norming Sessions",
    "description": "在正式大规模标注前进行小规模试点，通过同行评审和规范化会议（norming sessions）校准评分标准，修订存在分歧的描述符。",
    "source_url": "https://teaching-resources.delta.ncsu.edu/rubric_best-practices-examples-templates/",
    "crawl_time": "2026-04-10T08:25:00Z",
    "tags": ["pilot", "norming-session", "calibration", "rubric-revision"],
    "reliability": "A"
  },
  {
    "id": "seed_071",
    "category": "baseline_calculation",
    "title": "Random Baseline for Multiple-Choice Benchmarks",
    "description": "多选题 benchmark 的随机基线计算方法。以 MMLU 为例，每题4个选项，随机猜测的期望准确率为25%。",
    "source_url": "https://arxiv.org/html/2406.01574v4",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["random-baseline", "MMLU", "multiple-choice", "25-percent"],
    "reliability": "A"
  },
  {
    "id": "seed_072",
    "category": "baseline_calculation",
    "title": "Weak Baseline: TF-IDF + Logistic Regression",
    "description": "使用简单模型作为弱基线，在 benchmark 上独立训练，用于验证任务的非平凡性和数据的可学习性。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["weak-baseline", "TF-IDF", "logistic-regression", "learnability"],
    "reliability": "A"
  },
  {
    "id": "seed_073",
    "category": "baseline_calculation",
    "title": "Strong Baseline: GPT-4o and Claude 3.5 Sonnet",
    "description": "使用当前 SOTA 商业模型作为强基线，采用标准 prompt 和固定温度参数（通常 temperature=0），确保结果可复现。",
    "source_url": "https://crfm.stanford.edu/helm/lite/latest/#/leaderboard",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["strong-baseline", "GPT-4o", "Claude", "SOTA"],
    "reliability": "A"
  },
  {
    "id": "seed_074",
    "category": "baseline_calculation",
    "title": "Human Baseline via Expert Annotators",
    "description": "招募专业领域的人类标注员完成相同任务，其平均表现作为人类基线。常用于医疗、法律等专业 benchmark。",
    "source_url": "https://research.aimultiple.com/medical-annotation/",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["human-baseline", "expert-annotator", "medical", "legal"],
    "reliability": "A"
  },
  {
    "id": "seed_075",
    "category": "baseline_calculation",
    "title": "Relative Gain Formula",
    "description": "评估新模型相对提升的标准化公式：Relative Gain = (Model - Baseline) / (Human - Baseline) × 100%，用于消除不同 benchmark 的绝对分差异。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["relative-gain", "normalization", "model-improvement", "formula"],
    "reliability": "A"
  },
  {
    "id": "seed_076",
    "category": "baseline_calculation",
    "title": "Wilcoxon Signed-Rank Test for Model Comparison",
    "description": "推荐使用的配对非参数统计检验，避免了单测试集 t 检验低估方差的问题，适用于多次交叉验证后的模型对比。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["Wilcoxon", "non-parametric", "model-comparison", "paired-test"],
    "reliability": "A"
  },
  {
    "id": "seed_077",
    "category": "baseline_calculation",
    "title": "Friedman Test for Multi-Model Comparison",
    "description": "当比较3个及以上模型时使用的非参数检验，配合 Nemenyi post-hoc 检验可揭示模型间的显著性差异。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["Friedman-test", "multi-model", "post-hoc", "Nemenyi"],
    "reliability": "A"
  },
  {
    "id": "seed_078",
    "category": "baseline_calculation",
    "title": "Bootstrap Resampling for Confidence Intervals",
    "description": "通过有放回重采样构建模型表现的置信区间，不依赖正态分布假设，适用于小样本或复杂分布的 benchmark 结果。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["bootstrap", "confidence-interval", "resampling", "small-sample"],
    "reliability": "A"
  },
  {
    "id": "seed_079",
    "category": "baseline_calculation",
    "title": "Contamination Check and Temporal Cutoff",
    "description": "benchmark 基线计算前必须进行的污染检查：记录训练数据截止日期，排除测试集在训练数据中公开出现的情况。",
    "source_url": "https://arxiv.org/html/2510.25039v1",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["contamination-check", "temporal-cutoff", "data-leakage", "benchmark-integrity"],
    "reliability": "A"
  },
  {
    "id": "seed_080",
    "category": "baseline_calculation",
    "title": "DPP Benchmark Composite Score",
    "description": "数据预处理 benchmark 中的综合得分方法：单独应用每种预处理方法，在固定 Random Forest 上测量相对于基线的性能影响。",
    "source_url": "https://d-nb.info/1247171701/34",
    "crawl_time": "2026-04-10T08:30:00Z",
    "tags": ["DPP", "composite-score", "preprocessing", "Random-Forest"],
    "reliability": "A"
  },
  {
    "id": "seed_081",
    "category": "dataset_construction",
    "title": "Vendi Score as Dataset Diversity Metric",
    "description": "基于成对相似度矩阵的 von Neumann 熵的无监督多样性指标，定义为相似度矩阵的熵指数，可跨图像、文本、分子等领域使用。",
    "source_url": "https://arxiv.org/pdf/2210.02410",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["Vendi-Score", "dataset-diversity", "von-Neumann-entropy", "reference-free"],
    "reliability": "A"
  },
  {
    "id": "seed_082",
    "category": "dataset_construction",
    "title": "Controlled Sourcing from Kaggle, UCI, and BEIR",
    "description": "从权威公开仓库选择数据集，确保缺乏直接开源解决方案，以防止测试集在执行前被污染。MAAB benchmark 的数据选择策略。",
    "source_url": "https://openreview.net/pdf?id=9t7gfYUp0U",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["controlled-sourcing", "Kaggle", "UCI", "BEIR", "contamination-prevention"],
    "reliability": "A"
  },
  {
    "id": "seed_083",
    "category": "dataset_construction",
    "title": "Synthetic Data Evaluation with Physics-Guided Rules",
    "description": "现代 generative model 评估不仅依赖 BLEU/CLIPScore，还需结合物理规则、上下文关系和基于规则的正确性检查，以识别物理不一致但视觉真实的样本。",
    "source_url": "https://arxiv.org/html/2511.15204v2",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["synthetic-data", "physics-guided", "contextual-correctness", "beyond-BLEU"],
    "reliability": "A"
  },
  {
    "id": "seed_084",
    "category": "dataset_construction",
    "title": "Mixed-Methods Design for Complex Domain Benchmarks",
    "description": "网络安全等领域的 benchmark 构建采用分阶段混合方法：文献综述 → 数据集选择 → 算法实现 → 模型训练/验证 → 评估 → 框架开发 → 仿真。",
    "source_url": "https://ijcaonline.org/archives/volume187/number14/mahmud-2025-ijca-925147.pdf",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["mixed-methods", "cybersecurity", "phased-design", "simulation"],
    "reliability": "A"
  },
  {
    "id": "seed_085",
    "category": "dataset_construction",
    "title": "Metrology Quality Framework for Trustworthy ML",
    "description": "INRiM / Mathmet 提出的 ML 训练/验证/测试数据集规范指南，涵盖质量、平衡性、偏见、增强和预处理，以及标准化接口。",
    "source_url": "https://www.inrim.it/sites/default/files/2024-02/SRA_EMN_Mathmet_Version_1.0_web_2.pdf",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["metrology", "trustworthy-ML", "standardization", "quality-framework"],
    "reliability": "A"
  },
  {
    "id": "seed_086",
    "category": "dataset_construction",
    "title": "LLM-in-the-Loop for Dynamic Benchmark Refresh",
    "description": "动态 benchmark 使用 LLM 自动生成新的问题和任务，再通过人类审核，实现 benchmark 的自动更新和难度调制。",
    "source_url": "https://www.emergentmind.com/topics/dynamic-benchmark-methodology",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["LLM-in-the-loop", "dynamic-benchmark", "auto-refresh", "difficulty-control"],
    "reliability": "A"
  },
  {
    "id": "seed_087",
    "category": "dataset_construction",
    "title": "Train/Validation/Test Split Ratio Best Practices",
    "description": "常见的数据集拆分比例：70/15/15 适用于中等规模数据；80/10/10 适用于大数据；医学或小样本领域常用 60/20/20 或交叉验证。",
    "source_url": "https://www.nature.com/articles/s41598-024-56706-x",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["data-split", "train-test-validation", "cross-validation", "best-practices"],
    "reliability": "A"
  },
  {
    "id": "seed_088",
    "category": "dataset_construction",
    "title": "Class Balance Maintenance in CV Datasets",
    "description": "计算机视觉数据集构建中，建议每类至少200个实例。对于稀有类，使用合成增强（synthetic augmentation）来维持类别平衡。",
    "source_url": "https://imgmarker.com/blog/yolo-dataset-annotation-best-practices",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["class-balance", "synthetic-augmentation", "computer-vision", "200-instances"],
    "reliability": "A"
  },
  {
    "id": "seed_089",
    "category": "dataset_construction",
    "title": "Maximal Diversity with Minimal Images",
    "description": "生物学表型领域的数据集设计原则：ground-truth 数据应代表最大的样本多样性，同时使用最少的图像数量，以减少人工标注和计算成本。",
    "source_url": "https://pdfs.semanticscholar.org/46da/25493f3771ca395349b42d86cf67c6aa8b99.pdf",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["maximal-diversity", "minimal-images", "cost-reduction", "phenotyping"],
    "reliability": "A"
  },
  {
    "id": "seed_090",
    "category": "dataset_construction",
    "title": "Versioning and Dataset Artifact Release",
    "description": "负责任 benchmark 发布要求：开放代码、标准化评估 split、公开数据集 artifacts，并显式版本化以追踪污染和耗尽指标。",
    "source_url": "https://www.emergentmind.com/topics/dynamic-benchmark-methodology",
    "crawl_time": "2026-04-10T08:35:00Z",
    "tags": ["versioning", "artifact-release", "open-science", "reproducibility"],
    "reliability": "A"
  }
]