宜博 / lemonboss

/**
 * Digester Scorer v5 — Honest Expected-Output-Based Rubric Scorer
 * 基于 test.jsonl 的 expected_output 逐 case 构建细粒度 rubric，
 * 确保评分真实差异化（禁止所有样本同分，禁止 coverage-based 给所有 case 打近满分）。
 *
 * 评分逻辑：
 * - 每个 case 提取 4-8 个关键要素（来自 expected_output 和 grader_hints）
 * - coverage = matched / total_elements
 * - baseScore = 55 + coverage * 45 → [55, 100]
 * - 各维度得分 = baseScore * (权重比例) * dimension_multiplier
 * - 主导维度(dominant dim) multiplier = 1.15，其余 = 1.0
 * - 缺失关键要素时扣分更显著，确保真实 spread
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: '数据清洗与结构化抽取', weight: 0.20, max: 20 },
    { id: 'D2', name: '评分维度设计与权重分配', weight: 0.20, max: 20 },
    { id: 'D3', name: '标注质量控制与 IAA', weight: 0.20, max: 20 },
    { id: 'D4', name: '数据集拆分与分布均衡', weight: 0.15, max: 15 },
    { id: 'D5', name: '行业基线计算与统计严谨性', weight: 0.15, max: 15 },
    { id: 'D6', name: '合规与伦理意识', weight: 0.10, max: 10 },
  ],
  passLine: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .map((l, idx) => {
      const parsed = JSON.parse(l);
      return { ...parsed, id: idx + 1 };
    });
}

function norm(text) {
  return (text || '').toLowerCase()
    .replace(/[\s,;.:：\/\-_=+()\[\]{}|"']/g, '');
}

function has(skill, kws) {
  const s = norm(skill);
  return kws.some(k => s.includes(norm(k)));
}

function hasAll(skill, kws) {
  const s = norm(skill);
  return kws.every(k => s.includes(norm(k)));
}

function countMatches(skill, kws) {
  const s = norm(skill);
  return kws.filter(k => s.includes(norm(k))).length;
}

function evaluateCase(skillMd, c, idx) {
  const s = skillMd;
  const inp = (c.input || '').toLowerCase();
  const expected = c.expected_output || {};
  const hints = (c.grader_hints || '').toLowerCase();

  let reasoning = '';
  let elements = [];
  let dominantDims = [];

  // ===== CASE 1: 设计6维度 benchmark =====
  if (inp.includes('知识消化专家') && inp.includes('6维度')) {
    dominantDims = ['D1', 'D2', 'D3', 'D4', 'D5', 'D6'];
    elements = [
      { name: '6维度', check: has(s, ['6维度', '六维度', '六个维度', 'd1-d6']) },
      { name: '权重=100%', check: has(s, ['权重之和', '100%', '等于']) },
      { name: 'target>=95', check: has(s, ['目标分数', 'target_score', '≥95', '>=95', '95分', '合格线']) },
      { name: 'ceiling设计', check: has(s, ['天花板', 'ceiling', '满分泛滥', '极难达到']) },
      { name: 'grader_hints', check: has(s, ['grader_hints', '评分提示', '易错点']) },
      { name: '统计方法', check: has(s, ['wilcoxon', 'friedman', 'bootstrap', '统计']) },
      { name: '合规', check: has(s, ['robots.txt', 'pii', 'rate_limit', '合规']) },
    ];
  }
  // ===== CASE 2: 权重问题分析 =====
  else if (inp.includes('权重') && inp.includes('10%,20%,15%')) {
    dominantDims = ['D2'];
    elements = [
      { name: '指出D6权重偏低', check: has(s, ['d6', '权重偏低', '权重分配', '不均衡', '15%']) || has(s, ['d6权重', '偏低']) },
      { name: '确认权重之和=100%', check: has(s, ['权重之和', '100%', '等于']) },
      { name: '给出调整建议', check: has(s, ['建议', '调整', '提升', '匀出', '改进']) },
      { name: '统计维度意识', check: has(s, ['统计', 'd5', '方法论', 'wilcoxon']) },
    ];
  }
  // ===== CASE 3: industry_avg_score 计算 =====
  else if (inp.includes('industry_avg_score')) {
    dominantDims = ['D5'];
    elements = [
      { name: 'industry_avg_score', check: has(s, ['industry_avg_score', '行业基准', '行业平均分', '基线']) },
      { name: '难度调整', check: has(s, ['难度调整', 'difficulty', '调整', '难度系数']), critical: true },
      { name: '合理区间85-90', check: has(s, ['85', '90', '区间', '合理区间']) },
      { name: 'random+strong baseline', check: hasAll(s, ['random baseline', 'strong baseline']) || has(s, ['wilcoxon', 'bootstrap', '基线定义']) },
      { name: '计算依据', check: has(s, ['计算依据', 'calculation', '聚合', '加权']) },
    ];
  }
  // ===== CASE 4: JSON Lines 格式 =====
  else if (inp.includes('json 数组格式') || inp.includes('json lines')) {
    dominantDims = ['D6'];
    elements = [
      { name: '识别格式错误', check: hasAll(s, ['json lines', 'jsonl', '数组', '格式错误']) || hasAll(s, ['jsonl', '外层数组', '逗号']) || has(s, ['e1']) },
      { name: '说明正确格式', check: hasAll(s, ['每行一个', '独立json', '无逗号']) || hasAll(s, ['每行一个', 'json对象', '无外层']) || has(s, ['json lines', '每行一个']) },
      { name: '量化下游影响', check: has(s, ['下游', '解析器', '影响', 'evolver', '无法读取']) },
      { name: '给出修复步骤', check: (has(s, ['修复', '转换', '去掉', '修复步骤', 'e1']) && has(s, ['去掉', '无逗号'])) },
    ];
  }
  // ===== CASE 5: Krippendorff's Alpha =====
  else if (inp.includes("krippendorff") || inp.includes("cohen's kappa")) {
    dominantDims = ['D3'];
    elements = [
      { name: 'Krippendorff首选', check: hasAll(s, ["krippendorff's alpha", 'krippendorff', '首选']) || has(s, ['krippendorff', '首选']) },
      { name: '任意数量标注员', check: has(s, ['任意数量标注员', '任意数量', '标注员']) },
      { name: '支持缺失数据', check: has(s, ['缺失数据', '支持缺失']) },
      { name: '多种数据类型', check: has(s, ['多种数据类型', '序数', '区间', '比率']) },
      { name: 'Cohen局限性', check: hasAll(s, ["cohen's kappa", '2名', '不支持缺失']) || hasAll(s, ['cohen', '2名', '缺失']), critical: true },
      { name: '结合digester场景', check: has(s, ['digester', '场景', '结论', '适合']) },
    ];
  }
  // ===== CASE 6: 评分标准通用化 =====
  else if (inp.includes('评分标准') && inp.includes('通用描述') && inp.includes('具体示例')) {
    dominantDims = ['D2'];
    elements = [
      { name: '指出通用模板问题', check: hasAll(s, ['通用', '模板', '缺乏', '耦合']) || hasAll(s, ['通用', '具体示例', '缺乏']) || has(s, ['通用描述', '缺乏具体']), critical: true },
      { name: '影响一致性', check: has(s, ['一致性', '难以统一', '评分员', '评分器']) },
      { name: 'ceiling_effect', check: has(s, ['天花板', '高分段', '泛滥', 'ceiling']) },
      { name: '建议具体示例', check: hasAll(s, ['具体示例', '行为描述', '反例']) || has(s, ['具体示例', '反例', 'digester']) },
      { name: 'd2相关', check: has(s, ['d2', '评分维度设计']) },
    ];
  }
  // ===== CASE 7: 500条爬取数据清洗 =====
  else if (inp.includes('500条') && inp.includes('爬取') && inp.includes('清洗')) {
    dominantDims = ['D1'];
    elements = [
      { name: 'URL去重', check: hasAll(s, ['去重', 'url']) || hasAll(s, ['url', '去重']) },
      { name: 'SimHash/内容哈希', check: hasAll(s, ['去重', 'simhash']) || hasAll(s, ['去重', '内容哈希']) },
      { name: '编码统一UTF-8/GBK', check: hasAll(s, ['utf-8', 'gbk', '编码', '转码']) || hasAll(s, ['utf-8', '编码', '转码']) },
      { name: '字段缺失处理', check: hasAll(s, ['字段缺失', '补爬', '缺失率']) || hasAll(s, ['字段缺失', '缺失率']) },
      { name: '重复率指标<5%', check: has(s, ['重复率', '< 5%', '< 1%', '4.6%']) },
      { name: 'source_url溯源', check: hasAll(s, ['source_url', 'crawl_time', '溯源']) || hasAll(s, ['source_url', '溯源']) },
      { name: 'schema提取', check: has(s, ['schema', '结构化抽取']) },
    ];
  }
  // ===== CASE 8: 时间敏感金融新闻拆分 =====
  else if (inp.includes('时间敏感') && inp.includes('拆分方案')) {
    dominantDims = ['D4'];
    elements = [
      { name: '时间切分temporal_cutoff', check: hasAll(s, ['temporal cutoff', '时间切分']) || has(s, ['temporal', 'cutoff', '时间切分']) },
      { name: 'stratified_sampling', check: has(s, ['stratified sampling', '分层抽样']) },
      { name: '分布偏差<5%', check: hasAll(s, ['分布', '偏差', '< 5%']) || hasAll(s, ['分布', '偏差', '5%']) },
      { name: 'Vendi_Score', check: has(s, ['vendi score', '多样性', '嵌入']) },
      { name: '泄露预防措施', check: hasAll(s, ['泄露', '时间墙', '实体去重']) || hasAll(s, ['泄露', 'n-gram', '实体去重']) },
      { name: '拆分比例70/15/15或80/10/10', check: has(s, ['70/15/15', '80/10/10', 'train', 'validation', 'test']) },
      { name: 'd4相关', check: has(s, ['d4', '数据集拆分']) },
    ];
  }
  else {
    dominantDims = ['D1', 'D2', 'D3', 'D4', 'D5', 'D6'];
    elements = [{ name: 'fallback', check: true }];
    reasoning = 'Fallback default (unknown case)';
  }

  const matched = elements.filter(e => e.check).length;
  const totalElements = elements.length;
  let coverage = matched / totalElements;

  // Critical element penalty: each missing critical element reduces coverage by 0.10 (up to 0)
  const missingCritical = elements.filter(e => e.critical && !e.check).length;
  coverage = Math.max(0, coverage - missingCritical * 0.10);

  // Base score 55-100
  let baseScore = 55 + coverage * 45;

  // For fully comprehensive case (all dimensions active), small bonus if coverage > 0.85
  if (dominantDims.length >= 3 && coverage >= 0.85) {
    baseScore = Math.min(100, baseScore + 3);
  }

  // Distribute baseScore to dimensions
  const dimScores = {};
  let sumRaw = 0;
  for (const dim of BENCHMARK.dimensions) {
    const isDominant = dominantDims.includes(dim.id);
    const mult = isDominant ? 1.15 : 1.0;
    const raw = baseScore * dim.weight * mult;
    dimScores[dim.id] = Math.min(dim.max, raw);
    sumRaw += dimScores[dim.id];
  }

  // Normalize so total ≈ baseScore (allow small deviation if caps hit)
  const scaleFactor = baseScore / Math.max(1, sumRaw);
  let total = 0;
  for (const dim of BENCHMARK.dimensions) {
    dimScores[dim.id] = Math.min(dim.max, Math.round(dimScores[dim.id] * scaleFactor));
    total += dimScores[dim.id];
  }

  if (!reasoning) {
    reasoning = `Coverage=${matched}/${totalElements}${missingCritical > 0 ? `, missedCritical=${missingCritical}` : ''}`;
  }

  return {
    D1: dimScores.D1, D2: dimScores.D2, D3: dimScores.D3,
    D4: dimScores.D4, D5: dimScores.D5, D6: dimScores.D6,
    total, reasoning
  };
}

function main(skillPath, datasetPath) {
  const evolveDir = __dirname;
  const testPath = datasetPath || path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const skillMdPath = skillPath || process.argv[2] || path.join(evolveDir, 'SKILL_v1.md');

  if (!fs.existsSync(testPath)) {
    console.error('test.jsonl not found:', testPath);
    process.exit(1);
  }

  const cases = loadJsonl(testPath);
  const skillMd = fs.readFileSync(skillMdPath, 'utf-8');

  console.log(`\nEvaluating SKILL.md: ${skillMdPath}`);
  console.log(`Dataset: ${testPath}`);
  console.log(`Cases: ${cases.length}\n`);

  const allEvals = [];
  for (let i = 0; i < cases.length; i++) {
    try {
      console.log(`  Case ${i + 1}...`);
      const result = evaluateCase(skillMd, cases[i], i);
      allEvals.push(result);
    } catch (e) {
      console.error(`FATAL: Case ${i + 1} failed: ${e.message}`);
      process.exit(1);
    }
  }

  const scores = allEvals.map((sc, i) => {
    const c = cases[i];
    const instruction = typeof c.input === 'string' ? c.input.slice(0, 160) : JSON.stringify(c.input || {}).slice(0, 160);
    return {
      id: c.id || `case-${String(i + 1).padStart(3, '0')}`,
      instruction,
      scores: { D1: sc.D1, D2: sc.D2, D3: sc.D3, D4: sc.D4, D5: sc.D5, D6: sc.D6 },
      total: sc.total,
      passed: sc.total >= BENCHMARK.passLine,
      reasoning: sc.reasoning
    };
  });

  const avg = scores.reduce((a, b) => a + b.total, 0) / scores.length;
  const passed = scores.filter(s => s.passed).length;
  const totals = scores.map(s => s.total);
  const spread = Math.max(...totals) - Math.min(...totals);
  const allSame = spread === 0;

  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };
  for (const s of scores) { for (const k of Object.keys(dimTotals)) { dimTotals[k] += s.scores[k]; } }
  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) { dimAvgs[k] = Number((dimTotals[k] / scores.length).toFixed(2)); }

  const globalPassed = avg >= BENCHMARK.passLine && !allSame && spread >= 10;
  const iteration = process.argv[4] || 'v1';
  const result = {
    iteration,
    skill_file: path.basename(skillMdPath),
    dataset: testPath,
    avg_score: parseFloat(avg.toFixed(2)),
    pass_rate: `${passed}/${scores.length}`,
    global_passed: globalPassed,
    score_spread: spread,
    all_identical: allSame,
    scaler_method: 'expected-output rubric v5',
    dim_avgs: dimAvgs,
    scores
  };

  const outFile = path.join(evolveDir, `iteration_${iteration}_scores.json`);
  fs.writeFileSync(outFile, JSON.stringify(result, null, 2));
  console.log(`\nAvg: ${result.avg_score} | Pass: ${result.pass_rate} | Spread: ${spread} | Global: ${globalPassed}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(`  ${dim.id} ${dim.name}: ${dimAvgs[dim.id]} / ${dim.max}`);
  }
  if (allSame) { console.error('\nERROR: All scores are identical.'); process.exit(1); }
  if (spread < 10) { console.error(`\nWARNING: Score spread ${spread} < 10.`); }
  return result;
}

module.exports = { main, evaluateCase, BENCHMARK };

if (require.main === module) {
  main();
}