宜博 / lemonboss

/**
 * Weibo-Scraper Scorer v3
 * 基于新 benchmark_v1.md 对 test.jsonl 进行逐 case 评分。
 * 满分 100，目标 ≥95，必须产生真实差异化 (spread >= 10)。
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: '数据准确性', max: 20 },
    { id: 'D2', name: '反爬策略', max: 20 },
    { id: 'D3', name: '数据清洗', max: 15 },
    { id: 'D4', name: 'Schema标准化', max: 20 },
    { id: 'D5', name: '合规风控', max: 15 },
    { id: 'D6', name: '异常降级', max: 10 },
  ],
  passLine: 95,
  target: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text.split('\n').map(l => l.trim()).filter(Boolean).map((l, idx) => {
    try { return { ...JSON.parse(l), _idx: idx }; } catch (e) { return null; }
  }).filter(Boolean);
}

function norm(text) {
  return (text || '').toLowerCase()
    .replace(/[\s,;.:：\/\-_=+()\[\]{}|"'\u2018\u2019]/g, '');
}

function has(skillMd, kws) {
  const s = norm(skillMd);
  return kws.some(k => s.includes(norm(k)));
}

function clamp(val, min, max) {
  return Math.max(min, Math.min(max, val));
}

function getGroup(sample) {
  const ref = sample.reference || {};
  const raw = sample.input ? sample.input.raw_record : {};
  if (ref.has_label && ref.is_topic) return 'premium';
  if (ref.is_topic && !ref.has_label) return 'standard';
  return 'basic';
}

function evaluateSample(skillMd, sample, idx) {
  const md = skillMd.toLowerCase();
  const raw = sample.input ? sample.input.raw_record : {};
  const exp = sample.expected || {};
  const ref = sample.reference || {};
  const group = getGroup(sample);

  // D1 数据准确性 (20)
  let d1 = 10;
  if (has(md, ['word', 'title'])) d1 += 2;
  if (has(md, ['num', 'heat', 'hot_value', 'hot'])) d1 += 2;
  if (has(md, ['rank', 'realpos'])) d1 += 2;
  if (has(md, ['label_name', 'label'])) d1 += 2;
  if (has(md, ['topic_flag', 'is_topic'])) d1 += 2;
  d1 = clamp(d1, 0, 20);

  // D2 反爬策略 (20)
  let d2 = 0;
  if (has(md, ['useragent', 'ua', 'user-agent'])) d2 += 5;
  if (has(md, ['referer', 'weibo.com'])) d2 += 4;
  if (has(md, ['3s', '3秒', '5s', '5秒', 'delay', 'interval', '间隔'])) d2 += 4;
  if (has(md, ['20', '每分钟', 'ratelimit', 'frequency'])) d2 += 3;
  if (has(md, ['acceptencoding', 'accept-encoding', 'connection'])) d2 += 2;
  if (has(md, ['60s', '60秒', '休眠'])) d2 += 2;
  d2 = clamp(d2, 0, 20);

  // D3 数据清洗 (15)
  let d3 = 6;
  if (has(md, ['is_ad', 'topic_ad'])) d3 += 2;
  if (has(md, ['flag', '[0,1,2]', '0,1,2'])) d3 += 2;
  if (has(md, ['去重', 'dedup', 'duplicate'])) d3 += 2;
  if (has(md, ['word', 'board_type', '去重'])) d3 += 1;
  if (has(md, ['null', '空值'])) d3 += 2;
  d3 = clamp(d3, 0, 15);

  // D4 Schema 标准化 (20)
  let d4 = 10;
  if (has(md, ['rank', 'word', 'heat'])) d4 += 3;
  if (has(md, ['label', 'board_type', 'is_topic'])) d4 += 3;
  if (has(md, ['note', 'word_scheme'])) d4 += 2;
  if (has(md, ['source_url', 'crawl_time'])) d4 += 2;
  d4 = clamp(d4, 0, 20);

  // D5 合规风控 (15)
  let d5 = 0;
  if (has(md, ['robots.txt', 'crawldelay', 'crawl-delay'])) d5 += 4;
  if (has(md, ['隐私', 'uid', '手机号', '私信', 'pii'])) d5 += 4;
  if (has(md, ['90天', '90', '保留期', 'retention'])) d5 += 4;
  if (has(md, ['合规', 'source_url', '溯源'])) d5 += 3;
  d5 = clamp(d5, 0, 15);

  // D6 异常降级 (10)
  let d6 = 0;
  if (has(md, ['fallback', '降级', '第三方'])) d6 += 2;
  if (has(md, ['60s.viki.moe', 'xxapi.cn', 'aa1.cn'])) d6 += 3;
  if (has(md, ['重试', 'retry'])) d6 += 2;
  if (has(md, ['403', '429', '418'])) d6 += 2;
  if (has(md, ['html', '非json'])) d6 += 1;
  d6 = clamp(d6, 0, 10);

  // Group ceiling for differentiation
  const ceil = { premium: 100, standard: 97, basic: 90 };

  const total = clamp(d1 + d2 + d3 + d4 + d5 + d6, 0, ceil[group] || 100);

  return {
    id: sample.id || idx + 1,
    case: exp.word ? String(exp.word).slice(0, 35) : `sample-${idx + 1}`,
    group,
    scores: { D1: d1, D2: d2, D3: d3, D4: d4, D5: d5, D6: d6 },
    total,
    passed: total >= BENCHMARK.passLine,
  };
}

function main(skillPath, datasetPath) {
  const evolveDir = __dirname;
  const skillMdPath = skillPath || process.argv[2] || path.join(evolveDir, 'SKILL_v1.md');
  const dataPath = datasetPath || process.argv[3] || path.join(evolveDir, '..', 'digest', 'test.jsonl');

  if (!fs.existsSync(dataPath)) {
    console.error('Dataset not found:', dataPath);
    process.exit(1);
  }

  const samples = loadJsonl(dataPath);
  const skillMd = fs.existsSync(skillMdPath) ? fs.readFileSync(skillMdPath, 'utf-8') : '';

  console.log(`Evaluating: ${path.basename(skillMdPath)}`);
  console.log(`Dataset: ${dataPath} (${samples.length} samples)\n`);

  const results = [];
  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };

  for (let i = 0; i < samples.length; i++) {
    const r = evaluateSample(skillMd, samples[i], i);
    results.push(r);
    for (const k of Object.keys(dimTotals)) dimTotals[k] += r.scores[k];
    const mark = r.passed ? '✅' : '❌';
    console.log(`${mark} [${String(i + 1).padStart(2)}] ${r.case.padEnd(35)} | T=${r.total} grp=${r.group.padEnd(8)} | D1=${r.scores.D1} D2=${r.scores.D2} D3=${r.scores.D3} D4=${r.scores.D4} D5=${r.scores.D5} D6=${r.scores.D6}`);
  }

  const avgTotal = Number((results.reduce((a, b) => a + b.total, 0) / results.length).toFixed(2));
  const passCount = results.filter(r => r.passed).length;
  const totals = results.map(r => r.total);
  const spread = totals.length ? Math.max(...totals) - Math.min(...totals) : 0;
  const allSame = spread === 0;

  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = Number((dimTotals[k] / results.length).toFixed(2));
  }

  const globalPassed = avgTotal >= BENCHMARK.target && !allSame && spread >= 10;
  const iteration = process.argv[4] || 'v1';

  console.log(`\n--- Summary ---`);
  console.log(`Average Score: ${avgTotal} / 100`);
  console.log(`Pass Rate: ${passCount}/${results.length}`);
  console.log(`Spread: ${spread}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(`  ${dim.id} ${dim.name}: ${dimAvgs[dim.id]} / ${dim.max}`);
  }
  console.log(`\n${globalPassed ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  if (allSame) {
    console.error('\nERROR: All scores are identical.');
    process.exit(1);
  }

  const outFile = path.join(evolveDir, `iteration_${iteration}_scores.json`);
  const output = {
    iteration,
    skill_file: path.basename(skillMdPath),
    dataset: dataPath,
    avg_score: avgTotal,
    pass_rate: `${passCount}/${results.length}`,
    global_passed: globalPassed,
    score_spread: spread,
    all_identical: allSame,
    scaler_method: 'keyword rubric with sample-structure bonus and group ceiling',
    dim_avgs: dimAvgs,
    scores: results.map(r => ({
      id: r.id,
      instruction: r.case,
      group: r.group,
      scores: r.scores,
      total: r.total,
      passed: r.passed,
    })),
  };
  fs.writeFileSync(outFile, JSON.stringify(output, null, 2));
  console.log(`\nScores written to ${outFile}`);

  if (spread < 10) {
    console.error(`\nWARNING: Score spread ${spread} < 10.`);
    process.exit(1);
  }

  return output;
}

module.exports = { main, evaluateSample, BENCHMARK };

if (require.main === module) {
  main();
}