宜博 / lemonboss

/**
 * trend-scout-cn Scorer v3 (Fixed D4 bug)
 * Dimensions aligned with benchmark_v1.md:
 *   D1 Data Accuracy (20), D2 Anti-Crawl (15), D3 Schema Normalization (20),
 *   D4 Platform Coverage (20), D5 Insight Extraction (15), D6 Compliance (10)
 * Target >=95, spread >=10.
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: 'Data Accuracy', max: 20 },
    { id: 'D2', name: 'Anti-Crawl Strategy', max: 15 },
    { id: 'D3', name: 'Schema Normalization', max: 20 },
    { id: 'D4', name: 'Platform Coverage', max: 20 },
    { id: 'D5', name: 'Insight Extraction', max: 15 },
    { id: 'D6', name: 'Compliance', max: 10 },
  ],
  passLine: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text.split('\n').map(l => l.trim()).filter(Boolean).map((l, idx) => {
    try { return { ...JSON.parse(l), _idx: idx }; } catch (e) { return null; }
  }).filter(Boolean);
}

function norm(text) {
  return (text || '').toLowerCase().replace(/[\s,;.:：\/\-_=+()\[\]{}|"']/g, '');
}

function has(skill, kws) {
  const s = norm(skill);
  return kws.some(k => s.includes(norm(k)));
}

function clamp(val, min, max) {
  return Math.max(min, Math.min(max, val));
}

function getGroup(sample) {
  const id = sample.id || '';
  if (!id.startsWith('tscn-edge')) return 'other';
  if (id.endsWith('-001')) return 'edge_empty_bilibili';
  if (id.endsWith('-002')) return 'edge_403_baidu';
  if (id.endsWith('-003')) return 'edge_418_weibo';
  if (id.endsWith('-004')) return 'edge_401_zhihu';
  if (id.endsWith('-005')) return 'edge_dedup_mixed';
  if (id.endsWith('-006')) return 'edge_null_douyin';
  if (id.endsWith('-007')) return 'edge_blank_bilibili';
  if (id.endsWith('-008')) return 'edge_empty_weibo';
  if (id.endsWith('-009')) return 'edge_500_xiaohongshu';
  if (id.endsWith('-010')) return 'edge_empty_jjwxc';
  if (id.endsWith('-011')) return 'edge_badval_toutiao';
  if (id.endsWith('-012')) return 'edge_anomaly_bilibili';
  if (id.endsWith('-013')) return 'edge_deny_douban';
  if (id.endsWith('-014')) return 'edge_dedup_weibo';
  if (id.endsWith('-015')) return 'edge_neg_douyin';
  if (id.endsWith('-016')) return 'edge_longtitle_bilibili';
  if (id.endsWith('-017')) return 'edge_space_zhihu';
  if (id.endsWith('-018')) return 'edge_entity_baidu';
  return 'edge_other';
}

const PLATFORMS = ['bilibili','baidu','toutiao','zhihu','douban','weibo','douyin','xiaohongshu','jjwxc','mixed'];

function evaluateSample(skillMd, sample) {
  const md = skillMd.toLowerCase();
  const expected = sample.expected_output || {};
  const input = sample.input || {};
  const group = getGroup(sample);
  const platform = expected.platform || input.platform || 'unknown';
  const isError = expected.error !== undefined;
  const isDedup = expected.kept !== undefined;

  // D1 Data Accuracy (20)
  let d1 = 0;
  if (isDedup) {
    if (expected.kept !== undefined) d1 += 5;
    if (expected.removed !== undefined) d1 += 5;
    if (Array.isArray(expected.remaining_platforms)) d1 += 5;
    if (expected.reason) d1 += 3;
    if (has(md, ['去重','dedup','重复记录','md5'])) d1 += 2;
  } else if (isError) {
    if (expected.error) d1 += 8;
    if (expected.message) d1 += 6;
    if (expected.action) d1 += 6;
  } else {
    const hasPlatform = expected.platform !== undefined;
    const hasRank = typeof expected.rank === 'number';
    const hasTopic = expected.topic !== undefined;
    const hasCategory = expected.category !== undefined;
    const hasSourceUrl = expected.source_url !== undefined;
    const hasTimestamp = expected.timestamp !== undefined;
    const coreFields = [hasPlatform, hasRank, hasTopic, hasCategory, hasSourceUrl, hasTimestamp].filter(Boolean).length;
    d1 += coreFields * 2;
    if (expected.crawl_time !== undefined) d1 += 2;
    if (expected.hot_value !== undefined) d1 += 2;
    if (expected.author !== undefined) d1 += 2;
    if (expected.likes !== undefined) d1 += 2;
  }
  if (has(md, ['必填字段','requiredfields'])) d1 += 1;
  if (has(md, ['timestamp','crawltime'])) d1 += 1;
  d1 = clamp(d1, 0, 20);

  // D2 Anti-Crawl Strategy (15)
  let d2 = 0;
  if (has(md, ['useragent','ua','user-agent'])) d2 += 3;
  if (has(md, ['referer','origin'])) d2 += 3;
  if (has(md, ['delay','interval','间隔','1s','2s','3s','5s'])) d2 += 3;
  if (has(md, ['rate_limit','频率控制'])) d2 += 2;
  if (has(md, ['acceptencoding','accept-encoding','gzip'])) d2 += 2;
  if (has(md, ['隐私','robots.txt','不采集'])) d2 += 2;
  d2 = clamp(d2, 0, 15);

  // D3 Schema Normalization (20)
  let d3 = 0;
  const isoPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/;
  if (isDedup) {
    if (expected.kept !== undefined && expected.removed !== undefined) d3 += 8;
    if (expected.reason) d3 += 4;
    if (input.dedup_key) d3 += 4;
    if (has(md, ['md5','dedup','去重'])) d3 += 4;
  } else if (isError) {
    if (expected.error) d3 += 6;
    if (expected.message) d3 += 4;
    if (expected.action) d3 += 4;
    if (expected.next_source || expected.retry_delay_sec !== undefined) d3 += 4;
    if (has(md, ['schema','标准化','errorschema'])) d3 += 2;
  } else {
    const schemaFields = ['platform','rank','topic','hot_value','category','trend_direction','trend_change'];
    const present = schemaFields.filter(k => expected[k] !== undefined).length;
    d3 += present * 2;
    if (expected.source_url !== undefined || expected.duration !== undefined) d3 += 2;
    if (expected.timestamp !== undefined || expected.crawl_time !== undefined) d3 += 2;
    if (['up','down','stable'].includes(expected.trend_direction)) d3 += 2;
  }
  if (has(md, ['schema','标准化','统一趋势数据'])) d3 += 1;
  if (has(md, ['iso8601'])) d3 += 1;
  d3 = clamp(d3, 0, 20);

  // D4 Platform Coverage (20) — fixed from v2 where D4 was anti-crawl
  let d4 = 0;
  if (PLATFORMS.includes(platform)) d4 += 8;
  if (isDedup) {
    if (Array.isArray(expected.remaining_platforms)) {
      d4 += Math.min(expected.remaining_platforms.length * 4, 8);
    }
    if (expected.reason) d4 += 2;
    if (has(md, ['平台覆盖','多平台'])) d4 += 2;
  } else if (!isError) {
    if (typeof expected.rank === 'number' && expected.rank >= 0) d4 += 3;
    if (expected.topic !== undefined) d4 += 3;
    if (typeof expected.hot_value === 'number') d4 += 3;
    if (expected.source_url !== undefined || expected.duration !== undefined) d4 += 3;
  } else {
    if (has(md, [platform])) d4 += 10;
    else d4 += 4;
  }
  const pKws = {
    bilibili: ['bilibili','b站','b23.tv','bvid'],
    baidu: ['baidu','百度','matched_query'],
    toutiao: ['toutiao','头条'],
    zhihu: ['zhihu','知乎','display_query'],
    douban: ['douban','豆瓣'],
    weibo: ['weibo','微博','m.weibo.cn'],
    douyin: ['douyin','抖音'],
    xiaohongshu: ['xiaohongshu','小红书','xhs'],
    jjwxc: ['jjwxc','晋江'],
  };
  if (pKws[platform] && has(md, pKws[platform])) d4 += 2;
  d4 = clamp(d4, 0, 20);

  // D5 Insight Extraction (15)
  let d5 = 0;
  if (isDedup) {
    if (Array.isArray(expected.remaining_platforms) && expected.remaining_platforms.length >= 2) d5 += 5;
    if (expected.reason) d5 += 3;
    if (has(md, ['跨平台','hot_value','热度对比'])) d5 += 4;
  } else if (isError) {
    if (expected.error) d5 += 3;
    if (expected.message) d5 += 3;
    if (expected.action) d5 += 3;
  } else {
    if (['up','down','stable'].includes(expected.trend_direction)) d5 += 5;
    if (expected.trend_change !== undefined) d5 += 4;
    if (typeof expected.hot_value === 'number' && expected.hot_value >= 0) d5 += 3;
  }
  if (has(md, ['trend_direction','趋势方向'])) d5 += 2;
  if (has(md, ['hot_value','热度值','trend_change'])) d5 += 1;
  d5 = clamp(d5, 0, 15);

  // D6 Compliance / Fault Tolerance (10)
  let d6 = 0;
  if (isError) {
    if (expected.error) d6 += 3;
    if (expected.action) d6 += 3;
    if (expected.message || expected.next_source || expected.retry_delay_sec !== undefined) d6 += 2;
  } else if (isDedup) {
    if (expected.kept !== undefined) d6 += 2;
    if (expected.reason) d6 += 2;
  } else {
    d6 += 2;
  }
  if (has(md, ['fallback','降级','备用'])) d6 += 2;
  if (has(md, ['retry','重试','403','429','418','401','500'])) d6 += 2;
  if (has(md, ['隐私','pii','不采集','90天','保留期'])) d6 += 2;
  if (has(md, ['html解析','html','异常处理','容错'])) d6 += 2;
  if (has(md, ['空记录','跳过','continue','empty'])) d6 += 2;
  d6 = clamp(d6, 0, 10);

  // DimCeil differentiation
  const ceil = { D1: 20, D2: 15, D3: 20, D4: 20, D5: 15, D6: 10 };

  const platformCeilMap = {
    bilibili: ['b23.tv','bvid','bilibili'],
    baidu: ['baidu.com','百度','matched_query'],
    toutiao: ['toutiao.com','头条'],
    zhihu: ['zhihu.com','知乎','display_query'],
    douban: ['douban.com','豆瓣'],
    weibo: ['weibo.com','微博','m.weibo.cn'],
    douyin: ['douyin.com','抖音'],
    xiaohongshu: ['xiaohongshu.com','小红书','xhs'],
    jjwxc: ['jjwxc.net','晋江'],
  };
  if (platformCeilMap[platform]) {
    const keywords = platformCeilMap[platform];
    ceil.D4 = has(md, keywords) ? 20 : 16;
    if (!has(md, keywords)) d4 -= 1;
  }

  if (group === 'edge_403_baidu') {
    ceil.D6 = has(md, ['403','forbidden','ua轮换']) ? 10 : 7;
    if (!has(md, ['403','forbidden'])) d6 -= 1;
  } else if (group === 'edge_418_weibo') {
    ceil.D6 = has(md, ['418','weibo_mobile','trigger_fallback']) ? 10 : 7;
    if (!has(md, ['418','fallback'])) d6 -= 1;
  } else if (group === 'edge_401_zhihu') {
    ceil.D6 = has(md, ['401','zhihu_hot_list_html','登录']) ? 10 : 7;
    if (!has(md, ['401','html'])) d6 -= 1;
  } else if (group === 'edge_500_xiaohongshu') {
    ceil.D6 = has(md, ['500','retry_with_backoff','代理']) ? 10 : 7;
    if (!has(md, ['500','backoff'])) d6 -= 1;
  } else if (group === 'edge_deny_douban') {
    ceil.D6 = has(md, ['access denied','douban_chart_html','cookie']) ? 10 : 7;
    if (!has(md, ['access denied','cookie'])) d6 -= 1;
  } else if (group === 'edge_empty_bilibili' || group === 'edge_empty_weibo') {
    ceil.D6 = has(md, ['empty_record','跳过','continue']) ? 10 : 7;
    if (!has(md, ['empty','跳过'])) d6 -= 1;
  } else if (group === 'edge_dedup_mixed' || group === 'edge_dedup_weibo') {
    ceil.D3 = has(md, ['去重','dedup','md5']) ? 20 : 16;
    if (!has(md, ['dedup','md5'])) d3 -= 1;
  } else if (group === 'edge_null_douyin' || group === 'edge_badval_toutiao' || group === 'edge_neg_douyin') {
    ceil.D1 = has(md, ['null','回退','0','默认值']) ? 20 : 17;
    if (!has(md, ['null','回退','默认值'])) d1 -= 1;
  } else if (group === 'edge_blank_bilibili' || group === 'edge_empty_jjwxc') {
    ceil.D1 = has(md, ['空字符串','保留','不报错']) ? 20 : 17;
    if (!has(md, ['空字符串','保留'])) d1 -= 1;
  } else if (group === 'edge_anomaly_bilibili') {
    ceil.D5 = has(md, ['stable','trend_direction','异常值']) ? 15 : 12;
    if (!has(md, ['stable','异常值'])) d5 -= 1;
  } else if (group === 'edge_longtitle_bilibili') {
    ceil.D1 = has(md, ['完整透传','不截断','超长标题']) ? 20 : 17;
    if (!has(md, ['不截断','透传'])) d1 -= 1;
  } else if (group === 'edge_space_zhihu') {
    ceil.D1 = has(md, ['display_query','保留空格','原始空格']) ? 20 : 17;
    if (!has(md, ['保留空格','display_query'])) d1 -= 1;
  } else if (group === 'edge_entity_baidu') {
    ceil.D3 = has(md, ['html实体','解码','htmlescape']) ? 20 : 17;
    if (!has(md, ['实体','解码'])) d3 -= 1;
  }

  d1 = clamp(d1, 0, ceil.D1);
  d2 = clamp(d2, 0, ceil.D2);
  d3 = clamp(d3, 0, ceil.D3);
  d4 = clamp(d4, 0, ceil.D4);
  d5 = clamp(d5, 0, ceil.D5);
  d6 = clamp(d6, 0, ceil.D6);

  const total = d1 + d2 + d3 + d4 + d5 + d6;
  return {
    id: sample.id || sample._idx,
    group,
    platform,
    scores: { D1: d1, D2: d2, D3: d3, D4: d4, D5: d5, D6: d6 },
    total,
    passed: total >= BENCHMARK.passLine,
  };
}

function main(skillPath, datasetPath) {
  const evolveDir = __dirname;
  const skillMdPath = skillPath || process.argv[2] || path.join(evolveDir, 'SKILL_v1.md');
  const testPath = datasetPath || process.argv[3] || path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const trainPath = path.join(evolveDir, '..', 'digest', 'train.jsonl');

  const trainSamples = loadJsonl(trainPath);
  const testSamples = loadJsonl(testPath);
  const allSamples = [...trainSamples, ...testSamples];

  if (!allSamples.length) {
    console.error('No dataset found');
    process.exit(1);
  }

  const skillMd = fs.existsSync(skillMdPath) ? fs.readFileSync(skillMdPath, 'utf-8') : '';

  console.log(`Evaluating: ${path.basename(skillMdPath)}`);
  console.log(`Train: ${trainPath} (${trainSamples.length} samples)`);
  console.log(`Test: ${testPath} (${testSamples.length} samples)`);
  console.log(`Total: ${allSamples.length} samples\n`);

  const results = [];
  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };

  for (let i = 0; i < allSamples.length; i++) {
    const r = evaluateSample(skillMd, allSamples[i]);
    results.push(r);
    for (const k of Object.keys(dimTotals)) dimTotals[k] += r.scores[k];
    const mark = r.passed ? '✅' : '❌';
    const setLabel = i < trainSamples.length ? 'TRAIN' : 'TEST';
    console.log(`${mark} [${setLabel}] [${String(r.id).padEnd(18)}] ${r.platform.padEnd(12)} grp=${r.group.padEnd(22)} | T=${String(r.total).padStart(3)} | D1=${r.scores.D1} D2=${r.scores.D2} D3=${r.scores.D3} D4=${r.scores.D4} D5=${r.scores.D5} D6=${r.scores.D6}`);
  }

  const avgTotal = Number((results.reduce((a, b) => a + b.total, 0) / results.length).toFixed(2));
  const passCount = results.filter(r => r.passed).length;
  const totals = results.map(r => r.total);
  const spread = totals.length ? Math.max(...totals) - Math.min(...totals) : 0;
  const allSame = spread === 0;

  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = Number((dimTotals[k] / results.length).toFixed(2));
  }

  const globalPassed = avgTotal >= BENCHMARK.passLine && !allSame && spread >= 10;
  const iteration = process.argv[4] || 'v1';

  console.log(`\n--- Summary ---`);
  console.log(`Average Score: ${avgTotal} / 100`);
  console.log(`Pass Rate: ${passCount}/${results.length}`);
  console.log(`Spread: ${spread}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(`  ${dim.id} ${dim.name}: ${dimAvgs[dim.id]} / ${dim.max}`);
  }
  console.log(`\n${globalPassed ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  if (allSame) {
    console.error('\nERROR: All scores are identical.');
    process.exit(1);
  }
  if (spread < 10) {
    console.error(`\nWARNING: Score spread ${spread} < 10.`);
    process.exit(1);
  }

  const outFile = path.join(evolveDir, `iteration_${iteration}_scores.json`);
  const output = {
    iteration,
    skill_file: path.basename(skillMdPath),
    dataset: { train: trainPath, test: testPath },
    avg_score: avgTotal,
    pass_rate: `${passCount}/${results.length}`,
    global_passed: globalPassed,
    score_spread: spread,
    all_identical: allSame,
    scaler_method: 'benchmark-aligned rubric v3 (fixed D4, evaluates train+test)',
    dim_avgs: dimAvgs,
    scores: results.map(r => ({
      id: r.id,
      platform: r.platform,
      group: r.group,
      scores: r.scores,
      total: r.total,
      passed: r.passed,
    })),
  };
  fs.writeFileSync(outFile, JSON.stringify(output, null, 2));
  console.log(`\nScores written to ${outFile}`);

  return output;
}

module.exports = { main, evaluateSample, BENCHMARK };

if (require.main === module) {
  main();
}