宜博 / lemonboss

/**
 * trend-scout-global Scorer v1
 * Rule-based rubric scorer using train.jsonl + test.jsonl with per-case dimCeil.
 * Reads both datasets and saves unified iteration_scores_vN.json.
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: 'Cross-Platform Information Source Understanding', weight: 0.20, max: 20 },
    { id: 'D2', name: 'Trend Recognition & Theme Extraction', weight: 0.20, max: 20 },
    { id: 'D3', name: 'Data Cleaning & Structuring', weight: 0.20, max: 20 },
    { id: 'D4', name: 'Trend Grading & Forecasting', weight: 0.15, max: 15 },
    { id: 'D5', name: 'Report Output Specification', weight: 0.15, max: 15 },
    { id: 'D6', name: 'Exception Handling & Source Verification', weight: 0.10, max: 10 },
  ],
  passLine: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text.split('\n').map(l => l.trim()).filter(Boolean).map((l, idx) => {
    try { return { ...JSON.parse(l), _idx: idx }; } catch (e) { return null; }
  }).filter(Boolean);
}

function norm(text) {
  return (text || '').toLowerCase().replace(/[\s,;.:：\/\-_=+()\[\]{}|"']/g, '');
}

function has(skillNorm, kws) {
  return kws.some(k => skillNorm.includes(norm(k)));
}

function evaluateCase(skillMd, sample) {
  const mdNorm = norm(skillMd);
  const md = skillMd.toLowerCase();
  const input = sample.input || {};
  const expected = sample.expected_output || {};
  const dims = sample.dimensions || {};
  const hints = (sample.grader_hints || '').toLowerCase();

  // dimCeil derived from grader dimensions mapped to benchmark scale
  const ceil = {
    D1: Math.min(20, Math.round((dims['Cross-Platform Information Source Understanding'] || 5) * 2)),
    D2: Math.min(20, Math.round((dims['Trend Recognition & Theme Extraction'] || 5) * 2)),
    D3: Math.min(20, Math.round((dims['Data Cleaning & Structuring'] || 5) * 2)),
    D4: Math.min(15, Math.round((dims['Trend Grading & Forecasting'] || 5) * 1.5)),
    D5: Math.min(15, Math.round((dims['Report Output Specification'] || 5) * 1.5)),
    D6: Math.min(10, Math.round((dims['Exception Handling & Source Verification'] || 5) * 1)),
  };

  const rawEntry = input.raw_entry || {};
  const rawEntries = input.raw_entries || [];
  const platform = expected.platform || '';
  const hasMulti = rawEntries.length > 1;
  const hasStale = !!expected.stale_warning;
  const hasLowConf = expected.confidence === 'low';
  const hasUnknownPlat = platform === 'UNKNOWN';
  const hasConflictRes = !!expected.conflict_resolution;

  // ========== D1: Cross-Platform (ceil dynamic) ==========
  let d1Score = 0, d1Max = 0;
  d1Max += 4; if (has(mdNorm, ['platform', 'tiktok', 'instagram', 'youtube', 'x', 'pinterest'])) d1Score += 4;
  d1Max += 4; if (has(mdNorm, ['source_url', 'url'])) d1Score += 4;
  if (hints.includes('cross-language') || hints.includes('日本')) {
    d1Max += 4; if (has(mdNorm, ['ja-jp', 'japan', '日本版tiktok', 'cross-language'])) d1Score += 4;
  }
  if (hints.includes('region')) {
    d1Max += 4; if (has(mdNorm, ['region', 'brasil', 'brazil', 'jp', 'uk', 'de', 'in', 'ph', 'au', 'unknown'])) d1Score += 4;
  }
  if (hasUnknownPlat) {
    d1Max += 4; if (has(mdNorm, ['unknown', 'unrecognized', 'aggregator'])) d1Score += 4;
  }

  // ========== D2: Trend Recognition (ceil dynamic) ==========
  let d2Score = 0, d2Max = 0;
  d2Max += 5; if (has(mdNorm, ['name', 'hashtag', 'canonical'])) d2Score += 5;
  d2Max += 5; if (has(mdNorm, ['english', 'mapping', 'spanish', 'amor de vampiro', 'vampire love'])) d2Score += 5;
  d2Max += 5; if (has(mdNorm, ['related_hashtags', 'hashtags', 'related'])) d2Score += 5;
  d2Max += 5; if (has(mdNorm, ['ip_style', 'werewolf', 'vampire', 'mafia', 'revenge', 'billionaire', 'contract marriage', 'second chance', 'interactive story', 'vertical drama'])) d2Score += 5;

  // ========== D3: Data Cleaning (ceil dynamic) ==========
  let d3Score = 0, d3Max = 0;
  d3Max += 5; if (has(mdNorm, ['schema', 'required_fields'])) d3Score += 5;
  d3Max += 5; if (has(mdNorm, ['views_or_posts', 'numeric', 'parse', '1.3b', '24.5b'])) d3Score += 5;
  d3Max += 5; if (has(mdNorm, ['iso8601', 'crawl_time', 'timestamp'])) d3Score += 5;
  d3Max += 5; if (has(mdNorm, ['growth_7d', 'growth', 'percentage'])) d3Score += 5;

  // ========== D4: Trend Grading (ceil dynamic) ==========
  let d4Score = 0, d4Max = 0;
  d4Max += 5; if (has(mdNorm, ['rank', 'volume', 'prioritize'])) d4Score += 5;
  d4Max += 5; if (has(mdNorm, ['growth', 'breakout', 'plateau', 'declining'])) d4Score += 5;
  if (hasMulti || hasConflictRes) {
    d4Max += 5; if (has(mdNorm, ['conflict', 'resolution', 'fresher', 'crawl_time'])) d4Score += 5;
  } else {
    d4Max += 5; if (has(mdNorm, ['growth', 'forecasting'])) d4Score += 5;
  }

  // ========== D5: Report Output (ceil dynamic) ==========
  let d5Score = 0, d5Max = 0;
  d5Max += 5; if (has(mdNorm, ['report', 'json', 'markdown', 'table'])) d5Score += 5;
  d5Max += 5; if (has(mdNorm, ['summary', 'takeaway', 'actionable'])) d5Score += 5;
  d5Max += 5; if (has(mdNorm, ['platform', 'regional', 'highlights'])) d5Score += 5;

  // ========== D6: Exception Handling (ceil dynamic) ==========
  let d6Score = 0, d6Max = 0;
  d6Max += 3; if (has(mdNorm, ['confidence', 'low', 'unverified'])) d6Score += 3;
  d6Max += 3; if (has(mdNorm, ['fallback', 'default', 'null', 'missing'])) d6Score += 3;
  if (hasStale) {
    d6Max += 2; if (has(mdNorm, ['stale', 'warning', '7days'])) d6Score += 2;
  } else {
    d6Max += 2; if (has(mdNorm, ['fallback', 'graceful'])) d6Score += 2;
  }
  d6Max += 2; if (has(mdNorm, ['unknown', 'graceful', 'hallucination'])) d6Score += 2;

  const calc = (score, max, cap) => (max > 0 ? Math.min(cap, Math.round((score / max) * cap)) : 0);

  let D1 = calc(d1Score, d1Max, ceil.D1);
  let D2 = calc(d2Score, d2Max, ceil.D2);
  let D3 = calc(d3Score, d3Max, ceil.D3);
  let D4 = calc(d4Score, d4Max, ceil.D4);
  let D5 = calc(d5Score, d5Max, ceil.D5);
  let D6 = calc(d6Score, d6Max, ceil.D6);

  // Additional case-specific penalties for missing highly specific rules
  if (hasUnknownPlat && !has(mdNorm, ['unknown aggregator'])) D1 -= 2;
  if (hints.includes('stale') && !has(mdNorm, ['stale_warning'])) D6 -= 2;
  if (hasLowConf && !has(mdNorm, ['confidence: low'])) D6 -= 2;

  D1 = Math.max(0, Math.min(ceil.D1, D1));
  D2 = Math.max(0, Math.min(ceil.D2, D2));
  D3 = Math.max(0, Math.min(ceil.D3, D3));
  D4 = Math.max(0, Math.min(ceil.D4, D4));
  D5 = Math.max(0, Math.min(ceil.D5, D5));
  D6 = Math.max(0, Math.min(ceil.D6, D6));

  const total = D1 + D2 + D3 + D4 + D5 + D6;
  return {
    id: sample._idx + 1,
    platform: expected.platform || 'unknown',
    scores: { D1, D2, D3, D4, D5, D6 },
    total,
    passed: total >= BENCHMARK.passLine,
    reasoning: `ceil=${JSON.stringify(ceil)} d1=${d1Score}/${d1Max} d4=${d4Score}/${d4Max}`,
  };
}

function scoreDataset(skillMd, samples, label) {
  const results = [];
  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };

  for (let i = 0; i < samples.length; i++) {
    const r = evaluateCase(skillMd, samples[i]);
    results.push(r);
    for (const k of Object.keys(dimTotals)) dimTotals[k] += r.scores[k];
    const mark = r.passed ? 'PASS' : 'FAIL';
    console.log(`[${label}] ${mark} [${String(r.id).padStart(2)}] ${r.platform.padEnd(10)} | T=${String(r.total).padStart(2)} | D1=${r.scores.D1} D2=${r.scores.D2} D3=${r.scores.D3} D4=${r.scores.D4} D5=${r.scores.D5} D6=${r.scores.D6}`);
  }

  const avgTotal = results.length ? Number((results.reduce((a, b) => a + b.total, 0) / results.length).toFixed(2)) : 0;
  const passCount = results.filter(r => r.passed).length;
  const totals = results.map(r => r.total);
  const spread = totals.length ? Math.max(...totals) - Math.min(...totals) : 0;
  const allSame = spread === 0;

  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = results.length ? Number((dimTotals[k] / results.length).toFixed(2)) : 0;
  }

  return {
    count: results.length,
    avg_score: avgTotal,
    pass_rate: `${passCount}/${results.length}`,
    score_spread: spread,
    all_identical: allSame,
    dim_avgs: dimAvgs,
    scores: results.map(r => ({
      id: r.id,
      platform: r.platform,
      scores: r.scores,
      total: r.total,
      passed: r.passed,
      reasoning: r.reasoning,
    })),
  };
}

function main(skillPath, trainPath, testPath, iterationArg) {
  const evolveDir = __dirname;
  const skillMdPath = skillPath || process.argv[2] || path.join(evolveDir, 'SKILL_v1.md');
  const trainJsonl = trainPath || process.argv[3] || path.join(evolveDir, '..', 'digest', 'train.jsonl');
  const testJsonl = testPath || process.argv[4] || path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const iteration = iterationArg || process.argv[5] || 'v1';

  if (!fs.existsSync(trainJsonl)) {
    console.error('Train dataset not found:', trainJsonl);
    process.exit(1);
  }
  if (!fs.existsSync(testJsonl)) {
    console.error('Test dataset not found:', testJsonl);
    process.exit(1);
  }

  const trainSamples = loadJsonl(trainJsonl);
  const testSamples = loadJsonl(testJsonl);
  const skillMd = fs.existsSync(skillMdPath) ? fs.readFileSync(skillMdPath, 'utf-8') : '';

  console.log(`Evaluating: ${path.basename(skillMdPath)}`);
  console.log(`Train: ${trainJsonl} (${trainSamples.length} samples)`);
  console.log(`Test:  ${testJsonl} (${testSamples.length} samples)\n`);

  const trainResult = scoreDataset(skillMd, trainSamples, 'TRAIN');
  const testResult = scoreDataset(skillMd, testSamples, 'TEST');

  const combinedAvg = Number((
    (trainResult.avg_score * trainResult.count + testResult.avg_score * testResult.count)
    / (trainResult.count + testResult.count)
  ).toFixed(2));
  const combinedPassCount = parseInt(trainResult.pass_rate.split('/')[0], 10) + parseInt(testResult.pass_rate.split('/')[0], 10);
  const combinedTotalCount = trainResult.count + testResult.count;
  const globalPassed = combinedAvg >= BENCHMARK.passLine
    && !trainResult.all_identical && !testResult.all_identical
    && trainResult.score_spread >= 2 && testResult.score_spread >= 2;

  console.log(`\n--- Summary ---`);
  console.log(`Train Average: ${trainResult.avg_score} / 100`);
  console.log(`Test Average:  ${testResult.avg_score} / 100`);
  console.log(`Combined Average: ${combinedAvg} / 100`);
  console.log(`Combined Pass Rate: ${combinedPassCount}/${combinedTotalCount}`);
  console.log(`Train Spread: ${trainResult.score_spread} | Test Spread: ${testResult.score_spread}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(`  ${dim.id} ${dim.name}: train=${trainResult.dim_avgs[dim.id]} test=${testResult.dim_avgs[dim.id]} / ${dim.max}`);
  }
  console.log(`\n${globalPassed ? 'GLOBAL PASS' : 'GLOBAL FAIL'}`);

  if (trainResult.all_identical || testResult.all_identical) {
    console.error('\nERROR: At least one dataset has identical scores for all samples.');
    process.exit(1);
  }

  const outFile = path.join(evolveDir, `iteration_scores_${iteration}.json`);
  const output = {
    iteration,
    skill_file: path.basename(skillMdPath),
    train_dataset: trainJsonl,
    test_dataset: testJsonl,
    combined_average: combinedAvg,
    combined_pass_rate: `${combinedPassCount}/${combinedTotalCount}`,
    global_passed: globalPassed,
    train: trainResult,
    test: testResult,
    scaler_method: 'grader-dimCeil + skill keyword rubric v1',
  };
  fs.writeFileSync(outFile, JSON.stringify(output, null, 2));
  console.log(`\nScores written to ${outFile}`);

  return output;
}

module.exports = { main, evaluateCase, BENCHMARK, scoreDataset };

if (require.main === module) {
  main();
}