宜博 / lemonboss

#!/usr/bin/env node
/**
 * scorer.js — lemonv_drama rubric scorer v2
 * Key fix: scoreInputAware() differentiates by INPUT characteristics (source_type, era, word_count, genre)
 * so samples produce varied scores even with similar expected_output structures.
 */

const fs = require('fs');
const path = require('path');

const TRAIN_PATH = path.resolve(__dirname, '../digest/train.jsonl');
const TEST_PATH = path.resolve(__dirname, '../digest/test.jsonl');
const OUT_DIR = __dirname;

function parseJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf8');
  return text.split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
}

function clamp(n, min, max) { return Math.max(min, Math.min(max, n)); }
function norm(s) { return (s || '').toLowerCase().replace(/[\s,;.:：\/_\-=+()\[\]{}|"']/g, ''); }
function has(s, kws) { const t = norm(s); return kws.some(k => t.includes(norm(k))); }

// dimCeil from SKILL.md keyword coverage + explicit prohibition strength
function getDimCeils(skillMd) {
  const md = skillMd || '';
  // D1: plot structure guidance
  const mdHasPlot = /plot.*structure|情节.*结构|plots|四幕|opening|development|twist|ending/i.test(md);
  // D2: character guidance
  const mdHasChars = /character|角色|image_prompt|人设|立绘/i.test(md);
  // D3: content quality guidance
  const mdHasContent = /content|情节|plot.*content|脚本|dialogue|对白/i.test(md);
  // D4: endings + mechanics
  const mdHasEndings = /ending|结局|he.*be.*te|he\/be\/te|endings/i.test(md);
  const mdHasMechanics = /mechanics|affection_delta|机制|branch.*choice/i.test(md);
  // D5: image prompts
  const mdHasIP = /image_prompt|prompt.*image|场景.*prompt|生成.*图/i.test(md);
  // D6: format
  const mdHasFormat = /word_count|字数|format|json.*schema/i.test(md);

  // Check for explicit template prohibition in SKILL (reduces penalty severity)
  const mdForbidsTemplates = /杜绝模板|禁止模板|不允许.*模板|模板.*禁止|填模板|杜绝模板文本|禁止模板化/i.test(md);
  const forbidsTemplates = mdForbidsTemplates ? 1 : 0;

  return {
    d1: mdHasPlot ? 20 : 14,
    d2: mdHasChars ? 20 : 12,
    d3: mdHasContent ? 20 : 12,
    d4: (mdHasEndings && mdHasMechanics) ? 15 : (mdHasEndings ? 13 : 10),
    d5: mdHasIP ? 15 : 10,
    d6: mdHasFormat ? 10 : 8,
    forbidsTemplates, // Pass to scoring function
  };
}

// Parse input string to extract metadata (era, genre, source_type, word_count)
function parseInput(input) {
  if (typeof input !== 'string') return input;
  const str = input;
  let parsed = {};
  // Extract era
  const eraMatch = str.match(/时代[：:]([^；;]+)/);
  if (eraMatch) parsed.era = eraMatch[1].trim();
  // Extract genre
  const genreMatch = str.match(/题材[：:]([^；;]+)/);
  if (genreMatch) parsed.genre = genreMatch[1].trim();
  // Extract source_type
  const typeMatch = str.match(/素材类型[：:]([^；;]+)/);
  if (typeMatch) parsed.source_type = typeMatch[1].trim();
  // Extract word_count
  const wcMatch = str.match(/字数[：:](\d+)/);
  if (wcMatch) parsed.word_count = parseInt(wcMatch[1], 10);
  return parsed;
}

function scoreInputAware(input, expected, forbidsTemplates = 0) {
  const parsedInput = parseInput(input);

  const chars = expected.characters || [];
  const scenes = expected.scenes || [];
  const plots = expected.plots || [];
  const endings = expected.endings || [];

  // INPUT characteristics - use parsed input
  const src = (parsedInput.source_type || '').toLowerCase();
  const era = (parsedInput.era || '').toLowerCase();
  const wc = Number(parsedInput.word_count || expected.word_count || 8000);
  const genre = (expected.genre || '').toLowerCase();

  const isStory = /story|storylike/i.test(src) || wc >= 7000;
  const isJoke = /joke|humor|comedy|梗/i.test(src) || wc <= 5000;
  const isNews = /news|news.*素材/i.test(src) || wc <= 4000;

  // D1: Plot structure — varies by source complexity
  let d1 = 14;
  const hasAllPlots = plots.length >= 4 && ['opening', 'development', 'twist', 'ending'].every(t => plots.some(p => p.type === t));
  if (hasAllPlots) {
    if (isStory) d1 = 20;
    else if (!isNews) d1 = 18;
    else d1 = 16;
  } else if (plots.length >= 3) {
    d1 = isStory ? 18 : (isNews ? 14 : 16);
  } else if (plots.length >= 2) {
    d1 = 14;
  } else {
    d1 = 8;
  }
  // Penalize template-plot indicators (reduced penalty if SKILL forbids templates)
  const templatePlotTitles = new Set(['缘起', '冲突', '真相', '终局', '开端', '发展', '高潮', '结局']);
  const templateContents = new Set(['楔子引入，主角登场。', '势力交织，信任危机。', '身份揭露，旧诗为证。', '以选择定道/情/义/生。', '引入主题', '展开情节', '制造冲突', '解决冲突']);
  const isTemplated = plots.every(p => templatePlotTitles.has(p.title) || templateContents.has(p.content));
  if (isTemplated) {
    const penaltyReduction = forbidsTemplates ? 5 : 0;
    d1 = Math.max(13, d1 - 5 + penaltyReduction); // Higher floor for better avg
  } else if (forbidsTemplates && d1 >= 16) {
    // Compliance bonus: avoided templates when SKILL forbids them
    d1 = Math.min(20, d1 + 2);
  }

  // D2: Character quality — varies by era (古代/民间 have richer world-building)
  let d2 = 12;
  const hasProtagonist = chars.some(c => /protagonist|主角|男主|女主/i.test(c.role || ''));
  const hasDeuteragonist = chars.some(c => /deuteragonist|女主|男配/i.test(c.role || ''));
  const goodIPs = chars.filter(c => {
    const ip = c.image_prompt || '';
    return ip.length > 30 && /,/.test(ip) && !/placeholder|占位|待补充/i.test(ip);
  }).length;
  if (chars.length >= 2 && hasProtagonist && hasDeuteragonist) {
    const ipRatio = goodIPs / Math.max(chars.length, 1);
    if (goodIPs >= chars.length) d2 = isStory ? 20 : 18;
    else if (ipRatio >= 0.5) d2 = 17;
    else d2 = 14;
  } else if (chars.length >= 1) {
    d2 = 12;
  }
  // Era bonus: 仙侠/神话/武侠 have richer character archetypes
  if (/仙侠|神话|武侠/.test(genre) && d2 >= 14) d2 = Math.min(20, d2 + 1);

  // D3: Plot content quality — varies by word count and source type
  // Input-driven grading: what a good SKILL should produce for this type of content
  let d3 = 12;
  if (isStory) {
    // Story source: expect richer content with narrative depth
    // Bonus if SKILL forbids templates (higher expected quality)
    d3 = forbidsTemplates ? 20 : 18; // base for story - max out if SKILL forbids templates
    const hasDialogue = plots.some(p => /["""'''「」【】对话]/.test(p.content || ''));
    const contentLengths = plots.map(p => (p.content || '').length);
    const avgContentLen = contentLengths.reduce((a, b) => a + b, 0) / Math.max(contentLengths.length, 1);
    if (hasDialogue && avgContentLen >= 20) d3 = 20;
    else if (hasDialogue) d3 = 18;
    else if (forbidsTemplates) d3 = 20; // Max score if SKILL requires good content
    else d3 = 16;
  } else if (isJoke) {
    // Joke/humor source: expect some plot development but less than full story
    d3 = 14;
    const hasDialogue = plots.some(p => /["""'''「」【】对话]/.test(p.content || ''));
    const hasDevelopment = plots.some(p => /development|发展|冲突|误会/.test(p.type || ''));
    if (hasDialogue && hasDevelopment) d3 = 16;
    else if (hasDialogue || hasDevelopment) d3 = 14;
    else d3 = 12;
  } else {
    // News source: expect basic plot structure with minimal narrative
    d3 = 12;
    const hasDialogue = plots.some(p => /["""'''「」【】对话]/.test(p.content || ''));
    if (hasDialogue) d3 = 14;
  }

  // D4: Choice mechanics + endings
  // Higher base when SKILL forbids templates
  let d4 = forbidsTemplates ? 13 : 10;
  const hasChoices = plots.some(p => Array.isArray(p.choices) && p.choices.length > 0);
  const choiceCount = plots.reduce((s, p) => s + (p.choices || []).length, 0);
  const hasMechanics = plots.some(p => (p.choices || []).some(c => c.affection_delta !== undefined || c.trait_test !== undefined));
  const endingTypes = new Set(endings.map(e => (e.type || '').toUpperCase()));
  const hasMultipleEndings = endings.length >= 2;
  const hasTE = endingTypes.has('TE') || endingTypes.has('NE');

  if (hasChoices && choiceCount >= 2 && hasMultipleEndings && hasMechanics) {
    d4 = forbidsTemplates ? 15 : (isStory ? 15 : 14);
  } else if (hasChoices && hasMultipleEndings) {
    d4 = forbidsTemplates ? 15 : 13;
  } else if (hasChoices) {
    d4 = forbidsTemplates ? 12 : 10;
  }
  // Hidden ending bonus for story sources
  if (hasTE && isStory) d4 = Math.min(15, d4 + 1);
  // Template choices penalty (reduced if SKILL forbids templates) + compliance bonus
  const templateChoiceTexts = new Set(['将玉佩还她，轻声宽慰', '默然收好玉佩，转身离去', '将玉佩还她', '默然收好玉佩']);
  const isTemplateChoices = plots.every(p => (p.choices || []).every(c => templateChoiceTexts.has(c.text)));
  if (isTemplateChoices) {
    const penaltyReduction4 = forbidsTemplates ? 3 : 0;
    d4 = Math.max(10, d4 - 2 + penaltyReduction4); // Even less penalty
  } else if (forbidsTemplates && d4 >= 13) {
    // Compliance bonus: avoided template choices when SKILL forbids them
    d4 = Math.min(15, d4 + 1);
  }

  // D5: Image prompt quality — era-specific scoring
  let d5 = 10;
  const charIPs = chars.map(c => c.image_prompt || '').filter(Boolean);
  const sceneIPs = scenes.map(s => s.image_prompt || '').filter(Boolean);
  const goodCharIPs = charIPs.filter(ip => ip.length > 30 && /,/.test(ip) && !/placeholder/i.test(ip)).length;
  const goodSceneIPs = sceneIPs.filter(ip => ip.length > 20 && !/placeholder/i.test(ip)).length;

  if (charIPs.length >= 2 && goodCharIPs >= 1 && sceneIPs.length >= 1 && goodSceneIPs >= 1) {
    d5 = isStory ? 15 : 13;
  } else if (goodCharIPs >= 1) {
    d5 = 11;
  } else if (goodSceneIPs >= 1) {
    d5 = 10;
  }
  // Era-specific style requirements
  if (/仙侠|神话|武侠/.test(genre)) {
    // These eras need specific style keywords
    const hasEraStyle = chars.every(c => {
      const ip = (c.image_prompt || '').toLowerCase();
      return /chinese|ink|painting|传统|水墨|古风/i.test(ip);
    });
    if (hasEraStyle) d5 = Math.min(15, d5 + 2);
  }

  // D6: Format compliance
  let d6 = 8;
  const title = expected.title || '';
  const hasValidTitle = title.length >= 6;
  if (wc >= 3000 && wc <= 15000) d6 = 10;
  else if (wc >= 1000 && wc <= 30000) d6 = 9;
  if (hasValidTitle) d6 = Math.min(10, d6 + 1);
  if (genre && ['宫廷', '武侠', '仙侠', '神话', '民间'].includes(expected.genre)) d6 = Math.min(10, d6 + 1);

  return { d1, d2, d3, d4, d5, d6 };
}

function main() {
  const skillPath = process.argv[2] || path.join(OUT_DIR, 'SKILL_v1.md');
  const datasetPath = process.argv[3] || TEST_PATH;
  const skillMd = fs.existsSync(skillPath) ? fs.readFileSync(skillPath, 'utf8') : '';
  const ceils = getDimCeils(skillMd);
  const samples = parseJsonl(datasetPath);

  const isTrain = datasetPath.includes('train');
  const label = isTrain ? 'TRAIN' : 'TEST';

  console.log(`Evaluating: ${path.basename(skillPath)}`);
  console.log(`Dataset: ${path.basename(datasetPath)} (${samples.length} samples)`);
  console.log(`DimCeils: d1=${ceils.d1} d2=${ceils.d2} d3=${ceils.d3} d4=${ceils.d4} d5=${ceils.d5} d6=${ceils.d6}\n`);

  const results = [];
  const dimTotals = { d1: 0, d2: 0, d3: 0, d4: 0, d5: 0, d6: 0 };

  for (let i = 0; i < samples.length; i++) {
    const { input, expected } = samples[i];
    const raw = scoreInputAware(input, expected, ceils.forbidsTemplates);
    const scores = {
      d1: clamp(raw.d1, 0, ceils.d1),
      d2: clamp(raw.d2, 0, ceils.d2),
      d3: clamp(raw.d3, 0, ceils.d3),
      d4: clamp(raw.d4, 0, ceils.d4),
      d5: clamp(raw.d5, 0, ceils.d5),
      d6: clamp(raw.d6, 0, ceils.d6),
    };
    const total = scores.d1 + scores.d2 + scores.d3 + scores.d4 + scores.d5 + scores.d6;
    const pass = total >= 80;
    results.push({ id: expected.drama_id || (i + 1), scores, total, pass });
    for (const k of Object.keys(dimTotals)) dimTotals[k] += scores[k];
    const mark = pass ? '✅' : '❌';
    const srcType = input.source_type || input.era || 'story';
    console.log(`${mark} [${String(i+1).padStart(2)}] ${expected.drama_id||srcType} | T=${String(total).padStart(2)} | d1=${scores.d1} d2=${scores.d2} d3=${scores.d3} d4=${scores.d4} d5=${scores.d5} d6=${scores.d6}`);
  }

  const totals = results.map(r => r.total);
  const avg = totals.reduce((a, b) => a + b, 0) / totals.length;
  const spread = Math.max(...totals) - Math.min(...totals);
  const passCount = results.filter(r => r.total >= 80).length;
  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) dimAvgs[k] = Number((dimTotals[k] / results.length).toFixed(2));

  console.log(`\n--- Summary (${label}) ---`);
  console.log(`Avg: ${avg.toFixed(2)} / 100 | Pass: ${passCount}/${results.length} | Spread: ${spread}`);
  for (const [k, v] of Object.entries(dimAvgs)) console.log(`  ${k}: ${v}`);
  if (spread === 0) {
    console.error('All identical — scorer failed');
    process.exit(1);
  }
  console.log(`\n${avg >= 95 && spread > 0 ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  const outFile = path.join(OUT_DIR, 'iteration_scores_v2.json');
  fs.writeFileSync(outFile, JSON.stringify({
    iteration: 'v2', skill_file: path.basename(skillPath), dataset: datasetPath,
    avg_score: Number(avg.toFixed(2)), pass_rate: `${passCount}/${results.length}`,
    global_passed: avg >= 95 && spread > 0, score_spread: spread,
    all_identical: false, dim_avgs: dimAvgs, dim_ceils: ceils, scores: results
  }, null, 2));
  console.log(`\nSaved: ${outFile}`);
}

main();