scorer.js 12.4 KB
const fs = require('fs');
const path = require('path');

const TRAIN_PATH = path.resolve(__dirname, '../digest/train.jsonl');
const OUT_PATH = path.resolve(__dirname, 'iteration_scores_v1.json');

function loadTrainData() {
  const text = fs.readFileSync(TRAIN_PATH, 'utf-8');
  const lines = text.split(/\r?\n/).filter(l => l.trim());
  return lines.map((line, idx) => {
    try {
      return JSON.parse(line);
    } catch (e) {
      throw new Error(`Parse error at line ${idx + 1}: ${e.message}`);
    }
  });
}

function countWords(str) {
  if (!str) return 0;
  const c = str.trim();
  if (!c) return 0;
  const cn = (c.match(/[\u4e00-\u9fa5]/g) || []).length;
  const en = (c.match(/[a-zA-Z]+/g) || []).length;
  return cn + en;
}

function hasField(obj, ...keys) {
  let cur = obj;
  for (const k of keys) {
    if (cur == null || typeof cur !== 'object') return false;
    if (!(k in cur)) return false;
    cur = cur[k];
  }
  return true;
}

function getField(obj, ...keys) {
  let cur = obj;
  for (const k of keys) {
    if (cur == null || typeof cur !== 'object') return undefined;
    cur = cur[k];
  }
  return cur;
}

function isNonEmptyArray(v) {
  return Array.isArray(v) && v.length > 0;
}

function isNonEmptyString(v) {
  return typeof v === 'string' && v.trim().length > 0;
}

// ---------- Dimension 1: Structure Integrity (25%) ----------
function scoreStructureIntegrity(item) {
  const out = item.expected_output;
  let score = 0;
  const checks = {
    hook: isNonEmptyString(out.hook) || isNonEmptyString(out.title),
    sellingOrComps: isNonEmptyArray(out.selling_points) || isNonEmptyString(out.comps),
    world_building: hasField(out, 'world_building', 'era') && hasField(out, 'world_building', 'rules'),
    characters: isNonEmptyArray(out.characters) && out.characters.length >= 2,
    acts: hasField(out, 'acts', 'act1') && hasField(out, 'acts', 'act2') && hasField(out, 'acts', 'act3'),
    themes: isNonEmptyArray(out.themes),
    highlights: isNonEmptyArray(out.highlights),
    word_count_plan: isNonEmptyString(out.word_count_plan) || (out.target_word_count && out.target_word_count > 0),
    threeActDetail: hasField(out, 'acts', 'act1', 'summary') && hasField(out, 'acts', 'act2', 'summary') && hasField(out, 'acts', 'act3', 'summary'),
    beats: hasField(out, 'acts', 'act1', 'key_beats') && hasField(out, 'acts', 'act2', 'key_beats') && hasField(out, 'acts', 'act3', 'key_beats'),
  };

  const moduleKeys = ['hook', 'sellingOrComps', 'world_building', 'characters', 'acts', 'themes', 'highlights', 'word_count_plan'];
  const modulePass = moduleKeys.filter(k => checks[k]).length;
  score += (modulePass / moduleKeys.length) * 15;

  if (checks.threeActDetail && checks.beats) {
    const act1Beats = (getField(out, 'acts', 'act1', 'key_beats') || []).length;
    const act2Beats = (getField(out, 'acts', 'act2', 'key_beats') || []).length;
    const act3Beats = (getField(out, 'acts', 'act3', 'key_beats') || []).length;
    const totalBeats = act1Beats + act2Beats + act3Beats;
    if (totalBeats >= 10) score += 7;
    else if (totalBeats >= 7) score += 5;
    else if (totalBeats >= 4) score += 3;
    else score += 1;
  } else if (checks.acts) {
    score += 2;
  }

  if (modulePass >= 7 && checks.threeActDetail && checks.beats) {
    score += 3;
  } else if (modulePass >= 6) {
    score += 1.5;
  }

  return Math.min(25, Math.max(0, score));
}

// ---------- Dimension 2: Three-Act & Beat Logic (20%) ----------
function scoreThreeActBeatLogic(item) {
  const out = item.expected_output;
  let score = 0;

  const act1 = getField(out, 'acts', 'act1') || {};
  const act2 = getField(out, 'acts', 'act2') || {};
  const act3 = getField(out, 'acts', 'act3') || {};
  const a1b = Array.isArray(act1.key_beats) ? act1.key_beats : [];
  const a2b = Array.isArray(act2.key_beats) ? act2.key_beats : [];
  const a3b = Array.isArray(act3.key_beats) ? act3.key_beats : [];
  const allBeats = [...a1b, ...a2b, ...a3b];

  const beatCount = allBeats.length;
  if (beatCount >= 12) score += 10;
  else if (beatCount >= 9) score += 8;
  else if (beatCount >= 6) score += 6;
  else if (beatCount >= 4) score += 4;
  else if (beatCount >= 2) score += 2;

  const beatText = allBeats.join(' ').toLowerCase();
  const structuralTerms = ['触发', '事件', 'catalyst', 'break', '中点', 'midpoint', '高潮', 'climax', '灵魂黑夜', 'dark night', '一无所有', 'all is lost', '终场', 'finale', 'b故事', 'b story', '进入第二幕', '进入第三幕'];
  let termHits = structuralTerms.filter(t => beatText.includes(t.toLowerCase())).length;
  if (termHits >= 6) score += 6;
  else if (termHits >= 4) score += 4.5;
  else if (termHits >= 2) score += 3;
  else score += 1.5;

  const hasAct1 = a1b.length >= 2;
  const hasAct2 = a2b.length >= 3;
  const hasAct3 = a3b.length >= 2;
  if (hasAct1 && hasAct2 && hasAct3) score += 4;
  else if ((hasAct1 && hasAct2) || (hasAct2 && hasAct3)) score += 2.5;
  else score += 1;

  return Math.min(20, Math.max(0, score));
}

// ---------- Dimension 3: Character Depth (20%) ----------
function scoreCharacterDepth(item) {
  const chars = item.expected_output.characters || [];
  let score = 0;

  const protag = chars.find(c => c.role === 'protagonist') || chars[0];
  if (protag) {
    let p = 0;
    if (isNonEmptyString(protag.want)) p += 2;
    if (isNonEmptyString(protag.need)) p += 2;
    if (isNonEmptyString(protag.flaw)) p += 2;
    if (isNonEmptyString(protag.fear) || isNonEmptyString(protag.secret)) p += 2;
    score += p;
  }

  const supports = chars.filter(c => c && c.role !== 'protagonist');
  let supportScore = 0;
  for (const c of supports.slice(0, 3)) {
    let s = 0;
    if (isNonEmptyString(c.want)) s += 0.8;
    if (isNonEmptyString(c.need) || isNonEmptyString(c.motive)) s += 0.8;
    if (isNonEmptyString(c.flaw)) s += 0.7;
    if (isNonEmptyString(c.relation_to_protagonist) || isNonEmptyString(c.opposition)) s += 0.7;
    supportScore += s;
  }
  score += Math.min(7, supportScore);

  const hasAntagonist = chars.some(c => (c.role || '').includes('antagonist'));
  const hasMentor = chars.some(c => (c.role || '').includes('mentor'));
  const hasLoveInterest = chars.some(c => (c.role || '').includes('love_interest'));
  const hasFoil = chars.some(c => (c.role || '').includes('foil'));
  if (hasAntagonist) score += 2;
  if (hasMentor || hasLoveInterest) score += 1.5;
  if (hasFoil) score += 1.5;

  return Math.min(20, Math.max(0, score));
}

// ---------- Dimension 4: Genre Convention Fit (15%) ----------
function scoreGenreFit(item) {
  const out = item.expected_output;
  const genre = (item.input.genre || '').toLowerCase();
  let score = 0;

  const textPool = [
    out.title, out.hook,
    (out.selling_points || []).join(' '),
    out.comps,
    (out.highlights || []).join(' '),
    (out.themes || []).join(' ')
  ].filter(Boolean).join(' ').toLowerCase();

  const genreKeywords = {
    '玄幻': ['修炼', '境界', '血脉', '天道', '功法', '大陆', '世界', '逆天', '升级', '强者'],
    '虐恋': ['虐', '恨', '爱', '误会', '复仇', '火葬场', '情感', '家族', '伤害', '原谅'],
    '宫斗': ['宫', '后宫', '妃', '皇后', '医术', '权谋', '嫡庶'],
    '悬疑': ['案件', '凶手', '真相', '侦查', '推理', '命案', '犯罪', '线索', '侧写', '悬疑'],
    '甜宠': ['甜', '宠', '恋爱', '双箭头', '青梅竹马', '重逢', '撒糖', '高甜', '办公室'],
    '科幻': ['飞船', '未来', '科技', '文明', '太空', '人类', '生态', '伦理', '末日', '星际'],
    '系统': ['系统', '好感度', '攻略', '穿书', '剧情', '原著', '反派', '任务', '蝴蝶'],
    '无限流': ['副本', '游戏', '通关', '惊悚', '恐怖', '直播', '观众', 'npc', '主脑', '剧场'],
    '历史': ['朝堂', '改革', '明朝', '万历', '历史', '科举', '权谋', '赋税', '制度', '王朝'],
    '异能': ['异能', '觉醒', '能力', '超凡', '副本', '倒计时', '怪物', '灵潮', '末日'],
    '电竞': ['电竞', '比赛', '战队', '夺冠', '联赛', 'moba', '团战', 'bp', '职业', '冠军']
  };

  let matched = 0;
  let totalKw = 0;
  for (const [g, kws] of Object.entries(genreKeywords)) {
    if (genre.includes(g) || (genre.includes('游戏') && g === '无限流') || (genre.includes('电竞') && g === '电竞')) {
      totalKw = kws.length;
      matched = kws.filter(kw => textPool.includes(kw)).length;
      break;
    }
  }
  if (totalKw > 0) {
    score += (matched / totalKw) * 6;
  } else {
    score += 4;
  }

  const rules = getField(out, 'world_building', 'rules') || [];
  if (Array.isArray(rules) && rules.length >= 2) score += 3;
  else if (Array.isArray(rules) && rules.length >= 1) score += 1.5;

  const era = getField(out, 'world_building', 'era') || '';
  if (isNonEmptyString(era)) score += 2;

  if (isNonEmptyString(out.comps)) score += 2;
  if (isNonEmptyArray(out.selling_points) && out.selling_points.length >= 2) score += 2;

  return Math.min(15, Math.max(0, score));
}

// ---------- Dimension 5: Output Completeness & Format (10%) ----------
function scoreCompletenessAndFormat(item) {
  const out = item.expected_output;
  let score = 0;

  const sections = ['title', 'hook', 'world_building', 'characters', 'acts', 'themes'];
  let present = 0;
  for (const s of sections) {
    if (out[s] != null && (typeof out[s] !== 'string' || out[s].trim())) present++;
  }
  if (isNonEmptyArray(out.highlights)) present++;
  if (isNonEmptyString(out.word_count_plan) || (out.target_word_count && out.target_word_count > 0)) present++;
  score += (present / 8) * 5;

  const hasSchema = typeof out === 'object' && out !== null && !Array.isArray(out);
  if (hasSchema) score += 3;

  const summaries = [
    getField(out, 'acts', 'act1', 'summary'),
    getField(out, 'acts', 'act2', 'summary'),
    getField(out, 'acts', 'act3', 'summary')
  ];
  const summaryWords = summaries.filter(Boolean).map(s => countWords(s)).reduce((a, b) => a + b, 0);
  if (summaryWords > 80) score += 2;
  else if (summaryWords > 40) score += 1;

  return Math.min(10, Math.max(0, score));
}

// ---------- Dimension 6: Hook & Commercial Appeal (10%) ----------
function scoreHookAndAppeal(item) {
  const out = item.expected_output;
  let score = 0;

  const hook = out.hook || '';
  const hookWords = countWords(hook);
  if (hookWords >= 10 && hookWords <= 40) score += 3;
  else if (hookWords > 0) score += 1.5;
  if (/[!。?]/g.test(hook) && hookWords > 5) score += 1;

  const sp = out.selling_points || [];
  if (sp.length >= 3) score += 2;
  else if (sp.length >= 1) score += 1;
  if (isNonEmptyString(out.comps)) score += 1;

  const hl = out.highlights || [];
  if (hl.length >= 3) score += 2;
  else if (hl.length >= 1) score += 1;
  if (isNonEmptyString(out.word_count_plan) || (out.target_word_count && out.target_word_count > 0)) score += 1;

  return Math.min(10, Math.max(0, score));
}

function scoreCase(item, index) {
  const d1 = scoreStructureIntegrity(item);
  const d2 = scoreThreeActBeatLogic(item);
  const d3 = scoreCharacterDepth(item);
  const d4 = scoreGenreFit(item);
  const d5 = scoreCompletenessAndFormat(item);
  const d6 = scoreHookAndAppeal(item);

  const total = d1 + d2 + d3 + d4 + d5 + d6;
  return {
    index: index + 1,
    genre: item.input.genre,
    dimensions: {
      structure_integrity: round2(d1),
      three_act_beat_logic: round2(d2),
      character_depth: round2(d3),
      genre_convention_fit: round2(d4),
      output_completeness_format: round2(d5),
      hook_commercial_appeal: round2(d6)
    },
    total: round2(total)
  };
}

function round2(n) {
  return Math.round(n * 100) / 100;
}

function main() {
  const data = loadTrainData();
  const results = [];
  for (let i = 0; i < data.length; i++) {
    results.push(scoreCase(data[i], i));
  }

  const totals = results.map(r => r.total);
  const avg = totals.reduce((a, b) => a + b, 0) / totals.length;
  const min = Math.min(...totals);
  const max = Math.max(...totals);
  const spread = max - min;

  const output = {
    cases: results,
    summary: {
      count: results.length,
      average: round2(avg),
      min: round2(min),
      max: round2(max),
      spread: round2(spread)
    }
  };

  fs.writeFileSync(OUT_PATH, JSON.stringify(output, null, 2));
  console.log(`Scored ${results.length} cases.`);
  console.log(`Average: ${output.summary.average}`);
  console.log(`Min: ${output.summary.min}`);
  console.log(`Max: ${output.summary.max}`);
  console.log(`Spread: ${output.summary.spread}`);
  if (output.summary.average < 95) {
    console.log('WARNING: Average score below 95. Small loop needed.');
    process.exitCode = 1;
  }
}

main();