宜博 / lemonboss

/**
 * Clickbait Copywriter Scorer
 * 基于 benchmark_v1.md 对 test.jsonl 进行自动化评分
 *
 * 用法: node scorer.js [skill_md_path]
 * 逻辑: 使用本地 claude CLI 对 test.jsonl 中的 reference outputs 直接评分
 */

const fs = require('fs');
const path = require('path');
const { execFile } = require('child_process');
const os = require('os');

const BENCHMARK = {
  dimensions: [
    { id: 'ctr', name: 'CTR优化能力', max: 20 },
    { id: 'emotion', name: '情绪调动精准度', max: 20 },
    { id: 'platform_fit', name: '平台适配正确性', max: 20 },
    { id: 'rhetoric', name: '修辞技法丰富度', max: 15 },
    { id: 'compliance', name: '合规安全性', max: 15 },
    { id: 'creativity', name: '创意差异化', max: 10 },
  ],
  passLine: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .map((l, idx) => ({ ...JSON.parse(l), id: idx + 1 }));
}

function loadSkillMd(filePath) {
  if (!fs.existsSync(filePath)) return '';
  return fs.readFileSync(filePath, 'utf-8');
}

function buildScoringPrompt(cases) {
  const casesBlock = cases
    .map((c, i) => {
      const userContent = c.messages.find((m) => m.role === 'user')?.content || '';
      const assistantContent = c.messages.find((m) => m.role === 'assistant')?.content || '';
      return `--- CASE ${i + 1} ---\nUser Request:\n${userContent}\n\nAssistant Output:\n${assistantContent}\n`;
    })
    .join('\n');

  return `You are a senior cross-platform content marketing evaluator. Strictly score the following ${cases.length} clickbait copywriting outputs according to the 6 dimensions below. Each case has a user request and an assistant output.

Scoring Dimensions (total 100, passing line 95):
1. CTR Optimization (20 pts): Hook strength, information gap, 3-second attraction. Deductions: no clear hook (-5), info gap too vague (-4), first 10 chars no keyword/emotion (-3).
2. Emotional Precision (20 pts): Pain point resonance, emotion lever choice (curiosity/fear/greed/empathy/anger/social proof), immersion. Deductions: emotion mismatch with platform (-5), overuse of fear/anger causing反感 (-5), hollow emotion word stacking (-4).
3. Platform Fit (20 pts): Character limits, emoji/hashtag strategy, algorithm preference, red-line compliance. Deductions: banned superlatives (-8), Amazon with emoji or emotional hype (-6), character overflow (-4), wrong hashtag strategy (-3).
   Banned words (any = -8): 最, 第一, 绝对, 史上最强, 永久有效, 价格便宜; Amazon: Best Seller, Top Rated, #1 Choice, Amazing, On Sale, Free Shipping.
4. Rhetoric Richness (15 pts): Hyperbole, ellipsis, conflict, reversal, scarcity, identity label. 14-15 = 3+ techniques fused naturally; 11-13 = 2 techniques; 8-10 = 1 technique or forced.
5. Compliance Safety (15 pts): Zero superlatives, zero false info, no platform red lines. Health/medical claims must not overstep.
6. Creative Differentiation (10 pts): 10% Twist (unexpected element on proven formula), memorable, non-templatized. 9-10 = strong twist; 7-8 = weak twist; 5-6 = templatized.

Calibration examples from the benchmark standard:
- Example A (小红书): "怒花22w师从MGP，纠正四个误区" → ctr:19, emotion:18, platform_fit:19, rhetoric:14, compliance:15, creativity:9, total:94
- Example B (抖音): "我赌你没见过这么猛的祛痘方法" → ctr:19, emotion:19, platform_fit:18, rhetoric:13, compliance:14, creativity:8, total:91
- Example C (Amazon): "HydroCore Insulated Water Bottle — Stainless Steel, BPA-Free — 32 Oz — Keeps Drinks Cold 24 Hours..." → ctr:16, emotion:12, platform_fit:20, rhetoric:11, compliance:15, creativity:8, total:82

Please score consistently with these examples. Do not systematically deflate scores below the benchmark calibration.

${casesBlock}

Please output ONLY a JSON array with no markdown formatting. Each element must be:
{
  "case_id": number,
  "dimension_scores": {
    "ctr": 0-20,
    "emotion": 0-20,
    "platform_fit": 0-20,
    "rhetoric": 0-15,
    "compliance": 0-15,
    "creativity": 0-10
  },
  "total_score": 0-100,
  "passed": true/false,
  "feedback": "Brief: strongest point, weakest point, 1 actionable fix"
}

The response must be ONLY the JSON array, starting with [ and ending with ].`;
}

function callClaude(prompt) {
  return new Promise((resolve, reject) => {
    const tmpFile = path.join(os.tmpdir(), `cc_scorer_${Date.now()}.txt`);
    fs.writeFileSync(tmpFile, prompt, 'utf-8');
    execFile('bash', ['-c', `cat "${tmpFile}" | claude --print --dangerously-skip-permissions`], { timeout: 300000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
      try { fs.unlinkSync(tmpFile); } catch {}
      if (err) return reject(err);
      resolve(stdout.trim());
    });
  });
}

async function scoreCases(cases) {
  const prompt = buildScoringPrompt(cases);
  const raw = await callClaude(prompt);
  // Try to extract JSON array
  const match = raw.match(/\[[\s\S]*\]/);
  if (!match) throw new Error('No JSON array found in Claude response:\n' + raw);
  const parsed = JSON.parse(match[0]);
  return parsed;
}

async function main() {
  const evolveDir = __dirname;
  const testPath = process.argv[2] || path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const skillMdPath = process.argv[3] || path.join(evolveDir, 'SKILL_v1.md');

  if (!fs.existsSync(testPath)) {
    console.error('test.jsonl not found:', testPath);
    process.exit(1);
  }

  const cases = loadJsonl(testPath);
  console.log(`\nEvaluating ${cases.length} test cases with Claude CLI...\n`);

  const scores = await scoreCases(cases);

  let totalScore = 0;
  let passCount = 0;
  const dimTotals = { ctr: 0, emotion: 0, platform_fit: 0, rhetoric: 0, compliance: 0, creativity: 0 };

  for (const s of scores) {
    totalScore += s.total_score;
    if (s.passed) passCount++;
    for (const k of Object.keys(dimTotals)) {
      dimTotals[k] += s.dimension_scores[k];
    }
    const status = s.passed ? '✅' : '❌';
    console.log(`${status} [Case ${s.case_id}] Total: ${s.total_score} | CTR:${s.dimension_scores.ctr} EM:${s.dimension_scores.emotion} PF:${s.dimension_scores.platform_fit} RH:${s.dimension_scores.rhetoric} CO:${s.dimension_scores.compliance} CR:${s.dimension_scores.creativity}`);
    console.log(`   Feedback: ${s.feedback}`);
  }

  const avgTotal = totalScore / cases.length;
  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = dimTotals[k] / cases.length;
  }

  console.log(`\n--- Summary ---`);
  console.log(`Average Score: ${avgTotal.toFixed(2)} / 100`);
  console.log(`Pass Rate: ${passCount}/${cases.length}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(`  ${dim.name}: ${dimAvgs[dim.id].toFixed(2)} / ${dim.max}`);
  }

  const globalPassed = avgTotal >= BENCHMARK.passLine;
  console.log(`\n${globalPassed ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  const weakDims = BENCHMARK.dimensions
    .filter((d) => dimAvgs[d.id] < d.max * 0.85)
    .map((d) => d.name);
  if (weakDims.length) {
    console.log(`Weak dimensions (< 85% of max): ${weakDims.join(', ')}`);
  }

  // Write scores JSON
  const testBase = path.basename(testPath, '.jsonl');
  const outPath = path.join(evolveDir, `${testBase}_scores.json`);
  fs.writeFileSync(
    outPath,
    JSON.stringify(
      {
        iteration: Number(testBase.match(/iteration_(\d+)/)?.[1] || 1),
        skill_file: path.basename(skillMdPath),
        scores,
        summary: {
          avg_score: Number(avgTotal.toFixed(2)),
          pass_rate: `${passCount}/${cases.length}`,
          global_passed: globalPassed,
          weak_dimensions: weakDims,
        },
        dim_avgs: dimAvgs,
      },
      null,
      2
    )
  );
  console.log(`\nScores written to ${outPath}`);
  return { avgTotal, passCount, dimAvgs, globalPassed, scores };
}

module.exports = { main, BENCHMARK };

if (require.main === module) {
  main().catch((e) => {
    console.error(e);
    process.exit(1);
  });
}