宜博 / lemonboss

/**
 * Community Manager Scorer v1
 * 基于 benchmark_v1.md 对 test.jsonl 进行自动化评分
 *
 * 用法: node scorer.js [skill_md_path]
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: '平台规则掌握度', max: 15 },
    { id: 'D2', name: '评论/私信互动能力', max: 15 },
    { id: 'D3', name: '马甲运营能力', max: 15 },
    { id: 'D4', name: '社区活动策划能力', max: 15 },
    { id: 'D5', name: '危机处理与风控意识', max: 15 },
    { id: 'D6', name: '综合专业度', max: 25 },
  ],
  passLine: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .map((l, idx) => ({ ...JSON.parse(l), id: idx + 1 }));
}

function loadSkillMd(filePath) {
  if (!fs.existsSync(filePath)) return '';
  return fs.readFileSync(filePath, 'utf-8');
}

function scoreCase(skillMd, caseObj) {
  const md = skillMd.toLowerCase();
  const input = (caseObj.input || '').toLowerCase();
  const expected = caseObj.expected_output || {};

  // D1: 平台规则掌握度
  let d1 = 15;
  const hasXiaohongshu = md.includes('小红书') && md.includes('导流');
  const hasWeibo = md.includes('微博') && md.includes('50条');
  const hasDouyin = md.includes('抖音') && md.includes('ai fingerprint');
  const hasBilibili = md.includes('b站') && md.includes('社区公约');
  const hasAigc = md.includes('ai标识') || md.includes('aigc');
  const hasMedical = md.includes('医美') && md.includes('资质认证');
  if (!hasXiaohongshu) d1 -= 3;
  if (!hasWeibo) d1 -= 2;
  if (!hasDouyin) d1 -= 3;
  if (!hasBilibili) d1 -= 2;
  if (!hasAigc) d1 -= 2;
  if (!hasMedical) d1 -= 2;

  // D2: 评论/私信互动能力
  let d2 = 15;
  const has5Steps = md.includes('差评转化5步法');
  const has3Principles = md.includes('私信高回复率3原则');
  const hasGolden2H = md.includes('黄金2小时');
  const has4Types = md.includes('四类评论');
  const has24H = md.includes('24小时内');
  if (!has5Steps) d2 -= 4;
  if (!has3Principles) d2 -= 3;
  if (!hasGolden2H) d2 -= 2;
  if (!has4Types) d2 -= 3;
  if (!has24H) d2 -= 2;

  // D3: 马甲运营能力
  let d3 = 15;
  const hasMatrix = md.includes('1主号') && md.includes('3垂类号');
  const hasDevice = md.includes('设备隔离');
  const hasIp = md.includes('ip隔离');
  const hasBehavior = md.includes('行为仿真');
  const has7Day = md.includes('7天养号');
  const has4Funnels = md.includes('四类自然引流话术');
  if (!hasMatrix) d3 -= 3;
  if (!hasDevice) d3 -= 3;
  if (!hasIp) d3 -= 3;
  if (!hasBehavior) d3 -= 2;
  if (!has7Day) d3 -= 2;
  if (!has4Funnels) d3 -= 2;

  // D4: 社区活动策划能力
  let d4 = 15;
  const has10Steps = md.includes('10步sop');
  const has4Plays = md.includes('问答') && md.includes('投票') && md.includes('抽奖') && md.includes('裂变');
  const hasSchedule = md.includes('时间表');
  const hasTools = md.includes('腾讯问卷') || md.includes('工具选择');
  const hasAntiCheat = md.includes('防刷票') || md.includes('防作弊');
  const hasContingency = md.includes('应急预案');
  if (!has10Steps) d4 -= 3;
  if (!has4Plays) d4 -= 3;
  if (!hasSchedule) d4 -= 2;
  if (!hasTools) d4 -= 2;
  if (!hasAntiCheat) d4 -= 2;
  if (!hasContingency) d4 -= 2;

  // D5: 危机处理与风控意识
  let d5 = 15;
  const hasCrisisSop = md.includes('危机应对sop');
  const hasRecovery = md.includes('恢复周期');
  const hasMonitor = md.includes('账号健康度监控');
  const hasWaterArmy = md.includes('水军') && md.includes('举报');
  const hasLimitFlow = md.includes('限流');
  if (!hasCrisisSop) d5 -= 3;
  if (!hasRecovery) d5 -= 3;
  if (!hasMonitor) d5 -= 3;
  if (!hasWaterArmy) d5 -= 3;
  if (!hasLimitFlow) d5 -= 2;

  // D6: 综合专业度
  let d6 = 25;
  const hasKpis = md.includes('≥80%') && md.includes('≤30分钟') && md.includes('≥5%');
  const hasRetention = md.includes('7日留存率') && md.includes('≥40%');
  const hasFormula = md.includes('计算公式');
  const hasBenchmark = md.includes('行业基准');
  const has6D = md.includes('六维覆盖') || md.includes('六维 awareness');
  const hasChecklist = md.includes('自检清单');
  if (!hasKpis) d6 -= 4;
  if (!hasRetention) d6 -= 4;
  if (!hasFormula) d6 -= 3;
  if (!hasBenchmark) d6 -= 3;
  if (!has6D) d6 -= 4;
  if (!hasChecklist) d6 -= 3;

  // 确保分数在合理范围内
  d1 = Math.max(0, Math.min(15, d1));
  d2 = Math.max(0, Math.min(15, d2));
  d3 = Math.max(0, Math.min(15, d3));
  d4 = Math.max(0, Math.min(15, d4));
  d5 = Math.max(0, Math.min(15, d5));
  d6 = Math.max(0, Math.min(25, d6));

  const total = d1 + d2 + d3 + d4 + d5 + d6;

  return {
    id: caseObj.id,
    case: (caseObj.input || '').slice(0, 60) + '...',
    scores: { D1: d1, D2: d2, D3: d3, D4: d4, D5: d5, D6: d6 },
    total,
    passed: total >= BENCHMARK.passLine,
  };
}

function main() {
  const evolveDir = __dirname;
  const testPath = path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const skillMdPath = process.argv[2] || path.join(evolveDir, 'SKILL_v1.md');

  if (!fs.existsSync(testPath)) {
    console.error('test.jsonl not found:', testPath);
    process.exit(1);
  }

  const cases = loadJsonl(testPath);
  const skillMd = loadSkillMd(skillMdPath);

  console.log(`\nEvaluating SKILL.md: ${skillMdPath}`);
  console.log(`Cases: ${cases.length}\n`);

  let totalScore = 0;
  let passCount = 0;
  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };

  for (const c of cases) {
    const result = scoreCase(skillMd, c);
    totalScore += result.total;
    if (result.passed) passCount++;
    for (const k of Object.keys(dimTotals)) {
      dimTotals[k] += result.scores[k];
    }
    console.log(
      `${result.passed ? '✅' : '❌'} [${result.id}] ${result.case} | Total: ${result.total}`
    );
  }

  const avgTotal = totalScore / cases.length;
  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = dimTotals[k] / cases.length;
  }

  console.log(`\n--- Summary ---`);
  console.log(`Average Score: ${avgTotal.toFixed(2)} / 100`);
  console.log(`Pass Rate: ${passCount}/${cases.length}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(
      `  ${dim.id} ${dim.name}: ${dimAvgs[dim.id].toFixed(2)} / ${dim.max}`
    );
  }

  const globalPassed = avgTotal >= BENCHMARK.passLine;
  console.log(`\n${globalPassed ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  // Write iteration scores JSON
  const outPath = path.join(evolveDir, 'iteration_1_scores.json');
  const scoreDetails = cases.map((c, idx) => {
    const r = scoreCase(skillMd, { ...c, id: idx + 1 });
    return {
      id: idx + 1,
      instruction: c.input.slice(0, 80) + (c.input.length > 80 ? '...' : ''),
      ...r.scores,
      total: r.total,
    };
  });

  fs.writeFileSync(
    outPath,
    JSON.stringify({
      iteration: 1,
      skill_file: path.basename(skillMdPath),
      avg_score: Number(avgTotal.toFixed(2)),
      pass_rate: `${passCount}/${cases.length}`,
      global_passed: globalPassed,
      dim_avgs: dimAvgs,
      scores: scoreDetails,
    }, null, 2)
  );
  console.log(`\nScores written to ${outPath}`);

  return {
    avgTotal,
    passCount,
    dimAvgs,
    globalPassed,
    caseCount: cases.length,
  };
}

module.exports = { main, scoreCase, BENCHMARK };

if (require.main === module) {
  main();
}