宜博 / lemonboss

/**
 * Auto-Publisher Scorer v1
 * 基于 benchmark_v1.md 对 test.jsonl 进行自动化评分
 *
 * 用法: node scorer.js [skill_md_path]
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: '平台适配能力', max: 15 },
    { id: 'D2', name: '内容预处理', max: 15 },
    { id: 'D3', name: '发布执行', max: 15 },
    { id: 'D4', name: '状态追踪与重试', max: 15 },
    { id: 'D5', name: '合规与风控', max: 15 },
    { id: 'D6', name: '综合能力', max: 25 },
  ],
  passLine: 95,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .map((l, idx) => ({ ...JSON.parse(l), id: idx + 1 }));
}

function loadSkillMd(filePath) {
  if (!fs.existsSync(filePath)) return '';
  return fs.readFileSync(filePath, 'utf-8');
}

function scoreCase(skillMd, caseObj) {
  const md = skillMd.toLowerCase();
  const input = (caseObj.input || '').toLowerCase();
  const expected = caseObj.expected_output || {};

  // D1: 平台适配能力
  let d1 = 15;
  const has5Platforms =
    md.includes('weibo') && md.includes('wechat') && md.includes('douyin') &&
    md.includes('xiaohongshu') && md.includes('video_channel');
  const hasRpaFallback = md.includes('rpa') && md.includes('fallback');
  const hasAdapterPattern = md.includes('adapter') && md.includes('adaptcontent');
  if (!has5Platforms) d1 -= 3;
  if (!hasRpaFallback) d1 -= 2;
  if (!hasAdapterPattern) d1 -= 3;
  // 场景特化扣分
  if (input.includes('小红书') && !md.includes('≤20字')) d1 -= 1;

  // D2: 内容预处理
  let d2 = 15;
  const hasDedup = md.includes('dedup') || md.includes('去重');
  const hasAigc = md.includes('aigc') || md.includes('标识');
  const hasModeration = md.includes('moderation') || md.includes('敏感词');
  const hasMultimedia = md.includes('视频') || md.includes('转码');
  if (!hasDedup) d2 -= 3;
  if (!hasAigc) d2 -= 4;
  if (!hasModeration) d2 -= 3;
  if (!hasMultimedia) d2 -= 2;

  // D3: 发布执行
  let d3 = 15;
  const hasApiRpaMix = md.includes('api') && md.includes('rpa');
  const hasSchedule = md.includes('定时') || md.includes('delayed');
  const hasBatch = md.includes('批量') || md.includes('chunk');
  const hasErrorCodes = md.includes('429') && md.includes('401') && md.includes('500');
  const hasTokenRefresh = md.includes('access_token') && md.includes('刷新');
  if (!hasApiRpaMix) d3 -= 3;
  if (!hasSchedule) d3 -= 2;
  if (!hasBatch) d3 -= 2;
  if (!hasErrorCodes) d3 -= 3;
  if (!hasTokenRefresh) d3 -= 2;

  // D4: 状态追踪与重试
  let d4 = 15;
  const has7States =
    md.includes('pending') && md.includes('queued') && md.includes('publishing') &&
    md.includes('published') && md.includes('failed_retryable') &&
    md.includes('failed_permanent') && md.includes('cancelled');
  const hasExponentialBackoff = md.includes('指数退避') || md.includes('exponential');
  const hasJitter = md.includes('jitter');
  const hasDlq = md.includes('dlq') || md.includes('死信');
  const hasDataModel = md.includes('task_id') && md.includes('platform_post_id');
  if (!has7States) d4 -= 3;
  if (!hasExponentialBackoff) d4 -= 2;
  if (!hasJitter) d4 -= 1;
  if (!hasDlq) d4 -= 2;
  if (!hasDataModel) d4 -= 2;

  // D5: 合规与风控
  let d5 = 15;
  const hasPreCheck = md.includes('预审') || md.includes(' checklist');
  const hasBehaviorSim = md.includes('随机延迟') && md.includes('ip');
  const hasAigcCompliance = md.includes('强制显著标识') || md.includes('强制双标识');
  const hasAccountHealth = md.includes('账号') && md.includes('suspicious');
  if (!hasPreCheck) d5 -= 3;
  if (!hasBehaviorSim) d5 -= 3;
  if (!hasAigcCompliance) d5 -= 4;
  if (!hasAccountHealth) d5 -= 3;

  // D6: 综合能力
  let d6 = 25;
  const hasPipeline = md.includes('pipeline');
  const hasConfigAsCode = md.includes('configuration-as-code') || md.includes('配置化');
  const hasExtensibility = md.includes('扩展') || md.includes('新增平台');
  const hasDocs = md.includes('架构') && md.includes('数据模型');
  const hasCircuitBreaker = md.includes('circuit breaker') || md.includes('熔断');
  if (!hasPipeline) d6 -= 3;
  if (!hasConfigAsCode) d6 -= 4;
  if (!hasExtensibility) d6 -= 4;
  if (!hasDocs) d6 -= 3;
  if (!hasCircuitBreaker) d6 -= 3;

  // 按场景微调：如果某些维度在该场景中自然权重低，不额外扣分（已由上面处理）
  // 确保分数在合理范围内
  d1 = Math.max(0, Math.min(15, d1));
  d2 = Math.max(0, Math.min(15, d2));
  d3 = Math.max(0, Math.min(15, d3));
  d4 = Math.max(0, Math.min(15, d4));
  d5 = Math.max(0, Math.min(15, d5));
  d6 = Math.max(0, Math.min(25, d6));

  const total = d1 + d2 + d3 + d4 + d5 + d6;

  return {
    id: caseObj.id,
    case: (caseObj.input || '').slice(0, 60) + '...',
    scores: { D1: d1, D2: d2, D3: d3, D4: d4, D5: d5, D6: d6 },
    total,
    passed: total >= BENCHMARK.passLine,
  };
}

function main() {
  const evolveDir = __dirname;
  const testPath = path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const skillMdPath = process.argv[2] || path.join(evolveDir, 'SKILL_v2.md');

  if (!fs.existsSync(testPath)) {
    console.error('test.jsonl not found:', testPath);
    process.exit(1);
  }

  const cases = loadJsonl(testPath);
  const skillMd = loadSkillMd(skillMdPath);

  console.log(`\nEvaluating SKILL.md: ${skillMdPath}`);
  console.log(`Cases: ${cases.length}\n`);

  let totalScore = 0;
  let passCount = 0;
  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };

  for (const c of cases) {
    const result = scoreCase(skillMd, c);
    totalScore += result.total;
    if (result.passed) passCount++;
    for (const k of Object.keys(dimTotals)) {
      dimTotals[k] += result.scores[k];
    }
    console.log(
      `${result.passed ? '✅' : '❌'} [${result.id}] ${result.case} | Total: ${result.total}`
    );
  }

  const avgTotal = totalScore / cases.length;
  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = dimTotals[k] / cases.length;
  }

  console.log(`\n--- Summary ---`);
  console.log(`Average Score: ${avgTotal.toFixed(2)} / 100`);
  console.log(`Pass Rate: ${passCount}/${cases.length}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(
      `  ${dim.id} ${dim.name}: ${dimAvgs[dim.id].toFixed(2)} / ${dim.max}`
    );
  }

  const globalPassed = avgTotal >= BENCHMARK.passLine;
  console.log(`\n${globalPassed ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  return {
    avgTotal,
    passCount,
    dimAvgs,
    globalPassed,
    caseCount: cases.length,
  };
}

module.exports = { main, scoreCase, BENCHMARK };

if (require.main === module) {
  const result = main();
  const skillMd = loadSkillMd(process.argv[2] || path.join(__dirname, 'SKILL_v2.md'));
  const cases = loadJsonl(path.join(__dirname, '..', 'digest', 'test.jsonl')).map((c, idx) => {
    const r = scoreCase(skillMd, { ...c, id: idx + 1 });
    return {
      id: idx + 1,
      instruction: c.input.slice(0, 80) + (c.input.length > 80 ? '...' : ''),
      ...r.scores,
      total: r.total,
      eval_notes: r.passed ? 'SKILL.md covers all required dimensions for this scenario.' : 'Partial coverage; see weak dimensions.',
    };
  });

  // Write scores to JSON for evolve tracking
  const outPath = path.join(__dirname, 'iteration_scores_temp.json');
  fs.writeFileSync(
    outPath,
    JSON.stringify({
      iteration: 1,
      skill_file: process.argv[2] || 'SKILL_v2.md',
      modifications: [
        '基于 benchmark_v1.md 6维度全面强化 SKILL.md',
        '新增全维度覆盖原则：无论问题多聚焦，必须体现6维度 awareness',
        '引入 Circuit Breaker 熔断器层，完善状态追踪与错误处理',
        '细化 2025 AIGC 新规、敏感词预审、行为节奏模拟、账号健康监控',
        'PlatformConfig Schema 升级为 Configuration-as-Code + 热更新 + 版本控制',
        '新增扩展新平台标准流程（含零改动承诺）',
      ],
      scores: cases,
      summary: {
        avg_score: Number(result.avgTotal.toFixed(2)),
        min_score: Math.min(...cases.map(c => c.total)),
        max_score: Math.max(...cases.map(c => c.total)),
        case_count: result.caseCount,
        pass_rate: `${result.passCount}/${result.caseCount}`,
        global_passed: result.globalPassed,
        weak_dimensions: Object.entries(result.dimAvgs)
          .filter(([k, v]) => v < (k === 'D6' ? 22 : 13))
          .map(([k]) => k),
      },
      dim_avgs: result.dimAvgs,
    }, null, 2)
  );
  console.log(`\nScores written to ${outPath}`);
}