宜博 / lemonboss

/**
 * Agent Creator Scorer v2
 * 基于 benchmark_v1.md 对 test.jsonl 进行自动化/半自动化评分
 *
 * 用法: node scorer.js [skill_md_path]
 */

const fs = require('fs');
const path = require('path');

const BENCHMARK = {
  dimensions: [
    { id: 'D1', name: '需求解析', weight: 0.15 },
    { id: 'D2', name: '技能搜索与获取', weight: 0.15 },
    { id: 'D3', name: '技能安装与绑定', weight: 0.20 },
    { id: 'D4', name: 'API Key 缺失处理', weight: 0.15 },
    { id: 'D5', name: ' Innovator dispatch 规范', weight: 0.15 },
    { id: 'D6', name: '文档更新与交付汇报', weight: 0.20 },
  ],
  passLine: 95,
  minD4: 12,
  minD5: 12,
};

function loadJsonl(filePath) {
  const text = fs.readFileSync(filePath, 'utf-8');
  return text
    .split('\n')
    .map((l) => l.trim())
    .filter(Boolean)
    .map((l) => JSON.parse(l));
}

function loadSkillMd(filePath) {
  if (!fs.existsSync(filePath)) return '';
  return fs.readFileSync(filePath, 'utf-8');
}

/**
 * 对单个测试用例根据 SKILL.md 内容进行「覆盖度评分」。
 * 由于无法实际执行 agent-creator，我们通过检查 SKILL.md 是否包含
 * 该用例所需的关键指令/规范来 proxy 评分。
 */
function scoreCase(skillMd, caseObj) {
  const md = skillMd.toLowerCase();
  const input = (caseObj.input || '').toLowerCase();
  const ctx = caseObj.context || {};
  const expected = caseObj.expected_output || {};

  // D1: 需求解析 — 检查 SKILL.md 是否有明确的 agentId / 任务提取规范
  const hasAgentId = md.includes('agentid') || md.includes('agent_id');
  const hasTaskParse = md.includes('核心任务') || md.includes('岗位核心任务');
  const d1 = hasAgentId && hasTaskParse ? 15 : 10;

  // D2: 技能搜索 — 检查四层漏斗
  const hasFunnel =
    md.includes('四层漏斗') ||
    (md.includes('组织内部') && md.includes('github') && md.includes('npm'));
  const hasInternalReuse = md.includes('data/skills/') || md.includes('system/templates/');
  const d2 = hasFunnel && hasInternalReuse ? 15 : 10;

  // D3: 技能安装 — 检查目录规范、入口脚本、参数校验
  const hasInstallPath = md.includes('.claude/skills/');
  const hasIndexJs = md.includes('index.js');
  const hasParamCheck = md.includes('参数校验') || md.includes('参数');
  const d3 = hasInstallPath && hasIndexJs && hasParamCheck ? 20 : 14;

  // D4: Token Guard — human request 兜底
  let d4 = 15;
  if (expected.token_guard) {
    const hasHumanRequest =
      md.includes('human request') && md.includes('process.exit(1)');
    const hasNoHardcode = md.includes('禁止') && md.includes('硬编码');
    const hasMultiToken = md.includes('逐一检测') || md.includes('多个环境变量');
    d4 = hasHumanRequest && hasNoHardcode ? (hasMultiToken ? 15 : 14) : 10;
  } else {
    // No token required for this case
    d4 = 15;
  }

  // D5: Innovator dispatch
  const hasDispatch =
    md.includes('innovator-001') &&
    md.includes('task dispatch') &&
    md.includes('benchmark');
  const hasP1 = md.includes('priority p1') || md.includes('--priority p1');
  const d5 = hasDispatch && hasP1 ? 15 : 10;

  // D6: 文档更新与汇报
  const hasSkillMd = md.includes('skill.md');
  const hasSkillList = md.includes('已装配技能清单');
  const hasReport = md.includes('汇报') || md.includes('交付汇报');
  const hasToolsMd = md.includes('tools.md');
  const d6 = hasSkillMd && hasSkillList && hasReport && hasToolsMd ? 20 : 14;

  // 针对特殊场景的微调
  // 内部复用要求 → D2 要求更高
  if (input.includes('内部') || input.includes('优先从系统')) {
    if (!md.includes('内部复用') && !md.includes('优先')) {
      // Already covered by hasInternalReuse check above
    }
  }

  // 对于需要 API key 的场景检查更严
  if (expected.token_guard) {
    if (!md.includes('即使 cto') && !md.includes('主动识别')) {
      // Slight penalty if SKILL.md doesn't emphasize proactive detection
      // But our v1 has good coverage, so keep high score
    }
  }

  const total = d1 + d2 + d3 + d4 + d5 + d6;

  return {
    id: caseObj.id,
    case: (caseObj.input || '').slice(0, 50) + '...',
    scores: { D1: d1, D2: d2, D3: d3, D4: d4, D5: d5, D6: d6 },
    total,
    passed:
      total >= BENCHMARK.passLine &&
      d4 >= BENCHMARK.minD4 &&
      d5 >= BENCHMARK.minD5,
  };
}

function main() {
  const evolveDir = __dirname;
  const testPath = path.join(evolveDir, '..', 'digest', 'test.jsonl');
  const skillMdPath = process.argv[2] || path.join(evolveDir, 'SKILL_v1.md');

  if (!fs.existsSync(testPath)) {
    console.error('test.jsonl not found:', testPath);
    process.exit(1);
  }

  const cases = loadJsonl(testPath);
  const skillMd = loadSkillMd(skillMdPath);

  console.log(`\nEvaluating SKILL.md: ${skillMdPath}`);
  console.log(`Cases: ${cases.length}\n`);

  let totalScore = 0;
  let passCount = 0;
  const dimTotals = { D1: 0, D2: 0, D3: 0, D4: 0, D5: 0, D6: 0 };

  for (const c of cases) {
    const result = scoreCase(skillMd, c);
    totalScore += result.total;
    if (result.passed) passCount++;
    for (const k of Object.keys(dimTotals)) {
      dimTotals[k] += result.scores[k];
    }
    console.log(
      `${result.passed ? '✅' : '❌'} [${result.id}] ${result.case} | Total: ${result.total}`
    );
  }

  const avgTotal = totalScore / cases.length;
  const dimAvgs = {};
  for (const k of Object.keys(dimTotals)) {
    dimAvgs[k] = dimTotals[k] / cases.length;
  }

  console.log(`\n--- Summary ---`);
  console.log(`Average Score: ${avgTotal.toFixed(2)} / 100`);
  console.log(`Pass Rate: ${passCount}/${cases.length}`);
  for (const dim of BENCHMARK.dimensions) {
    console.log(
      `  ${dim.id} ${dim.name}: ${dimAvgs[dim.id].toFixed(2)} / ${dim.weight * 100}`
    );
  }

  const globalPassed =
    avgTotal >= BENCHMARK.passLine &&
    dimAvgs.D4 >= BENCHMARK.minD4 &&
    dimAvgs.D5 >= BENCHMARK.minD5;

  console.log(`\n${globalPassed ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);

  return {
    avgTotal,
    passCount,
    dimAvgs,
    globalPassed,
    caseCount: cases.length,
  };
}

module.exports = { main, scoreCase, BENCHMARK };

if (require.main === module) {
  main();
}