宜博 / lemonboss

/**
 * Benchmark Runner — 6 维度评分 + ELO 排名引擎
 *
 * MVP 版本: 使用 heuristic-based 评分（基于 Skill 元数据质量）
 * 后续 Phase 4: 接入 Innovator 做真实沙盒执行评测
 *
 * 评分维度:
 *   successRate  — 成功率 (元数据完整性 + 状态)
 *   accuracy     — 准确性 (描述质量 + 依赖匹配)
 *   speed        — 速度 (代码量 → 推断执行耗时)
 *   cost         — 成本 (层级 + 依赖数 → 推断资源消耗)
 *   robustness   — 鲁棒性 (错误处理关键词 + 依赖丰富度)
 *   completeness — 完成度 (L3 独有, 端到端覆盖度)
 */

const path = require('path');
const fs = require('fs');
const benchmarkDB = require('./benchmark-db');

const config = require('../config');
const skillsDir = config.skillsCodeDir;
const skillDb = require('./skill-db');

// ═══════════════════════════════════════════
//  配置
// ═══════════════════════════════════════════

// 权重矩阵 [L1, L2, L3]
const WEIGHTS = {
  successRate:  [0.40, 0.30, 0.25],
  accuracy:     [0.30, 0.25, 0.20],
  speed:        [0.20, 0.15, 0.10],
  cost:         [0.05, 0.15, 0.15],
  robustness:   [0.05, 0.10, 0.10],
  completeness: [0.00, 0.05, 0.20],
};

// K-factor 衰减
function getKFactor(benchmarkRuns) {
  if (benchmarkRuns < 10) return 40;
  if (benchmarkRuns < 50) return 20;
  return 10;
}

// ═══════════════════════════════════════════
//  Skill 列表读写
// ═══════════════════════════════════════════

function loadSkills() {
  return skillDb.listSkills();
}

function saveSkills(skills) {
  for (const skill of skills) {
    const updates = { ...skill };
    delete updates.skillId;
    delete updates.skillName;
    skillDb.updateSkill(skill.skillName, updates);
  }
}

// ═══════════════════════════════════════════
//  Heuristic-Based 评分引擎
// ═══════════════════════════════════════════

/**
 * 为单个 Skill 生成 6 维度分数（基于元数据质量的 heuristic）
 */
function scoreSkill(skill) {
  const name = skill.skillName || '';
  const desc = skill.description || '';
  const deps = skill.dependencies || [];
  const tags = skill.tags || skill.keywords || [];
  const layer = skill.layer || 1;
  const status = skill.status || 'enable';

  // 伪随机种子 (基于技能名，确保一致性)
  let seed = 0;
  for (let i = 0; i < name.length; i++) {
    seed = ((seed << 5) - seed + name.charCodeAt(i)) | 0;
  }
  const seededRandom = (min, max) => {
    seed = (seed * 16807 + 12345) & 0x7fffffff;
    return min + (seed / 0x7fffffff) * (max - min);
  };

  // === successRate (0-100) ===
  let successRate = 70; // 基础分
  if (status === 'enable') successRate += 10;
  if (desc.length > 20) successRate += 5;
  if (desc.length > 80) successRate += 5;
  if (tags.length > 0) successRate += 3;
  if (skill.version) successRate += 2;
  successRate += seededRandom(-5, 5);
  successRate = Math.max(50, Math.min(100, successRate));

  // === accuracy (0-100) ===
  let accuracy = 65;
  if (desc.length > 30) accuracy += 8;
  if (desc.length > 100) accuracy += 7;
  if (tags.length >= 3) accuracy += 5;
  if (deps.length > 0 && layer >= 2) accuracy += 5;
  if (skill.domain) accuracy += 3;
  accuracy += seededRandom(-8, 8);
  accuracy = Math.max(40, Math.min(100, accuracy));

  // === speed (seconds, lower is better) ===
  let speed = 1.0; // base seconds
  if (layer === 2) speed = 2.0;
  if (layer === 3) speed = 4.0;
  speed += deps.length * 0.3;
  speed += seededRandom(-0.3, 0.8);
  speed = Math.max(0.1, speed);

  // === cost (dollars per run, lower is better) ===
  let cost = 0.001;
  if (layer === 2) cost = 0.005;
  if (layer === 3) cost = 0.02;
  cost += deps.length * 0.002;
  cost += seededRandom(-0.001, 0.005);
  cost = Math.max(0.0001, cost);

  // === robustness (0-100) ===
  let robustness = 60;
  if (deps.length > 0) robustness += 5;
  if (tags.length >= 2) robustness += 5;
  if (desc.includes('error') || desc.includes('retry') || desc.includes('timeout')) robustness += 10;
  if (desc.includes('validate') || desc.includes('check')) robustness += 5;
  if (skill.version && skill.version !== '1.0.0') robustness += 5; // iterated
  robustness += seededRandom(-8, 8);
  robustness = Math.max(30, Math.min(100, robustness));

  // === completeness (0-100, mainly for L3) ===
  let completeness = 0;
  if (layer >= 2) {
    completeness = 50;
    completeness += deps.length * 8;
    completeness += (desc.length > 100 ? 10 : 0);
    completeness += seededRandom(-10, 10);
    completeness = Math.max(20, Math.min(100, completeness));
  }

  // 模拟测试用例
  const testCasesTotal = 5 + Math.floor(seededRandom(0, 10));
  const passRate = successRate / 100;
  const testCasesPassed = Math.floor(testCasesTotal * passRate);

  return {
    successRate: parseFloat(successRate.toFixed(1)),
    accuracy: parseFloat(accuracy.toFixed(1)),
    speed: parseFloat(speed.toFixed(2)),
    cost: parseFloat(cost.toFixed(4)),
    robustness: parseFloat(robustness.toFixed(1)),
    completeness: layer >= 2 ? parseFloat(completeness.toFixed(1)) : null,
    testCasesTotal,
    testCasesPassed,
  };
}

/**
 * 计算加权综合分 (0-100 scale)
 */
function computeWeightedScore(scores, layer) {
  const layerIdx = Math.max(0, Math.min(2, (layer || 1) - 1));
  let total = 0;

  // 正向维度 (越高越好)
  total += (scores.successRate || 0) * WEIGHTS.successRate[layerIdx];
  total += (scores.accuracy || 0) * WEIGHTS.accuracy[layerIdx];
  total += (scores.robustness || 0) * WEIGHTS.robustness[layerIdx];
  total += (scores.completeness || 0) * WEIGHTS.completeness[layerIdx];

  // 反向维度 (越低越好) → 转换为 0-100 分
  const speedScore = Math.max(0, 100 - scores.speed * 10); // 10s → 0分, 0s → 100分
  total += speedScore * WEIGHTS.speed[layerIdx];

  const costScore = Math.max(0, 100 - scores.cost * 1000); // $0.1 → 0分, $0 → 100分
  total += costScore * WEIGHTS.cost[layerIdx];

  return parseFloat(total.toFixed(2));
}

// ═══════════════════════════════════════════
//  ELO 计算
// ═══════════════════════════════════════════

/**
 * 计算 ELO 预期得分
 */
function expectedScore(eloA, eloB) {
  return 1 / (1 + Math.pow(10, (eloB - eloA) / 400));
}

/**
 * 更新 ELO (返回 [newEloA, newEloB])
 */
function updateElo(eloA, eloB, scoreA, scoreB, kA, kB) {
  const expA = expectedScore(eloA, eloB);
  const expB = expectedScore(eloB, eloA);

  // 实际结果: 1=赢, 0=输, 0.5=平
  const actualA = scoreA > scoreB ? 1 : scoreA === scoreB ? 0.5 : 0;
  const actualB = 1 - actualA;

  const newA = Math.round(eloA + kA * (actualA - expA));
  const newB = Math.round(eloB + kB * (actualB - expB));

  return [newA, newB];
}

/**
 * 在同场景 + 同 Layer 内做全量 ELO 更新
 */
function runEloTournament(skills) {
  // 按 scene + layer 分组
  const groups = {};
  for (const s of skills) {
    const key = `${s.scene || 'unknown'}_L${s.layer || 0}`;
    if (!groups[key]) groups[key] = [];
    groups[key].push(s);
  }

  let totalComparisons = 0;

  for (const [groupKey, group] of Object.entries(groups)) {
    if (group.length < 2) continue;

    // 两两比较
    for (let i = 0; i < group.length; i++) {
      for (let j = i + 1; j < group.length; j++) {
        const a = group[i];
        const b = group[j];

        const scoreA = a.arena?.scores ? computeWeightedScore(a.arena.scores, a.layer) : 50;
        const scoreB = b.arena?.scores ? computeWeightedScore(b.arena.scores, b.layer) : 50;

        const eloA = a.arena?.elo || 1200;
        const eloB = b.arena?.elo || 1200;

        const kA = getKFactor(a.arena?.benchmarkRuns || 0);
        const kB = getKFactor(b.arena?.benchmarkRuns || 0);

        const [newA, newB] = updateElo(eloA, eloB, scoreA, scoreB, kA, kB);

        // 更新 ELO
        if (a.arena) a.arena.elo = newA;
        if (b.arena) b.arena.elo = newB;

        // 记录到 SQLite
        const winner = scoreA > scoreB ? a.skillName
                     : scoreB > scoreA ? b.skillName
                     : 'tie';

        benchmarkDB.insertComparison({
          skillA: a.skillName,
          skillB: b.skillName,
          scene: a.scene,
          layer: a.layer,
          winner,
          scoreA,
          scoreB,
        });

        totalComparisons++;
      }
    }
  }

  return totalComparisons;
}

// ═══════════════════════════════════════════
//  主流程
// ═══════════════════════════════════════════

/**
 * 对单个 Skill 执行 benchmark
 */
function benchmarkSkill(skill) {
  // 确保有 arena 字段
  if (!skill.arena) {
    skill.arena = {
      elo: 1200,
      eloMargin: 50,
      rank: null,
      benchmarkRuns: 0,
      lastBenchmark: null,
      scores: {},
      securityScan: { status: 'pending' },
      highlighted: false,
      trending: false,
    };
  }

  const eloBefore = skill.arena.elo || 1200;

  // Step 1: 评分
  const scores = scoreSkill(skill);
  const weightedScore = computeWeightedScore(scores, skill.layer);

  // Step 2: 更新 Skill 元数据
  skill.arena.scores = {
    successRate: scores.successRate,
    accuracy: scores.accuracy,
    speed: scores.speed,
    cost: scores.cost,
    robustness: scores.robustness,
    completeness: scores.completeness,
  };
  skill.arena.benchmarkRuns = (skill.arena.benchmarkRuns || 0) + 1;
  skill.arena.lastBenchmark = new Date().toISOString();

  // Step 3: 记录到 SQLite
  benchmarkDB.insertBenchmarkRun({
    skillName: skill.skillName,
    version: skill.version || null,
    successRate: scores.successRate,
    accuracy: scores.accuracy,
    speedP50: scores.speed,
    speedP95: scores.speed * 1.5,
    costPerRun: scores.cost,
    robustness: scores.robustness,
    completeness: scores.completeness,
    weightedScore,
    eloBefore,
    eloAfter: eloBefore, // ELO 在 tournament 阶段更新
    testCasesTotal: scores.testCasesTotal,
    testCasesPassed: scores.testCasesPassed,
    errorLog: null,
  });

  return { scores, weightedScore };
}

/**
 * 对所有 Skill 执行 benchmark + ELO tournament
 */
function benchmarkAll() {
  // 初始化数据库
  benchmarkDB.initDB();

  const skills = loadSkills();
  const results = [];

  console.log(`[BenchmarkRunner] Starting benchmark for ${skills.length} skills...`);

  // Step 1: 评分所有 Skill
  for (const skill of skills) {
    const { scores, weightedScore } = benchmarkSkill(skill);
    results.push({
      skillName: skill.skillName,
      layer: skill.layer,
      scene: skill.scene,
      weightedScore,
      scores,
    });
  }

  // Step 2: ELO Tournament
  const totalComparisons = runEloTournament(skills);

  // Step 3: 更新 benchmark_runs 中的 elo_after
  for (const skill of skills) {
    // 更新最后一条记录的 elo_after
    const db = benchmarkDB.initDB();
    db.prepare(`
      UPDATE benchmark_runs
      SET elo_after = ?
      WHERE skill_name = ?
      ORDER BY id DESC
      LIMIT 1
    `).run(skill.arena?.elo || 1200, skill.skillName);
  }

  // Step 4: 计算排名
  computeRanks(skills);

  // Step 5: 保存
  saveSkills(skills);

  console.log(`[BenchmarkRunner] Completed: ${skills.length} skills scored, ${totalComparisons} ELO comparisons.`);

  return {
    totalSkills: skills.length,
    totalComparisons,
    results: results.sort((a, b) => b.weightedScore - a.weightedScore),
  };
}

/**
 * 计算排名
 */
function computeRanks(skills) {
  // 按场景 + Layer 分组排名
  const groups = {};
  for (const s of skills) {
    const key = `${s.scene || 'unknown'}_L${s.layer || 0}`;
    if (!groups[key]) groups[key] = [];
    groups[key].push(s);
  }

  for (const [, group] of Object.entries(groups)) {
    group.sort((a, b) => (b.arena?.elo || 1200) - (a.arena?.elo || 1200));
    group.forEach((s, i) => {
      if (s.arena) s.arena.rank = i + 1;
    });
  }

  // 全局排名
  const allSorted = [...skills].sort((a, b) => (b.arena?.elo || 1200) - (a.arena?.elo || 1200));
  allSorted.forEach((s, i) => {
    if (s.arena) s.arena.globalRank = i + 1;
  });

  // 标记 highlighted (每个场景 top 3)
  for (const [, group] of Object.entries(groups)) {
    group.forEach((s, i) => {
      if (s.arena) s.arena.highlighted = (i < 3);
    });
  }
}

module.exports = {
  scoreSkill,
  computeWeightedScore,
  benchmarkSkill,
  benchmarkAll,
  runEloTournament,
  getKFactor,
};