宜博 / lemonboss

#!/usr/bin/env node
/**
 * fix-missing-skills.js — 补导 data/skills/ 中存在但 SQLite 中缺失的 skill
 *
 * 逻辑：
 *   1. 从 SQLite 获取所有已有 skill_name 集合
 *   2. 扫描 data/skills/ 目录
 *   3. 找出差集（目录中有，但 SQLite 中没有的）
 *   4. 只导入这些缺失的 skill（从 SKILL.md/details.json 提取信息）
 *
 * 用法:
 *   node scripts/fix-missing-skills.js              # 补导缺失
 *   node scripts/fix-missing-skills.js --dry-run    # 预览，不写入
 */

const fs = require('fs');
const path = require('path');

// 复用 skill-db 的 schema 和数据库连接
const skillDb = require('../server/services/skill-db');

const args = process.argv.slice(2);
const DRY_RUN = args.includes('--dry-run');

const SKILLS_DIR = path.resolve(__dirname, '../../data/skills');

function extractDescription(dirPath) {
  // 优先从 SKILL.md 提取
  const skillMd = path.join(dirPath, 'SKILL.md');
  if (fs.existsSync(skillMd)) {
    const content = fs.readFileSync(skillMd, 'utf8');
    // 提取 YAML frontmatter 中的 description
    const fmMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
    if (fmMatch) {
      const descMatch = fmMatch[1].match(/description:\s*(.+)/);
      if (descMatch) return descMatch[1].trim().slice(0, 500);
    }
    // fallback: 取第一行非标题非空文本
    const lines = content.split('\n').filter(l => l.trim() && !l.startsWith('#') && !l.startsWith('---'));
    if (lines.length > 0) return lines[0].trim().slice(0, 500);
  }

  // 从 details.json 提取
  const detailsJson = path.join(dirPath, 'details.json');
  if (fs.existsSync(detailsJson)) {
    try {
      const details = JSON.parse(fs.readFileSync(detailsJson, 'utf8'));
      return (details.skill?.summary || '').slice(0, 500);
    } catch {}
  }

  return '';
}

function main() {
  console.log('╔═══════════════════════════════════════════════╗');
  console.log('║  补导缺失 Skills → SQLite                     ║');
  console.log('╚═══════════════════════════════════════════════╝');
  console.log();

  if (DRY_RUN) console.log('⚠️  DRY RUN 模式 — 不会实际写入数据库\n');

  // 1. 检查目录
  if (!fs.existsSync(SKILLS_DIR)) {
    console.error(`❌ Skills 目录不存在: ${SKILLS_DIR}`);
    process.exit(1);
  }

  // 2. 获取 DB 中已有的 skill_name 集合
  const db = skillDb.getDb();
  const existingNames = new Set(
    db.prepare('SELECT skill_name FROM skills').all().map(r => r.skill_name)
  );
  console.log(`📊 数据库中已有: ${existingNames.size} 个 skill`);

  // 3. 扫描目录
  const allDirs = fs.readdirSync(SKILLS_DIR, { withFileTypes: true })
    .filter(d => d.isDirectory() && !d.name.startsWith('.') && d.name !== '_meta')
    .map(d => d.name);
  console.log(`📁 目录中共有: ${allDirs.length} 个子目录`);

  // 4. 找出缺失的
  const missing = allDirs.filter(name => !existingNames.has(name));
  console.log(`🔍 缺失数量: ${missing.length}`);

  if (missing.length === 0) {
    console.log('\n✅ 没有缺失的 skill，无需补导');
    return;
  }

  // 5. 准备 SQL
  const skillCols = [
    'skill_id', 'skill_name', 'directory', 'description', 'domain', 'scope',
    'status', 'version', 'benchmark_version', 'industry_avg_score', 'elite_avg_score',
    'current_score', 'use_count', 'download_count', 'update_count', 'last_used',
    'last_updated', 'created_at', 'keywords', 'type', 'agent_name', 'layer',
    'dependencies', 'scene', 'author', 'license', 'source', 'tags',
    'roles', 'scenes', 'platforms',
  ];
  const placeholders = skillCols.map(() => '?').join(', ');
  const insertSql = `INSERT OR IGNORE INTO skills (${skillCols.join(', ')}) VALUES (${placeholders})`;
  const stmtInsert = db.prepare(insertSql);

  const stmtArena = db.prepare(
    `INSERT OR IGNORE INTO skill_arena (skill_id, elo, elo_margin, security_status) VALUES (?, 1200, 50, 'pending')`
  );

  // 6. 批量导入（事务）
  let added = 0;
  let errors = 0;
  const startTime = Date.now();

  console.log('\n--- 开始逐个导入 ---');

  const batchSize = 500;
  for (let i = 0; i < missing.length; i += batchSize) {
    const batch = missing.slice(i, i + batchSize);

    if (!DRY_RUN) {
      const tx = db.transaction(() => {
        for (const name of batch) {
          try {
            const dir = path.join(SKILLS_DIR, name);
            const desc = extractDescription(dir);

            // 检查是否有 details.json 来判断 source
            const hasDetails = fs.existsSync(path.join(dir, 'details.json'));
            const source = hasDetails ? 'clawhub' : 'local';
            const skillId = hasDetails ? `crawled-${name}` : `skill-local-${name}`;
            const now = new Date().toISOString();

            const row = {
              skill_id: skillId,
              skill_name: name,
              directory: dir,
              description: desc,
              domain: '',
              scope: '',
              status: 'enable',
              version: '1.0.0',
              benchmark_version: '1.0.0',
              industry_avg_score: null,
              elite_avg_score: null,
              current_score: null,
              use_count: 0,
              download_count: 0,
              update_count: 0,
              last_used: null,
              last_updated: now,
              created_at: now,
              keywords: '[]',
              type: 'executor',
              agent_name: null,
              layer: null,
              dependencies: '[]',
              scene: '',
              author: source === 'clawhub' ? 'clawhub' : 'local',
              license: 'MIT',
              source: source,
              tags: '[]',
              roles: '[]',
              scenes: '[]',
              platforms: '[]',
            };

            stmtInsert.run(...skillCols.map(c => row[c]));
            stmtArena.run(skillId);
            added++;
          } catch (e) {
            errors++;
            if (errors <= 10) {
              console.error(`  ❌ ${name}: ${e.message}`);
            }
          }
        }
      });
      tx();
    } else {
      added += batch.length;
    }

    const progress = Math.min(i + batchSize, missing.length);
    const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
    process.stdout.write(`\r  📦 已处理 ${progress}/${missing.length} | 用时 ${elapsed}s`);
  }

  console.log('\n');

  // 7. 结果汇总
  const totalTime = ((Date.now() - startTime) / 1000).toFixed(2);
  const totalInDb = DRY_RUN ? '(dry-run)' : db.prepare('SELECT COUNT(*) as c FROM skills').get().c;

  console.log('┌──────────────────────────────────────┐');
  console.log('│         补导结果                      │');
  console.log('├──────────────────────────────────────┤');
  console.log(`│  ✅ 新增导入: ${String(added).padStart(8)}               │`);
  console.log(`│  ❌ 错误:     ${String(errors).padStart(8)}               │`);
  console.log(`│  ⏱️  总耗时:   ${String(totalTime + 's').padStart(8)}               │`);
  console.log(`│  📊 数据库总数: ${String(totalInDb).padStart(7)}              │`);
  console.log('└──────────────────────────────────────┘');

  // 8. 验证 tomato
  if (!DRY_RUN) {
    const t = db.prepare("SELECT skill_name, status, source FROM skills WHERE skill_name = 'tomato_short_story'").get();
    console.log(`\n  tomato_short_story: ${t ? `✅ 已导入 (source=${t.source}, status=${t.status})` : '❌ 未找到'}`);

    // 重建 FTS5 索引
    console.log('\n🔍 重建 FTS5 全文搜索索引...');
    try {
      db.exec("INSERT INTO skills_fts(skills_fts) VALUES('rebuild');");
      const ftsCount = db.prepare('SELECT COUNT(*) as c FROM skills_fts').get().c;
      console.log(`   FTS5 索引: ${ftsCount} 条`);
    } catch (e) {
      console.warn(`   FTS5 重建失败: ${e.message}`);
    }
  }

  console.log('\n✨ 完成!');
}

main();