宜博 / lemonboss

/**
 * Innovator API Routes — Skill 管理 + Conversations + API Logs
 */
const express = require('express');
const path = require('path');
const fs = require('fs');
const router = express.Router();
const { InnovatorService } = require('../services/innovator-service');
const config = require('../config');
const { requireAuth } = require('../middleware/jwt-auth');
const { JobManager } = require('../services/job-manager');
const { RetrainPipeline } = require('../services/retrain-pipeline');
const { watchdog: retrainWatchdog } = require('../services/retrain-watchdog');
const { MasterSession } = require('../src/core/master-session');

const h = (fn) => (req, res) => {
  try { fn(req, res); } catch (e) { console.error(`[Innovator] ${req.method} ${req.path}:`, e.message); res.status(500).json({ error: e.message }); }
};

// ── 单例 ──
let _innovator = null;
function getInnovator() {
  if (!_innovator) _innovator = new InnovatorService({ skillsDir: config.skillsCodeDir, skillsDataDir: config.skillsDataDir, trainedSkillsDir: config.trainedSkillsDir, conversationsDir: config.conversationsDir });
  return _innovator;
}

const retrainJobManager = new JobManager({ storePath: path.resolve(__dirname, '../../.data/jobs-retrain.json') });
let _retrainMasterSession = null;
function getRetrainMasterSession() {
  if (_retrainMasterSession) return _retrainMasterSession;
  try {
    const backends = JSON.parse(fs.readFileSync(path.join(__dirname, '../config/backends.json'), 'utf-8'));
    const { loadConfigFromSkills } = require('../src/core/skill-config-loader');
    _retrainMasterSession = new MasterSession(loadConfigFromSkills(config.skillsCodeDir, backends), config.skillsCodeDir);
    return _retrainMasterSession;
  } catch (err) { console.error('[Retrain] ⚠️ 无法创建 Master session:', err.message); return null; }
}

setTimeout(() => retrainWatchdog.start(retrainJobManager, getInnovator, getRetrainMasterSession), 5000);

// ══════════════════════════════════════
// Skills CRUD
// ══════════════════════════════════════

router.get('/skills', h((req, res) => {
  const skillDb = require('../services/skill-db');
  const page = parseInt(req.query.page) || 1;
  const pageSize = Math.min(parseInt(req.query.pageSize) || 50, 200);
  const { search, status, storageType, orderBy = 'created_at', order = 'DESC' } = req.query;
  const offset = (page - 1) * pageSize;

  if (search?.trim()) {
    const results = skillDb.searchSkills(search, { status: status || undefined, limit: pageSize, offset });
    const total = results.length < pageSize ? offset + results.length : offset + pageSize + 1;
    return res.json({ skills: results, total, page, pageSize, totalPages: Math.ceil(total / pageSize) });
  }
  const total = skillDb.countSkills({ status: status || undefined, storageType: storageType || undefined });
  const skills = skillDb.listSkills({ limit: pageSize, offset, status: status || undefined, storageType: storageType || undefined, orderBy, order: order.toUpperCase() === 'ASC' ? 'ASC' : 'DESC' });
  res.json({ skills, total, page, pageSize, totalPages: Math.ceil(total / pageSize) });
}));

router.get('/skills/:name', h((req, res) => {
  const skill = getInnovator().registry.getSkill(req.params.name);
  skill ? res.json(skill) : res.status(404).json({ error: 'Skill not found' });
}));

router.delete('/skills/:name', h((req, res) => {
  const removed = getInnovator().registry.removeSkill(req.params.name);
  removed ? res.json({ success: true }) : res.status(404).json({ error: 'Skill not found' });
}));

router.patch('/skills/:name', h((req, res) => res.json(getInnovator().registry.updateSkill(req.params.name, req.body))));

router.post('/skills/:name/rename', h((req, res) => {
  if (!req.body.newName) return res.status(400).json({ error: '缺少 newName' });
  res.json(getInnovator().registry.renameSkill(req.params.name, req.body.newName));
}));

router.get('/skills/:name/evolve-log', h((req, res) => {
  const skill = getInnovator().registry.getSkill(req.params.name);
  if (!skill) return res.status(404).json({ error: 'Skill not found' });
  try { res.json({ content: fs.readFileSync(path.join(getInnovator().skillsDir, req.params.name, 'evolve', 'evolve_log.md'), 'utf-8') }); }
  catch { res.json({ content: '' }); }
}));

router.post('/skills/scan', h((req, res) => {
  const registered = getInnovator().autoRegisterNewSkills();
  res.json({ registered, count: registered.length });
}));

// ── Retrain ──
router.post('/skills/:name/retrain', requireAuth, h((req, res) => {
  const innovator = getInnovator();
  const skillName = req.params.name;
  const skill = innovator.registry.getSkill(skillName);
  if (!skill) return res.status(404).json({ error: 'Skill not found' });

  const job = retrainJobManager.createJob({ type: 'retrain', skillName, status: 'queued', progress: '排队中...', instructions: req.body?.instructions || '' });
  innovator.registry.updateSkill(skillName, { status: 'training' });

  const pipeline = new RetrainPipeline({ job, skill, innovator, jobManager: retrainJobManager, instructions: req.body?.instructions || '', userId: req.user.id });
  const masterSession = getRetrainMasterSession();
  (masterSession ? pipeline.runWithMaster(masterSession) : pipeline.run()).catch(err => {
    retrainJobManager.updateJob(job.jobId, { status: 'failed', progress: `失败: ${err.message}` });
    try { innovator.registry.updateSkill(skillName, { status: 'enable' }); } catch {}
  });

  res.json({ jobId: job.jobId, skillName, status: 'queued', mode: masterSession ? 'master-orchestrated' : 'node-pipeline' });
}));

router.get('/jobs/:jobId', (req, res) => {
  const job = retrainJobManager.getJob(req.params.jobId);
  job ? res.json(job) : res.status(404).json({ error: 'Job not found' });
});

// ══════════════════════════════════════
// Conversations
// ══════════════════════════════════════

function safeReadDir(dir) { try { return fs.readdirSync(dir); } catch { return []; } }
function safeFilesInDir(dir) { return safeReadDir(dir).filter(f => { try { return !fs.statSync(path.join(dir, f)).isDirectory(); } catch { return false; } }); }

router.get('/conversations', requireAuth, h((req, res) => {
  const baseConversationsDir = getInnovator().conversationsDir;

  // Admin can see all users' conversations with ?all=true
  const isAdmin = req.user.role === 'admin';
  const showAll = isAdmin && req.query.all === 'true';

  // Collect user directories to scan
  const userDirs = [];
  if (showAll) {
    // Scan all user-* directories
    try {
      const entries = fs.readdirSync(baseConversationsDir);
      for (const entry of entries) {
        if (entry.startsWith('user-') && fs.statSync(path.join(baseConversationsDir, entry)).isDirectory()) {
          userDirs.push({ userId: entry.replace('user-', ''), dirPath: path.join(baseConversationsDir, entry) });
        }
      }
    } catch {}
  } else {
    const convDir = path.join(baseConversationsDir, `user-${req.user.id}`);
    if (fs.existsSync(convDir)) {
      userDirs.push({ userId: String(req.user.id), dirPath: convDir });
    }
  }

  if (userDirs.length === 0) return res.json([]);

  const allConversations = [];

  for (const { userId, dirPath: convDir } of userDirs) {
    const dirs = fs.readdirSync(convDir)
      .filter(d => !d.startsWith('chatbot-') && fs.statSync(path.join(convDir, d)).isDirectory())
      .map(d => {
        const dirPath = path.join(convDir, d);
        const stat = fs.statSync(dirPath);
        const useDirPath = path.join(dirPath, 'use');

        // Extract skill name from dispatchResult
        let skillName = null;
        for (const drPath of [path.join(useDirPath, 'dispatchResult.md'), path.join(dirPath, 'train', 'dispatchResult.md'), path.join(dirPath, 'dispatchResult.md')]) {
          if (skillName) break;
          try {
            const dr = fs.readFileSync(drPath, 'utf-8');
            for (const pat of [/Matched Skill[*]*[:\s]+([a-zA-Z0-9_-]+)/i, /Suggested Skill[*]*[:\s]+([a-zA-Z0-9_-]+)/i, /matchedSkillName[:\s"]*([a-zA-Z0-9_-]+)/i]) {
              const m = dr.match(pat);
              if (m && m[1] !== 'none' && m[1] !== 'null') { skillName = m[1]; break; }
            }
          } catch {}
        }

        // Extract title from meta.json
        let title = null;
        let metaData = null;
        try { metaData = JSON.parse(fs.readFileSync(path.join(dirPath, 'meta.json'), 'utf-8')); title = metaData.title; } catch {}
        if (!title) {
          for (const dlPath of [path.join(useDirPath, 'dispatch.log'), path.join(dirPath, 'dispatch.log')]) {
            try { const m = fs.readFileSync(dlPath, 'utf-8').match(/"question"\s*:\s*"([^"]+)"/); if (m) { title = m[1].substring(0, 60); break; } } catch {}
          }
        }

        // Collect files from all locations
        let useFiles = safeFilesInDir(useDirPath);
        const trainFiles = safeFilesInDir(path.join(dirPath, 'train'));
        const rootFiles = safeFilesInDir(dirPath);

        // Support ChatSession format: if messages.jsonl exists at root but no use/messages.log,
        // include it so the frontend knows to load from jsonl
        const hasJsonl = rootFiles.includes('messages.jsonl');
        const hasUseMessagesLog = useFiles.includes('messages.log');
        if (hasJsonl && !hasUseMessagesLog) {
          useFiles = [...useFiles, 'messages.jsonl'];
        }

        return {
          id: d, createdAt: stat.birthtime, skillName, title,
          userId,
          files: rootFiles, useFiles,
          trainFiles,
          outputFiles: safeFilesInDir(path.join(dirPath, 'output')),
          messageCount: metaData?.messageCount || metaData?.stats?.messageCount || 0,
          status: metaData?.status || null,
        };
      });

    allConversations.push(...dirs);
  }

  allConversations.sort((a, b) => new Date(b.createdAt) - new Date(a.createdAt));
  res.json(allConversations);
}));

router.get('/conversations/:id/file/*', requireAuth, h((req, res) => {
  const baseDir = getInnovator().conversationsDir;
  const convId = req.params.id;
  const relFile = req.params[0];

  // Try current user's directory first
  let userConvDir = path.join(baseDir, `user-${req.user.id}`);
  let filePath = path.join(userConvDir, convId, relFile);

  // If not found and user is admin, search all user-* directories
  if (!fs.existsSync(filePath) && req.user.role === 'admin') {
    try {
      const entries = fs.readdirSync(baseDir);
      for (const entry of entries) {
        if (!entry.startsWith('user-')) continue;
        const candidatePath = path.join(baseDir, entry, convId, relFile);
        if (fs.existsSync(candidatePath)) {
          userConvDir = path.join(baseDir, entry);
          filePath = candidatePath;
          break;
        }
      }
    } catch {}
  }

  if (!path.resolve(filePath).startsWith(path.resolve(baseDir))) return res.status(403).json({ error: 'Access denied' });
  if (!fs.existsSync(filePath)) return res.status(404).json({ error: 'File not found' });
  res.json({ name: relFile, content: fs.readFileSync(filePath, 'utf-8'), size: fs.statSync(filePath).size });
}));

router.get('/conversations/:id/evolve', requireAuth, h((req, res) => {
  const evolveDir = path.join(getInnovator().conversationsDir, `user-${req.user.id}`, req.params.id, 'evolve');
  if (!fs.existsSync(evolveDir)) return res.json([]);
  const versions = fs.readdirSync(evolveDir)
    .filter(d => d.match(/^v\d+$/) && fs.statSync(path.join(evolveDir, d)).isDirectory())
    .map(v => {
      const vDir = path.join(evolveDir, v);
      const files = fs.readdirSync(vDir).map(f => ({ name: f, path: `evolve/${v}/${f}`, size: fs.statSync(path.join(vDir, f)).size }));
      let score = null;
      try { const d = JSON.parse(fs.readFileSync(path.join(vDir, 'score.json'), 'utf-8')); score = d.totalScore || d.score; } catch {}
      return { version: v, files, score };
    })
    .sort((a, b) => parseInt(a.version.slice(1)) - parseInt(b.version.slice(1)));
  res.json(versions);
}));

router.get('/conversations/:id/evolve/:version/:file', requireAuth, h((req, res) => {
  const filePath = path.join(getInnovator().conversationsDir, `user-${req.user.id}`, req.params.id, 'evolve', req.params.version, req.params.file);
  if (!fs.existsSync(filePath)) return res.status(404).json({ error: 'File not found' });
  res.json({ name: req.params.file, content: fs.readFileSync(filePath, 'utf-8'), size: fs.statSync(filePath).size });
}));

// ══════════════════════════════════════
// Skill Artifacts
// ══════════════════════════════════════

function resolveSkillDir(skillName) {
  const innovator = getInnovator();
  const skill = innovator.registry.getSkill(skillName);
  let dir = skill?.directory;
  if (dir && fs.existsSync(dir)) return dir;
  const basename = dir ? path.basename(dir) : null;
  if (basename && basename !== skillName) { dir = path.join(config.skillsDataDir, basename); if (fs.existsSync(dir)) return dir; }
  dir = path.join(config.skillsDataDir, skillName); if (fs.existsSync(dir)) return dir;
  dir = path.join(innovator.skillsDir, skillName); if (fs.existsSync(dir)) return dir;
  // 搜索训练目录
  const { resolveTrainedSkillDir: resolveTrainedDir } = require('../services/skill-dir-manager');
  const trainedDir = resolveTrainedDir(skillName);
  if (trainedDir) return trainedDir;
  return null;
}

router.get('/skills/:name/artifacts', h((req, res) => {
  const skillDir = resolveSkillDir(req.params.name);
  if (!skillDir) return res.status(404).json({ error: 'Skill directory not found' });
  const maxDepth = Math.min(parseInt(req.query.depth) || 3, 10);

  function walk(dir, prefix = '', depth = 0) {
    const items = [];
    for (const entry of safeReadDir(dir)) {
      const full = path.join(dir, entry), rel = prefix ? `${prefix}/${entry}` : entry;
      let stat; try { stat = fs.statSync(full); } catch { continue; }
      if (stat.isDirectory()) {
        items.push(depth < maxDepth
          ? { path: rel, type: 'dir', children: walk(full, rel, depth + 1) }
          : { path: rel, type: 'dir', children: [], truncated: true });
      } else {
        items.push({ path: rel, type: 'file', size: stat.size, modified: stat.mtime });
      }
    }
    return items;
  }
  res.json(walk(skillDir));
}));

router.get('/skills/:name/artifact/*', h((req, res) => {
  const skillDir = resolveSkillDir(req.params.name);
  if (!skillDir) return res.status(404).json({ error: 'Skill not found' });
  const filePath = path.join(skillDir, req.params[0]);
  if (!path.resolve(filePath).startsWith(path.resolve(skillDir))) return res.status(403).json({ error: 'Access denied' });
  if (!fs.existsSync(filePath)) return res.status(404).json({ error: 'File not found' });
  res.json({ path: req.params[0], content: fs.readFileSync(filePath, 'utf-8'), size: fs.statSync(filePath).size, modified: fs.statSync(filePath).mtime });
}));

// ══════════════════════════════════════
// API Conversation Logs
// ══════════════════════════════════════

const API_CONV_DIR = path.resolve(__dirname, '../../apiconversation');

function parseLogLines(filePath) {
  try { return fs.readFileSync(filePath, 'utf-8').split('\n').filter(l => l.trim()); } catch { return []; }
}

router.get('/apiconversation', h((req, res) => {
  if (!fs.existsSync(API_CONV_DIR)) return res.json([]);
  const results = [];
  for (const entry of fs.readdirSync(API_CONV_DIR, { withFileTypes: true })) {
    if (entry.isDirectory() && /^\d{4}-\d{2}-\d{2}$/.test(entry.name)) {
      const dateDir = path.join(API_CONV_DIR, entry.name);
      const files = fs.readdirSync(dateDir).filter(f => f.endsWith('.jsonl'));
      let totalCount = 0, totalSize = 0;
      const jobFiles = [];
      for (const f of files) {
        totalCount += parseLogLines(path.join(dateDir, f)).length;
        totalSize += fs.statSync(path.join(dateDir, f)).size;
        if (f !== 'access.jsonl') jobFiles.push(f.replace('.jsonl', ''));
      }
      results.push({ date: entry.name, count: totalCount, size: totalSize, files: files.length, jobs: jobFiles });
    } else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
      const filePath = path.join(API_CONV_DIR, entry.name);
      results.push({ date: entry.name.replace('.jsonl', ''), count: parseLogLines(filePath).length, size: fs.statSync(filePath).size, files: 1, jobs: [], legacy: true });
    }
  }
  res.json(results.sort((a, b) => b.date.localeCompare(a.date)));
}));

router.get('/apiconversation/:date', h((req, res) => {
  const { date } = req.params;
  const filterFile = req.query.file;
  let all = [];
  const dateDir = path.join(API_CONV_DIR, date);
  const legacyFile = path.join(API_CONV_DIR, `${date}.jsonl`);

  if (fs.existsSync(dateDir) && fs.statSync(dateDir).isDirectory()) {
    for (const f of fs.readdirSync(dateDir).filter(f => f.endsWith('.jsonl'))) {
      const source = f.replace('.jsonl', '');
      if (filterFile && source !== filterFile) continue;
      all.push(...parseLogLines(path.join(dateDir, f)).map(l => { try { return { ...JSON.parse(l), _source: source }; } catch { return null; } }).filter(Boolean));
    }
  } else if (fs.existsSync(legacyFile)) {
    all = parseLogLines(legacyFile).map(l => { try { return { ...JSON.parse(l), _source: 'access' }; } catch { return null; } }).filter(Boolean);
  } else {
    return res.status(404).json({ error: 'Log not found' });
  }

  all.reverse();
  const categories = { search: { label: '🔍 搜索', entries: [] }, download: { label: '📦 下载', entries: [] }, create: { label: '🛠 创建', entries: [] }, feedback: { label: '💬 反馈', entries: [] }, jobs: { label: '⏳ Job', entries: [] }, auth_fail: { label: '🔒 认证失败', entries: [] }, other: { label: '📋 其他', entries: [] } };
  for (const e of all) {
    const p = e.path || '';
    const cat = (e.status === 401 || e.status === 403) ? 'auth_fail' : p.includes('/search') ? 'search' : p.includes('/download') ? 'download' : p.includes('/feedback') ? 'feedback' : (p.includes('/create') || e.action?.includes('创建')) ? 'create' : p.includes('/jobs') ? 'jobs' : 'other';
    categories[cat].entries.push(e);
  }

  const summary = Object.fromEntries(Object.entries(categories).map(([k, v]) => [k, { label: v.label, count: v.entries.length }]));
  const fileList = fs.existsSync(dateDir) && fs.statSync(dateDir).isDirectory() ? fs.readdirSync(dateDir).filter(f => f.endsWith('.jsonl')).map(f => f.replace('.jsonl', '')) : [];
  res.json({ date, total: all.length, summary, categories, files: fileList });
}));

module.exports = router;