宜博 / lemonboss

#!/usr/bin/env node
/**
 * Extract spec content from PRD HTML to markdown.
 *
 *   node scripts/extract-spec.js [source-dir]
 *
 * Default source-dir: docs/prd/newfunction/
 * Output: <source-dir>/_spec/<name>.md
 *
 * Three modes (auto-classified per file):
 *   full-extract   — default; strips HTML chrome, folds <pre class="code">
 *                    while preserving <pre class="code shape"> as TS code.
 *   shell-strip    — for agents-*.html (Agent system-prompt docs);
 *                    keeps ALL <pre> content (incl. few-shot) unfolded.
 *   skip-svg-heavy — when SVG bytes > 50% of total; writes only a stub
 *                    pointer back to the HTML (Coder reads the original).
 *
 * Pure regex (no cheerio) — the HTML is hand-crafted and consistent.
 */

const fs   = require('fs');
const path = require('path');

// CLI: [--check] [source-dir]
//   default mode writes to disk; --check verifies disk matches what would be generated.
const args        = process.argv.slice(2);
const CHECK_MODE  = args.includes('--check');
const SOURCE_DIR  = path.resolve(args.find(a => !a.startsWith('--')) || path.join(__dirname, '..', 'docs/prd/newfunction'));
const OUT_DIR     = path.join(SOURCE_DIR, '_spec');
const SVG_RATIO_THRESHOLD = 0.5;

// ───────────────────────── HTML helpers ─────────────────────────

const decode = s => s
  .replace(/&lt;/g,   '<')
  .replace(/&gt;/g,   '>')
  .replace(/&quot;/g, '"')
  .replace(/&#39;/g,  "'")
  .replace(/&nbsp;/g, ' ')
  .replace(/&amp;/g,  '&');

const stripSpans     = s => s.replace(/<\/?span[^>]*>/g, '');
const stripCodeWrap  = s => s.replace(/<code[^>]*>/g, '`').replace(/<\/code>/g, '`');
const inlineFmt      = s => s
  .replace(/<(?:b|strong)>/g, '**').replace(/<\/(?:b|strong)>/g, '**')
  .replace(/<i>/g, '_').replace(/<\/i>/g, '_')
  .replace(/<\/?u>/g, '');
const inlineLinks    = s => s.replace(
  /<a\s+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g,
  (_, href, text) => `[${text.trim()}](${href})`
);

const cleanInline = s => decode(inlineFmt(stripSpans(stripCodeWrap(inlineLinks(s)))))
  .trim()
  .replace(/\s+/g, ' ');

const cleanCode = s => decode(stripSpans(s)).replace(/^\n+/, '').replace(/\s+$/, '');

// Final pass: strip leading whitespace on non-code-fence lines + collapse blank-line runs.
// Code blocks (between ```...```) keep their indent intact.
function cleanupWhitespace(md) {
  const lines = md.split('\n');
  let inFence = false;
  const out = [];
  for (const line of lines) {
    if (line.startsWith('```')) {
      inFence = !inFence;
      out.push(line);
      continue;
    }
    out.push(inFence ? line : line.replace(/^[ \t]+/, ''));
  }
  return out.join('\n').replace(/\n{3,}/g, '\n\n').trim();
}

// ───────────────────────── meta + classify ─────────────────────────

function extractMeta(html) {
  const titleM = html.match(/<title>([^<]+)<\/title>/);
  const h1M    = html.match(/<h1>([\s\S]*?)<\/h1>/);
  const deckM  = html.match(/<p class="deck">([\s\S]*?)<\/p>/);
  return {
    title: h1M ? cleanInline(h1M[1]) : (titleM ? titleM[1].trim() : 'Untitled'),
    deck:  deckM ? cleanInline(deckM[1]) : '',
  };
}

function svgRatio(html) {
  let svgBytes = 0;
  html.replace(/<svg[\s\S]*?<\/svg>/g, m => { svgBytes += m.length; return ''; });
  return html.length === 0 ? 0 : svgBytes / html.length;
}

function classify(name, html) {
  if (/^agents-/.test(name))           return 'shell-strip';
  if (svgRatio(html) > SVG_RATIO_THRESHOLD) return 'skip-svg-heavy';
  return 'full-extract';
}

// ───────────────────────── block converters ─────────────────────────

function tableToMd(tableHtml) {
  const headers = [];
  const rows    = [];

  const headM = tableHtml.match(/<thead>([\s\S]*?)<\/thead>/);
  if (headM) {
    headM[1].replace(/<th[^>]*>([\s\S]*?)<\/th>/g, (_, c) => {
      headers.push(cleanInline(c));
      return '';
    });
  }

  const bodyM    = tableHtml.match(/<tbody>([\s\S]*?)<\/tbody>/);
  const bodyHtml = bodyM ? bodyM[1] : tableHtml;

  bodyHtml.replace(/<tr>([\s\S]*?)<\/tr>/g, (_, tr) => {
    const cells = [];
    tr.replace(/<td[^>]*>([\s\S]*?)<\/td>/g, (__, c) => {
      cells.push(cleanInline(c));
      return '';
    });
    if (cells.length) rows.push(cells);
    return '';
  });

  if (!headers.length && rows.length) {
    headers.push(...rows[0].map(() => ''));
  }

  if (!rows.length) return '';
  const sep = headers.map(() => '---').join(' | ');
  return `| ${headers.join(' | ')} |\n| ${sep} |\n` + rows.map(r => `| ${r.join(' | ')} |`).join('\n');
}

function listToMd(listHtml, prefix = '- ') {
  const items = [];
  listHtml.replace(/<li[^>]*>([\s\S]*?)<\/li>/g, (_, c) => {
    items.push(prefix + cleanInline(c));
    return '';
  });
  return items.join('\n');
}

function olToMd(listHtml) {
  let i = 0;
  const items = [];
  listHtml.replace(/<li[^>]*>([\s\S]*?)<\/li>/g, (_, c) => {
    i++;
    items.push(`${i}. ` + cleanInline(c));
    return '';
  });
  return items.join('\n');
}

// ───────────────────────── full-extract mode ─────────────────────────

function fullExtract(html, meta) {
  const articleM = html.match(/<article class="body">([\s\S]*?)<\/article>/);
  if (!articleM) return `# ${meta.title}\n\n[ERROR: <article class="body"> not found]\n`;
  let body = articleM[1];

  // 1. Replace SVG blocks with placeholder comment (preserve structure of doc)
  body = body.replace(/<svg[\s\S]*?<\/svg>/g, '<!-- 图见原 HTML -->');

  // 2. <pre class="code shape"> → ts code block (NORMATIVE signatures)
  body = body.replace(/<pre class="code shape"[^>]*>([\s\S]*?)<\/pre>/g, (_, inner) =>
    '\n```ts\n' + cleanCode(inner) + '\n```\n');

  // 3. <pre class="code reference"> → ts code block, marked as non-normative
  body = body.replace(/<pre class="code reference"[^>]*>([\s\S]*?)<\/pre>/g, (_, inner) =>
    '\n```ts\n// (参考实现, 非规范)\n' + cleanCode(inner) + '\n```\n');

  // 4. <pre class="code"> default (no shape/reference) → js code block
  body = body.replace(/<pre class="code">([\s\S]*?)<\/pre>/g, (_, inner) =>
    '\n```js\n' + cleanCode(inner) + '\n```\n');

  // 5. <pre class="tree"> → text code block
  body = body.replace(/<pre class="tree">([\s\S]*?)<\/pre>/g, (_, inner) =>
    '\n```\n' + cleanCode(inner) + '\n```\n');

  // 6. Remaining <pre> → text block
  body = body.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/g, (_, inner) =>
    '\n```\n' + cleanCode(inner) + '\n```\n');

  // 7. <table> → markdown table
  body = body.replace(/<table[^>]*>([\s\S]*?)<\/table>/g, (m) => {
    const md = tableToMd(m);
    return md ? '\n\n' + md + '\n\n' : '';
  });

  // 8. Specialized lists first
  body = body.replace(/<ul class="contract">([\s\S]*?)<\/ul>/g, (m) => '\n' + listToMd(m) + '\n');
  body = body.replace(/<ul class="tests">([\s\S]*?)<\/ul>/g,    (m) => '\n' + listToMd(m, '- tests: ') + '\n');

  // 9. Generic ordered + unordered
  body = body.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/g, (m) => '\n' + olToMd(m)   + '\n');
  body = body.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/g, (m) => '\n' + listToMd(m) + '\n');

  // 10. <details><summary>X</summary><div class="inner">Y</div></details> → ### X + Y
  body = body.replace(
    /<details[^>]*>\s*<summary>([\s\S]*?)<\/summary>\s*<div class="inner">([\s\S]*?)<\/div>\s*<\/details>/g,
    (_, sum, inner) => '\n### ' + cleanInline(sum) + '\n' + inner + '\n'
  );

  // 11. <code class="path mono"> → "· `path/to/file`"
  body = body.replace(/<code class="path mono"[^>]*>([\s\S]*?)<\/code>/g, (_, c) =>
    '· `' + decode(stripSpans(c)).trim() + '`');

  // 12. Headings
  body = body.replace(/<h2>([\s\S]*?)<\/h2>/g, (_, c) => '\n## '   + cleanInline(c) + '\n');
  body = body.replace(/<h3>([\s\S]*?)<\/h3>/g, (_, c) => '\n### '  + cleanInline(c) + '\n');
  body = body.replace(/<h4>([\s\S]*?)<\/h4>/g, (_, c) => '\n#### ' + cleanInline(c) + '\n');

  // 13. <p class="sig"> → italic note (blockquote)
  body = body.replace(/<p class="sig">([\s\S]*?)<\/p>/g, (_, c) =>
    '\n> _' + cleanInline(c) + '_\n');

  // 14. <p class="deck"> already extracted at meta level — strip
  body = body.replace(/<p class="deck">[\s\S]*?<\/p>/g, '');

  // 15. <p> default
  body = body.replace(/<p[^>]*>([\s\S]*?)<\/p>/g, (_, c) => '\n' + cleanInline(c) + '\n');

  // 16. Strip wrappers (callout / section / aside / div) — keep inner content
  body = body.replace(/<div class="callout[^"]*">([\s\S]*?)<\/div>/g, '\n\n$1\n\n');
  body = body.replace(/<aside[^>]*>([\s\S]*?)<\/aside>/g, '\n$1\n');
  body = body.replace(/<\/?(?:section|article|div|nav|header|footer)[^>]*>/g, '\n');

  // 17. Final inline cleanup pass (catches anything missed)
  body = stripCodeWrap(body);
  body = inlineFmt(body);
  body = inlineLinks(body);
  body = stripSpans(body);
  body = decode(body);

  // 18. Collapse blank lines
  body = cleanupWhitespace(body);

  return `# ${meta.title}\n\n${meta.deck ? '> ' + meta.deck + '\n\n' : ''}${body}\n`;
}

// ───────────────────────── shell-strip mode ─────────────────────────

function shellStrip(html, meta) {
  const articleM = html.match(/<article class="body">([\s\S]*?)<\/article>/);
  if (!articleM) return `# ${meta.title}\n\n[ERROR: <article class="body"> not found]\n`;
  let body = articleM[1];

  // SVG → placeholder
  body = body.replace(/<svg[\s\S]*?<\/svg>/g, '<!-- 图见原 HTML -->');

  // ALL <pre> → fenced code block (preserve full content, incl. few-shot)
  body = body.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/g, (_, inner) =>
    '\n```\n' + cleanCode(inner) + '\n```\n');

  // Tables
  body = body.replace(/<table[^>]*>([\s\S]*?)<\/table>/g, (m) => {
    const md = tableToMd(m);
    return md ? '\n\n' + md + '\n\n' : '';
  });

  // Lists
  body = body.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/g, (m) => '\n' + olToMd(m)   + '\n');
  body = body.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/g, (m) => '\n' + listToMd(m) + '\n');

  // Details
  body = body.replace(
    /<details[^>]*>\s*<summary>([\s\S]*?)<\/summary>\s*<div class="inner">([\s\S]*?)<\/div>\s*<\/details>/g,
    (_, sum, inner) => '\n### ' + cleanInline(sum) + '\n' + inner + '\n'
  );

  // Headings
  body = body.replace(/<h2>([\s\S]*?)<\/h2>/g, (_, c) => '\n## '   + cleanInline(c) + '\n');
  body = body.replace(/<h3>([\s\S]*?)<\/h3>/g, (_, c) => '\n### '  + cleanInline(c) + '\n');
  body = body.replace(/<h4>([\s\S]*?)<\/h4>/g, (_, c) => '\n#### ' + cleanInline(c) + '\n');

  // Paragraphs
  body = body.replace(/<p class="deck">[\s\S]*?<\/p>/g, '');
  body = body.replace(/<p class="sig">([\s\S]*?)<\/p>/g, (_, c) => '\n> _' + cleanInline(c) + '_\n');
  body = body.replace(/<p[^>]*>([\s\S]*?)<\/p>/g, (_, c) => '\n' + cleanInline(c) + '\n');

  // Path-mono codes
  body = body.replace(/<code class="path mono"[^>]*>([\s\S]*?)<\/code>/g, (_, c) =>
    '· `' + decode(stripSpans(c)).trim() + '`');

  // Wrappers
  body = body.replace(/<div class="callout[^"]*">([\s\S]*?)<\/div>/g, '\n\n$1\n\n');
  body = body.replace(/<aside[^>]*>([\s\S]*?)<\/aside>/g, '\n$1\n');
  body = body.replace(/<\/?(?:section|article|div|nav|header|footer)[^>]*>/g, '\n');

  body = stripCodeWrap(body);
  body = inlineFmt(body);
  body = inlineLinks(body);
  body = stripSpans(body);
  body = decode(body);
  body = cleanupWhitespace(body);

  return `# ${meta.title}\n\n> _Shell-stripped: preserves full prompt content for Agent system templates_\n\n${meta.deck ? '> ' + meta.deck + '\n\n' : ''}${body}\n`;
}

// ───────────────────────── skip stub ─────────────────────────

function skipStub(name, meta, ratio) {
  return `# ${meta.title}\n\n` +
    `> _SVG ratio: ${(ratio * 100).toFixed(0)}% — 图密集页，整页见 [../${name}.html](../${name}.html)_\n\n` +
    (meta.deck ? meta.deck + '\n' : '');
}

// ───────────────────────── main ─────────────────────────

function processFile(filePath) {
  const name  = path.basename(filePath, '.html');
  const html  = fs.readFileSync(filePath, 'utf8');
  const meta  = extractMeta(html);
  const ratio = svgRatio(html);
  const mode  = classify(name, html);

  let md;
  if      (mode === 'skip-svg-heavy') md = skipStub(name, meta, ratio);
  else if (mode === 'shell-strip')    md = shellStrip(html, meta);
  else                                md = fullExtract(html, meta);

  const outPath = path.join(OUT_DIR, name + '.md');
  let drift = false;
  if (CHECK_MODE) {
    const existing = fs.existsSync(outPath) ? fs.readFileSync(outPath, 'utf8') : null;
    drift = existing !== md;
  } else {
    fs.writeFileSync(outPath, md);
  }

  return { name, mode, srcBytes: html.length, dstBytes: md.length, svgRatio: ratio, drift };
}

function main() {
  if (!CHECK_MODE && !fs.existsSync(OUT_DIR)) fs.mkdirSync(OUT_DIR, { recursive: true });

  const files = fs.readdirSync(SOURCE_DIR)
    .filter(f => f.endsWith('.html'))
    .map(f => path.join(SOURCE_DIR, f))
    .sort();

  if (!files.length) {
    console.error(`No .html files in ${SOURCE_DIR}`);
    process.exit(1);
  }

  const results = [];
  for (const f of files) {
    try {
      results.push(processFile(f));
    } catch (e) {
      console.error(`✗ ${path.basename(f)}: ${e.message}`);
    }
  }

  if (CHECK_MODE) {
    const drifted = results.filter(r => r.drift);
    if (drifted.length) {
      console.error('');
      console.error(`✗ ${drifted.length} _spec/*.md file(s) out of sync with HTML source:`);
      drifted.forEach(r => console.error(`    ${r.name}.md`));
      console.error('');
      console.error('Run:  npm run spec:extract  (then stage docs/prd/newfunction/_spec/)');
      console.error('');
      process.exit(1);
    }
    console.log(`✓ All ${results.length} _spec/*.md files in sync with HTML source.`);
    return;
  }

  const fullExt = results.filter(r => r.mode === 'full-extract');
  const shell   = results.filter(r => r.mode === 'shell-strip');
  const skip    = results.filter(r => r.mode === 'skip-svg-heavy');

  const totalSrc = results.reduce((a, r) => a + r.srcBytes, 0);
  const totalDst = results.reduce((a, r) => a + r.dstBytes, 0);
  const ratio    = totalSrc ? Math.round(totalDst * 100 / totalSrc) : 0;
  const tokSaved = Math.round((totalSrc - totalDst) / 4);

  console.log('');
  console.log('─'.repeat(78));
  console.log(`Processed ${results.length} files → ${path.relative(process.cwd(), OUT_DIR)}/`);
  console.log('─'.repeat(78));
  console.log('');
  console.log(`  full-extract:    ${fullExt.length} files`);
  console.log(`  shell-strip:     ${shell.length} files  ${shell.length ? '(' + shell.map(r => r.name).join(', ') + ')' : ''}`);
  console.log(`  skip-svg-heavy:  ${skip.length} files  ${skip.length ? '(' + skip.map(r => r.name).join(', ') + ')' : ''}`);
  console.log('');
  console.log(`  total HTML:      ${(totalSrc / 1024).toFixed(1)}KB  (~${Math.round(totalSrc / 4).toLocaleString()} tokens)`);
  console.log(`  total MD:        ${(totalDst / 1024).toFixed(1)}KB  (~${Math.round(totalDst / 4).toLocaleString()} tokens)`);
  console.log(`  ratio:           ${ratio}%   (saved ~${tokSaved.toLocaleString()} tokens / full load)`);
  console.log('');
  console.log('  Top 15 by source size:');
  results
    .sort((a, b) => b.srcBytes - a.srcBytes)
    .slice(0, 15)
    .forEach(r => {
      const pct = Math.round(r.dstBytes * 100 / r.srcBytes);
      const svg = r.svgRatio > 0 ? ` (svg ${(r.svgRatio*100).toFixed(0)}%)` : '';
      console.log(`    ${r.name.padEnd(30)} ${r.mode.padEnd(15)} ${(r.srcBytes/1024).toFixed(1).padStart(5)}KB → ${(r.dstBytes/1024).toFixed(1).padStart(5)}KB (${pct}%)${svg}`);
    });
  console.log('');
}

main();