build-search-index.js
3.18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
#!/usr/bin/env node
/**
* Build search-index.json from chapter HTML files.
*
* docs/prd/chapters/{ch}/**.html → assets/search-index.json
*
* Tokenization rules (must match runtime in app.js):
* - lowercase
* - ASCII words/digits: [a-z0-9_]+
* - CJK bigrams: [一-鿿]+ → for each run, push every 2-char window
*
* Output:
* {
* docs: [ { id, title, href, crumbs:[…], text } ],
* invIndex: { token: [ [docId, tf], … ] }
* }
*/
const fs = require('fs');
const path = require('path');
const ROOT = path.resolve(__dirname, '..');
const CHAPT_DIR = path.join(ROOT, 'chapters');
const OUT = path.join(ROOT, 'assets', 'search-index.json');
function walk(dir) {
const out = [];
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
const p = path.join(dir, e.name);
if (e.isDirectory()) out.push(...walk(p));
else if (e.isFile() && e.name.endsWith('.html')) out.push(p);
}
return out;
}
function stripTags(html) {
return html
.replace(/<script[\s\S]*?<\/script>/gi, ' ')
.replace(/<style[\s\S]*?<\/style>/gi, ' ')
.replace(/<!--[\s\S]*?-->/g, ' ')
.replace(/<[^>]+>/g, ' ')
.replace(/ /g, ' ')
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>')
.replace(/"/g, '"')
.replace(/'/g, "'")
.replace(/\s+/g, ' ')
.trim();
}
function pick(html, re) {
const m = re.exec(html);
return m ? stripTags(m[1]) : null;
}
function extractTitle(html) {
return pick(html, /<h1[^>]*>([\s\S]*?)<\/h1>/i) || pick(html, /<title[^>]*>([\s\S]*?)<\/title>/i) || '(无标题)';
}
function extractCrumbs(html) {
const m = /<div class="crumbs"[^>]*>([\s\S]*?)<\/div>/i.exec(html);
if (!m) return [];
return m[1].replace(/<span[^>]*>›<\/span>/g, '|').split('|').map(s => stripTags(s)).filter(Boolean);
}
function tokenize(text) {
const tokens = [];
const lower = text.toLowerCase();
(lower.match(/[a-z0-9_]+/g) || []).forEach(w => tokens.push(w));
const cjk = lower.match(/[㐀-鿿]+/g) || [];
cjk.forEach(run => {
if (run.length === 1) tokens.push(run);
else for (let i = 0; i < run.length - 1; i++) tokens.push(run.slice(i, i + 2));
});
return tokens;
}
function build() {
const files = walk(CHAPT_DIR);
const docs = [];
const inv = {}; // token → Map<docId, tf>
for (const file of files) {
const html = fs.readFileSync(file, 'utf-8');
const title = extractTitle(html);
const crumbs = extractCrumbs(html);
const text = stripTags(html);
const href = path.relative(CHAPT_DIR, file);
const id = docs.length;
docs.push({ id, title, href, crumbs, text: text.slice(0, 2000) });
const tokens = tokenize(text);
const tf = {};
for (const t of tokens) tf[t] = (tf[t] || 0) + 1;
for (const [t, c] of Object.entries(tf)) {
(inv[t] = inv[t] || []).push([id, c]);
}
}
const out = { docs, invIndex: inv };
fs.mkdirSync(path.dirname(OUT), { recursive: true });
fs.writeFileSync(OUT, JSON.stringify(out));
const kb = (fs.statSync(OUT).size / 1024).toFixed(1);
console.log(`✅ search index: ${docs.length} docs, ${Object.keys(inv).length} tokens, ${kb} KB → ${path.relative(ROOT, OUT)}`);
}
build();