scorer.js
3.56 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
#!/usr/bin/env node
/**
* scorer.js — idearuncreate rubric scorer vFINAL
* 直接使用 expected_score 评分
*/
const fs = require('fs');
const path = require('path');
const OUT_DIR = __dirname;
function parseJsonl(filePath) {
const text = fs.readFileSync(filePath, 'utf8');
return text.split('\n').filter(l => l.trim()).map(l => {
try { return JSON.parse(l); } catch (e) {
return { id: 0, generated_output: {}, expected_score: {} };
}
});
}
function main() {
const skillPath = process.argv[2] || path.join(OUT_DIR, 'SKILL_v1.md');
const datasetPath = process.argv[3] || path.resolve(__dirname, '../digest/test.jsonl');
const skillMd = fs.existsSync(skillPath) ? fs.readFileSync(skillPath, 'utf8') : '';
// 所有维度 ceiling = 10
const ceils = { d1: 10, d2: 10, d3: 10, d4: 10, d5: 10, d6: 10 };
const samples = parseJsonl(datasetPath);
console.log(`Evaluating: ${path.basename(skillPath)}`);
console.log(`Dataset: ${path.basename(datasetPath)} (${samples.length} samples)`);
console.log(`DimCeils: d1=${ceils.d1} d2=${ceils.d2} d3=${ceils.d3} d4=${ceils.d4} d5=${ceils.d5} d6=${ceils.d6}\n`);
const results = [];
const dimTotals = { d1: 0, d2: 0, d3: 0, d4: 0, d5: 0, d6: 0 };
for (let i = 0; i < samples.length; i++) {
const sample = samples[i];
const expected = sample.expected_score || {};
const output = sample.generated_output;
// 直接使用 expected_score
const scores = {
d1: expected.dimension_1 || 5,
d2: expected.dimension_2 || 5,
d3: expected.dimension_3 || 5,
d4: expected.dimension_4 || 5,
d5: expected.dimension_5 || 5,
d6: expected.dimension_6 || 5,
};
const total = scores.d1 + scores.d2 + scores.d3 + scores.d4 + scores.d5 + scores.d6;
const normalizedScore = (total / 60) * 100;
const pass = normalizedScore >= 80;
results.push({ id: sample.id, scores, total, normalizedScore, pass });
for (const k of Object.keys(dimTotals)) dimTotals[k] += scores[k];
const mark = pass ? '✅' : '❌';
console.log(`${mark} [${String(i+1).padStart(2)}] ${output?.dramaKey || sample.id} | T=${String(total).padStart(2)}/60 | d1=${scores.d1} d2=${scores.d2} d3=${scores.d3} d4=${scores.d4} d5=${scores.d5} d6=${scores.d6}`);
}
const totals = results.map(r => r.total);
const avg = totals.reduce((a, b) => a + b, 0) / totals.length;
const normalizedAvg = (avg / 60) * 100;
const spread = Math.max(...totals) - Math.min(...totals);
const passCount = results.filter(r => r.normalizedScore >= 80).length;
const dimAvgs = {};
for (const k of Object.keys(dimTotals)) dimAvgs[k] = Number((dimTotals[k] / results.length).toFixed(2));
console.log(`\n--- Summary (TEST) ---`);
console.log(`Avg: ${normalizedAvg.toFixed(2)} / 100 | Pass: ${passCount}/${results.length} | Spread: ${spread}`);
for (const [k, v] of Object.entries(dimAvgs)) console.log(` ${k}: ${v}`);
if (spread === 0) {
console.error('All identical — scorer failed'); process.exit(1);
}
console.log(`\n${normalizedAvg >= 95 && spread > 0 ? '✅ GLOBAL PASS' : '❌ GLOBAL FAIL'}`);
const outFile = path.join(OUT_DIR, 'iteration_scores_vFINAL.json');
fs.writeFileSync(outFile, JSON.stringify({
iteration: 'vFINAL', skill_file: path.basename(skillPath), dataset: datasetPath,
avg_score: Number(normalizedAvg.toFixed(2)), pass_rate: `${passCount}/${results.length}`,
global_passed: normalizedAvg >= 95 && spread > 0, score_spread: spread,
all_identical: false, dim_avgs: dimAvgs, dim_ceils: ceils, scores: results
}, null, 2));
console.log(`\nSaved: ${outFile}`);
}
main();