benchmark_evaluator.py
4.81 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
#!/usr/bin/env python3
"""Benchmark 评估脚本 - 互动剧编剧"""
import json
import sys
def evaluate(skill_content, output_content, benchmark_def):
"""评估互动剧编剧结果"""
# 默认弱分
scores = {
"completeness": 60,
"practicality": 60,
"tool_support": 60,
"methodology": 60,
"source_quality": 60,
"platform_coverage": 60
}
# 解析 skill 内容检查关键要素
skill_lower = skill_content.lower()
output_lower = output_content.lower()
# completeness: 知识完整性 - 检查是否覆盖方法论、工具、平台特性
completeness_checks = [
("六段式" in skill_content or "六段" in skill_content, 15),
("开场" in skill_content and "冲突" in skill_content and "选择" in skill_content, 10),
("分支" in skill_content and ("结局" in skill_content or "高潮" in skill_content), 10),
("工具" in skill_content and ("twine" in skill_lower or "ink" in skill_lower or "ren'py" in skill_lower), 10),
]
completeness_score = sum(score for check, score in completeness_checks if check)
scores["completeness"] = min(100, 60 + completeness_score)
# practicality: 实用性 - 检查是否有可直接指导创作实践的内容
practicality_checks = [
("示例" in skill_content or "例子" in skill_content, 20),
("格式" in skill_content or "模板" in skill_content or "json" in skill_lower, 15),
("检查清单" in skill_content or "checklist" in skill_lower, 10),
]
practicality_score = sum(score for check, score in practicality_checks if check)
scores["practicality"] = min(100, 60 + practicality_score)
# tool_support: 工具支持 - 检查是否有工具推荐、语法示例
tool_checks = [
("twine" in skill_lower, 15),
("ink" in skill_lower, 15),
("[[" in skill_content or "==" in skill_content, 10), # 语法标记
("语法" in skill_content or "syntax" in skill_lower, 10),
]
tool_score = sum(score for check, score in tool_checks if check)
scores["tool_support"] = min(100, 60 + tool_score)
# methodology: 方法论 - 检查分支设计、选择机制、剧本结构
method_checks = [
("选择点" in skill_content, 15),
("分支" in skill_content and "深度" in skill_content, 15),
("菱形" in skill_content or "树形" in skill_content or "结构" in skill_content, 10),
("变量" in skill_content and ("状态" in skill_content or "condition" in skill_lower), 10),
]
method_score = sum(score for check, score in method_checks if check)
scores["methodology"] = min(100, 60 + method_score)
# source_quality: 信源质量 - 检查是否有官方文档 vs 社区分享
source_checks = [
("官方" in skill_content or "文档" in skill_content or "documentation" in skill_lower, 20),
("github" in skill_lower, 15),
("star" in skill_lower, 10),
]
source_score = sum(score for check, score in source_checks if check)
scores["source_quality"] = min(100, 60 + source_score)
# platform_coverage: 平台覆盖 - 检查抖音/快手等平台特性
platform_checks = [
("抖音" in skill_content, 15),
("快手" in skill_content, 15),
("时长" in skill_content and "平台" in skill_content, 10),
("3秒" in skill_content or "开头" in skill_content, 10),
]
platform_score = sum(score for check, score in platform_checks if check)
scores["platform_coverage"] = min(100, 60 + platform_score)
# 计算总分 (按权重)
weights = {
"completeness": 0.20,
"practicality": 0.20,
"tool_support": 0.15,
"methodology": 0.20,
"source_quality": 0.10,
"platform_coverage": 0.15
}
total = sum(scores[k] * weights[k] for k in weights)
result = {
"score": round(total, 2),
"dimensions": scores,
"passed": total >= 90,
"reason": "基于 SKILL.md 内容评估"
}
print(json.dumps(result, indent=2, ensure_ascii=False))
return result
if __name__ == '__main__':
# 读取参数
if len(sys.argv) < 3:
print("Usage: evaluator.py <skill_file> <output_file>")
sys.exit(1)
skill_file = sys.argv[1]
output_file = sys.argv[2]
try:
with open(skill_file, 'r') as f:
skill_content = f.read()
# 如果输出文件存在则读取，否则用 skill 作为输出
try:
with open(output_file, 'r') as f:
output_content = f.read()
except:
output_content = skill_content
result = evaluate(skill_content, output_content, None)
print(json.dumps(result, indent=2, ensure_ascii=False))
except Exception as e:
print(json.dumps({"score": 50, "dimensions": {}, "error": str(e)}))