benchmark_evaluator.py
3.69 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/usr/bin/env python3
"""Benchmark 评估器 - 互动视频短剧选题"""
import json
import sys
import re
def evaluate(result_json, benchmark_file, testdata_dir):
"""评估选题结果"""
try:
result = json.loads(result_json)
except:
return {
"score": 50,
"dimensions": {},
"reason": "无法解析输出JSON"
}
scores = {}
# D1: 题材适配度 (20%)
# 检查是否有冲突性、情感张力、讨论空间
summary = result.get('summary', '')
title = result.get('title', '')
# 高冲突性特征：争吵、矛盾、冲突、出轨、离婚、分手
conflict_indicators = ['矛盾', '冲突', '争吵', '分歧', '对立']
has_conflict = any(word in summary or word in title for word in conflict_indicators)
# 情感张力特征
emotion_words = ['崩溃', '愤怒', '无奈', '绝望', '爆发', '哭泣']
has_emotion = any(word in summary or word in title for word in emotion_words)
if has_conflict and has_emotion:
scores['D1'] = 90
elif has_conflict:
scores['D1'] = 75
else:
scores['D1'] = 55
# D2: 剧情潜力 (20%)
# 检查是否有角色、场景、分支点
roles = result.get('roles', [])
scenes = result.get('scenes', [])
branch_points = result.get('branch_points', [])
if len(roles) >= 2 and len(branch_points) >= 1:
scores['D2'] = 90
elif len(roles) >= 2:
scores['D2'] = 75
else:
scores['D2'] = 55
# D3: 可演绎性 (15%)
# 检查推荐形式是否合理
form = result.get('recommended_form', '')
if form in ['对话形式', '对话形式为主', '对话+旁白混合']:
scores['D3'] = 90
elif form in ['旁白形式', '旁白为主']:
scores['D3'] = 70
else:
scores['D3'] = 60
# D4: 受众广度 (15%)
# 检查目标人群是否广泛
# 婆媳、职场类话题通常受众广泛
if '婆媳' in summary or '婆媳' in title or '家庭' in result.get('recommended_interactive', ''):
scores['D4'] = 85
else:
scores['D4'] = 70
# D5: 互动空间 (20%)
# 检查分支设计
interactive = result.get('recommended_interactive', '')
if interactive in ['AB选择', '投票PK', '多结局'] and len(branch_points) > 0:
scores['D5'] = 90
elif interactive:
scores['D5'] = 75
else:
scores['D5'] = 50
# D6: 时效周期 (10%)
# 婆媳是永恒话题
if '婆媳' in summary or '婆媳' in title or '家庭' in summary:
scores['D6'] = 85
else:
scores['D6'] = 70
# 计算总分
total = (scores['D1'] * 0.20 + scores['D2'] * 0.20 +
scores['D3'] * 0.15 + scores['D4'] * 0.15 +
scores['D5'] * 0.20 + scores['D6'] * 0.10)
result = {
"score": round(total, 2),
"dimensions": scores,
"breakdown": {
"D1_题材适配度": f"{scores['D1']} × 20% = {scores['D1'] * 0.20:.1f}",
"D2_剧情潜力": f"{scores['D2']} × 20% = {scores['D2'] * 0.20:.1f}",
"D3_可演绎性": f"{scores['D3']} × 15% = {scores['D3'] * 0.15:.1f}",
"D4_受众广度": f"{scores['D4']} × 15% = {scores['D4'] * 0.15:.1f}",
"D5_互动空间": f"{scores['D5']} × 20% = {scores['D5'] * 0.20:.1f}",
"D6_时效周期": f"{scores['D6']} × 10% = {scores['D6'] * 0.10:.1f}"
},
"reason": "基于 benchmark_v1.md 6维度评估"
}
print(json.dumps(result, indent=2, ensure_ascii=False))
return result
if __name__ == '__main__':
result_file = '/tmp/result.json'
with open(result_file, 'r') as f:
result_json = f.read()
evaluate(result_json, '', '')