#!/usr/bin/env python3 """Benchmark 评估器 - create-drama 技能""" import json import sys import re def evaluate(result_json, benchmark_file, testdata_dir): """评估剧本生成结果""" try: result = json.loads(result_json) except: return { "score": 50, "dimensions": {}, "reason": "无法解析输出JSON" } scores = {} # D1: 素材分析准确性 (20%) # 检查是否识别素材类型、提取核心元素、语言检测、风格适配 name = result.get('name', '') title = result.get('title', '') description = result.get('description', '') language = result.get('language', 'zh') # 核心元素评估 d1_score = 0 # 角色和冲突提取 - 检查剧本是否有角色和台词(代表有故事) characters = result.get('characters', []) script = result.get('script', []) if len(characters) >= 2: d1_score += 25 if len(script) >= 5: d1_score += 25 # 语言检测 - 中文字符 vs 英文字符 has_cn = any('\u4e00' <= c <= '\u9fff' for c in (name + title + description)) if (language == 'zh' and has_cn) or (language == 'en' and not has_cn): d1_score += 25 elif language in ['zh', 'en', 'multi']: d1_score += 15 # 风格适配检测 description_len = len(description) if description_len >= 10: d1_score += 25 scores['D1'] = min(100, d1_score) # D2: 角色配置合理性 (20%) # 角色数量(2-5)、音色匹配、站位配置 d2_score = 0 char_count = len(characters) if 2 <= char_count <= 5: d2_score += 40 elif char_count > 5: d2_score += 20 # 音色匹配检查 valid_voices = ['zh_male_m191', 'zh_female_cancan', 'zh_male_shaonianzixin', 'zh_female_sajiaoxuemei', 'zh_male_dayi', 'zh_male_sunwukong', 'en_male_tim', 'en_female_dacey', 'en_female_stokie'] voice_match = 0 for char in characters: voice = char.get('voiceId', '') if any(v in voice for v in valid_voices): voice_match += 1 if char_count > 0: voice_ratio = voice_match / char_count d2_score += int(60 * voice_ratio) scores['D2'] = min(100, d2_score) # D3: 剧本结构完整性 (20%) # Classic 结构(开场2→铺垫4→冲突6→反转4→结局2)、台词数量(10-20)、台词≤30字 d3_score = 0 # 台词数量 script_count = len(script) if 10 <= script_count <= 20: d3_score += 30 elif 5 <= script_count < 10: d3_score += 20 elif script_count > 0: d3_score += 10 # 台词长度检查 long_lines = 0 for line in script: dialogue = line.get('dialogue', '') if len(dialogue) > 30: long_lines += 1 long_ratio = long_lines / script_count if script_count > 0 else 1 if long_ratio < 0.2: d3_score += 30 elif long_ratio < 0.5: d3_score += 20 else: d3_score += 10 # 结构完整性 - 检查是否有 position 字段 has_position = any('position' in line for line in script) if has_position: d3_score += 40 else: d3_score += 20 scores['D3'] = min(100, d3_score) # D4: 图片生成质量 (20%) # 立绘 prompt 含空白脸部、背景图/封面图 prompt d4_score = 0 # 检查空白脸部 blank_face_count = 0 for char in characters: prompt = char.get('imagePrompt', '') if '空白' in prompt or '不要画' in prompt or '纯肉色' in prompt or 'blank' in prompt.lower(): blank_face_count += 1 if char_count > 0: blank_ratio = blank_face_count / char_count if blank_ratio >= 0.8: d4_score += 60 elif blank_ratio >= 0.5: d4_score += 40 elif blank_ratio > 0: d4_score += 20 # 背景图 background_prompt = result.get('backgroundPrompt', '') if background_prompt: d4_score += 20 else: d4_score += 10 # 封面图 cover_prompt = result.get('coverPrompt', '') if cover_prompt: d4_score += 20 else: d4_score += 10 scores['D4'] = min(100, d4_score) # D5: 数据库写入正确性 (20%) # JSON 格式规范性、必填字段完整性 d5_score = 0 # 必填字段检查 required_fields = ['name', 'title', 'language', 'characters', 'script'] field_count = sum(1 for f in required_fields if f in result and result[f]) d5_score += int(40 * (field_count / len(required_fields))) # systemPrompt 检查 system_prompt = result.get('systemPrompt', '') if len(system_prompt) >= 100: d5_score += 30 elif len(system_prompt) >= 50: d5_score += 20 elif system_prompt: d5_score += 10 # tags 和 description 检查 tags = result.get('tags', '') if tags and description: d5_score += 30 elif tags or description: d5_score += 15 scores['D5'] = min(100, d5_score) # 计算总分 total = (scores['D1'] * 0.20 + scores['D2'] * 0.20 + scores['D3'] * 0.20 + scores['D4'] * 0.20 + scores['D5'] * 0.20) # 评分等级 if total >= 95: grade = 'A+' elif total >= 85: grade = 'A' elif total >= 80: grade = 'B+' elif total >= 70: grade = 'B' elif total >= 60: grade = 'C' else: grade = 'D' result_output = { "score": round(total, 2), "grade": grade, "dimensions": scores, "breakdown": { "D1_素材分析准确性": f"{scores['D1']} × 20% = {scores['D1'] * 0.20:.1f}", "D2_角色配置合理性": f"{scores['D2']} × 20% = {scores['D2'] * 0.20:.1f}", "D3_剧本结构完整性": f"{scores['D3']} × 20% = {scores['D3'] * 0.20:.1f}", "D4_图片生成质量": f"{scores['D4']} × 20% = {scores['D4'] * 0.20:.1f}", "D5_数据库写入正确性": f"{scores['D5']} × 20% = {scores['D5'] * 0.20:.1f}" }, "reason": "基于 benchmark_v1.md 5维度评估", "char_count": char_count, "script_count": script_count } print(json.dumps(result_output, indent=2, ensure_ascii=False)) return result_output if __name__ == '__main__': result_file = '/tmp/result.json' with open(result_file, 'r') as f: result_json = f.read() evaluate(result_json, '', '')