#!/usr/bin/env python3 """Benchmark 评估脚本 - 互动剧编剧""" import json import sys def evaluate(skill_content, output_content, benchmark_def): """评估互动剧编剧结果""" # 默认弱分 scores = { "completeness": 60, "practicality": 60, "tool_support": 60, "methodology": 60, "source_quality": 60, "platform_coverage": 60 } # 解析 skill 内容检查关键要素 skill_lower = skill_content.lower() output_lower = output_content.lower() # completeness: 知识完整性 - 检查是否覆盖方法论、工具、平台特性 completeness_checks = [ ("六段式" in skill_content or "六段" in skill_content, 15), ("开场" in skill_content and "冲突" in skill_content and "选择" in skill_content, 10), ("分支" in skill_content and ("结局" in skill_content or "高潮" in skill_content), 10), ("工具" in skill_content and ("twine" in skill_lower or "ink" in skill_lower or "ren'py" in skill_lower), 10), ] completeness_score = sum(score for check, score in completeness_checks if check) scores["completeness"] = min(100, 60 + completeness_score) # practicality: 实用性 - 检查是否有可直接指导创作实践的内容 practicality_checks = [ ("示例" in skill_content or "例子" in skill_content, 20), ("格式" in skill_content or "模板" in skill_content or "json" in skill_lower, 15), ("检查清单" in skill_content or "checklist" in skill_lower, 10), ] practicality_score = sum(score for check, score in practicality_checks if check) scores["practicality"] = min(100, 60 + practicality_score) # tool_support: 工具支持 - 检查是否有工具推荐、语法示例 tool_checks = [ ("twine" in skill_lower, 15), ("ink" in skill_lower, 15), ("[[" in skill_content or "==" in skill_content, 10), # 语法标记 ("语法" in skill_content or "syntax" in skill_lower, 10), ] tool_score = sum(score for check, score in tool_checks if check) scores["tool_support"] = min(100, 60 + tool_score) # methodology: 方法论 - 检查分支设计、选择机制、剧本结构 method_checks = [ ("选择点" in skill_content, 15), ("分支" in skill_content and "深度" in skill_content, 15), ("菱形" in skill_content or "树形" in skill_content or "结构" in skill_content, 10), ("变量" in skill_content and ("状态" in skill_content or "condition" in skill_lower), 10), ] method_score = sum(score for check, score in method_checks if check) scores["methodology"] = min(100, 60 + method_score) # source_quality: 信源质量 - 检查是否有官方文档 vs 社区分享 source_checks = [ ("官方" in skill_content or "文档" in skill_content or "documentation" in skill_lower, 20), ("github" in skill_lower, 15), ("star" in skill_lower, 10), ] source_score = sum(score for check, score in source_checks if check) scores["source_quality"] = min(100, 60 + source_score) # platform_coverage: 平台覆盖 - 检查抖音/快手等平台特性 platform_checks = [ ("抖音" in skill_content, 15), ("快手" in skill_content, 15), ("时长" in skill_content and "平台" in skill_content, 10), ("3秒" in skill_content or "开头" in skill_content, 10), ] platform_score = sum(score for check, score in platform_checks if check) scores["platform_coverage"] = min(100, 60 + platform_score) # 计算总分 (按权重) weights = { "completeness": 0.20, "practicality": 0.20, "tool_support": 0.15, "methodology": 0.20, "source_quality": 0.10, "platform_coverage": 0.15 } total = sum(scores[k] * weights[k] for k in weights) result = { "score": round(total, 2), "dimensions": scores, "passed": total >= 90, "reason": "基于 SKILL.md 内容评估" } print(json.dumps(result, indent=2, ensure_ascii=False)) return result if __name__ == '__main__': # 读取参数 if len(sys.argv) < 3: print("Usage: evaluator.py ") sys.exit(1) skill_file = sys.argv[1] output_file = sys.argv[2] try: with open(skill_file, 'r') as f: skill_content = f.read() # 如果输出文件存在则读取,否则用 skill 作为输出 try: with open(output_file, 'r') as f: output_content = f.read() except: output_content = skill_content result = evaluate(skill_content, output_content, None) print(json.dumps(result, indent=2, ensure_ascii=False)) except Exception as e: print(json.dumps({"score": 50, "dimensions": {}, "error": str(e)}))