selfreleaseandDevin
5c21e8fcd0
包含功能: - 法务参谋/经营决策/生活助理/情绪中心四大AI对话模块 - 专家咨询系统(用户选定→专家确认→咨询→回复→评价) - 角色登录(用户/专家/管理员)+ ProtectedRoute 权限控制 - 管理后台(用户管理/专家管理/AI使用统计/专家活跃度) - 专家工作台(待确认服务/待回复咨询/已回复咨询/服务中用户) - 服务关系取消(用户和专家均可取消,需自己确认) - 全页面移动端响应式适配 - 知识库RAG检索 - 对话历史(普通/保密模式) Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
94 lines
3.2 KiB
Python
94 lines
3.2 KiB
Python
"""法务评测集批量测试:用 assess() 分诊器跑全部评测问题,输出准确率报告。
|
|
|
|
用法: python3 run_eval.py
|
|
输出: eval_report.json + 控制台统计
|
|
"""
|
|
import json
|
|
from collections import Counter
|
|
from bossai import assess
|
|
|
|
|
|
def run_eval(eval_file: str = "eval_set.json") -> dict:
|
|
"""跑全部评测问题,对比预期风险等级和实际分诊结果。"""
|
|
with open(eval_file, encoding="utf-8") as f:
|
|
items = json.load(f)
|
|
|
|
results = []
|
|
correct = 0
|
|
total = len(items)
|
|
|
|
for item in items:
|
|
card = assess(item["question"])
|
|
actual = card.level
|
|
expected = item["expected_risk"]
|
|
is_correct = actual == expected
|
|
if is_correct:
|
|
correct += 1
|
|
results.append({
|
|
"id": item["id"],
|
|
"question": item["question"],
|
|
"category": item["category"],
|
|
"region": item["region"],
|
|
"expected_risk": expected,
|
|
"actual_risk": actual,
|
|
"correct": is_correct,
|
|
})
|
|
|
|
# 按分类统计
|
|
by_category = {}
|
|
for r in results:
|
|
cat = r["category"]
|
|
if cat not in by_category:
|
|
by_category[cat] = {"total": 0, "correct": 0}
|
|
by_category[cat]["total"] += 1
|
|
if r["correct"]:
|
|
by_category[cat]["correct"] += 1
|
|
|
|
# 按风险等级统计
|
|
by_risk = {}
|
|
for r in results:
|
|
risk = r["expected_risk"]
|
|
if risk not in by_risk:
|
|
by_risk[risk] = {"total": 0, "correct": 0}
|
|
by_risk[risk]["total"] += 1
|
|
if r["correct"]:
|
|
by_risk[risk]["correct"] += 1
|
|
|
|
# 混淆矩阵
|
|
confusion = Counter()
|
|
for r in results:
|
|
confusion[(r["expected_risk"], r["actual_risk"])] += 1
|
|
|
|
accuracy = correct / total if total else 0
|
|
report = {
|
|
"total": total,
|
|
"correct": correct,
|
|
"accuracy": round(accuracy, 4),
|
|
"by_category": {k: {**v, "accuracy": round(v["correct"] / v["total"], 4)} for k, v in by_category.items()},
|
|
"by_risk": {k: {**v, "accuracy": round(v["correct"] / v["total"], 4)} for k, v in by_risk.items()},
|
|
"confusion_matrix": {f"{k[0]}->{k[1]}": v for k, v in sorted(confusion.items())},
|
|
"results": results,
|
|
}
|
|
|
|
with open("eval_report.json", "w", encoding="utf-8") as f:
|
|
json.dump(report, f, ensure_ascii=False, indent=2)
|
|
|
|
return report
|
|
|
|
|
|
if __name__ == "__main__":
|
|
report = run_eval()
|
|
print(f"\n===== 法务评测报告 =====")
|
|
print(f"总数: {report['total']} 正确: {report['correct']} 准确率: {report['accuracy']:.1%}")
|
|
print(f"\n按分类:")
|
|
for cat, stats in sorted(report["by_category"].items()):
|
|
print(f" {cat:4} {stats['correct']}/{stats['total']} = {stats['accuracy']:.1%}")
|
|
print(f"\n按风险等级:")
|
|
for risk, stats in sorted(report["by_risk"].items()):
|
|
print(f" {risk:6} {stats['correct']}/{stats['total']} = {stats['accuracy']:.1%}")
|
|
print(f"\n混淆矩阵(expected->actual: count):")
|
|
for k, v in sorted(report["confusion_matrix"].items()):
|
|
if not k.startswith(("green->green", "amber->amber", "orange->orange", "red->red")):
|
|
print(f" {k}: {v}")
|
|
print(f"\n详细报告已写入 eval_report.json")
|