feat: BossAi 老板AI助手初始版本
包含功能: - 法务参谋/经营决策/生活助理/情绪中心四大AI对话模块 - 专家咨询系统(用户选定→专家确认→咨询→回复→评价) - 角色登录(用户/专家/管理员)+ ProtectedRoute 权限控制 - 管理后台(用户管理/专家管理/AI使用统计/专家活跃度) - 专家工作台(待确认服务/待回复咨询/已回复咨询/服务中用户) - 服务关系取消(用户和专家均可取消,需自己确认) - 全页面移动端响应式适配 - 知识库RAG检索 - 对话历史(普通/保密模式) Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
+93
@@ -0,0 +1,93 @@
|
||||
"""法务评测集批量测试:用 assess() 分诊器跑全部评测问题,输出准确率报告。
|
||||
|
||||
用法: python3 run_eval.py
|
||||
输出: eval_report.json + 控制台统计
|
||||
"""
|
||||
import json
|
||||
from collections import Counter
|
||||
from bossai import assess
|
||||
|
||||
|
||||
def run_eval(eval_file: str = "eval_set.json") -> dict:
|
||||
"""跑全部评测问题,对比预期风险等级和实际分诊结果。"""
|
||||
with open(eval_file, encoding="utf-8") as f:
|
||||
items = json.load(f)
|
||||
|
||||
results = []
|
||||
correct = 0
|
||||
total = len(items)
|
||||
|
||||
for item in items:
|
||||
card = assess(item["question"])
|
||||
actual = card.level
|
||||
expected = item["expected_risk"]
|
||||
is_correct = actual == expected
|
||||
if is_correct:
|
||||
correct += 1
|
||||
results.append({
|
||||
"id": item["id"],
|
||||
"question": item["question"],
|
||||
"category": item["category"],
|
||||
"region": item["region"],
|
||||
"expected_risk": expected,
|
||||
"actual_risk": actual,
|
||||
"correct": is_correct,
|
||||
})
|
||||
|
||||
# 按分类统计
|
||||
by_category = {}
|
||||
for r in results:
|
||||
cat = r["category"]
|
||||
if cat not in by_category:
|
||||
by_category[cat] = {"total": 0, "correct": 0}
|
||||
by_category[cat]["total"] += 1
|
||||
if r["correct"]:
|
||||
by_category[cat]["correct"] += 1
|
||||
|
||||
# 按风险等级统计
|
||||
by_risk = {}
|
||||
for r in results:
|
||||
risk = r["expected_risk"]
|
||||
if risk not in by_risk:
|
||||
by_risk[risk] = {"total": 0, "correct": 0}
|
||||
by_risk[risk]["total"] += 1
|
||||
if r["correct"]:
|
||||
by_risk[risk]["correct"] += 1
|
||||
|
||||
# 混淆矩阵
|
||||
confusion = Counter()
|
||||
for r in results:
|
||||
confusion[(r["expected_risk"], r["actual_risk"])] += 1
|
||||
|
||||
accuracy = correct / total if total else 0
|
||||
report = {
|
||||
"total": total,
|
||||
"correct": correct,
|
||||
"accuracy": round(accuracy, 4),
|
||||
"by_category": {k: {**v, "accuracy": round(v["correct"] / v["total"], 4)} for k, v in by_category.items()},
|
||||
"by_risk": {k: {**v, "accuracy": round(v["correct"] / v["total"], 4)} for k, v in by_risk.items()},
|
||||
"confusion_matrix": {f"{k[0]}->{k[1]}": v for k, v in sorted(confusion.items())},
|
||||
"results": results,
|
||||
}
|
||||
|
||||
with open("eval_report.json", "w", encoding="utf-8") as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2)
|
||||
|
||||
return report
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
report = run_eval()
|
||||
print(f"\n===== 法务评测报告 =====")
|
||||
print(f"总数: {report['total']} 正确: {report['correct']} 准确率: {report['accuracy']:.1%}")
|
||||
print(f"\n按分类:")
|
||||
for cat, stats in sorted(report["by_category"].items()):
|
||||
print(f" {cat:4} {stats['correct']}/{stats['total']} = {stats['accuracy']:.1%}")
|
||||
print(f"\n按风险等级:")
|
||||
for risk, stats in sorted(report["by_risk"].items()):
|
||||
print(f" {risk:6} {stats['correct']}/{stats['total']} = {stats['accuracy']:.1%}")
|
||||
print(f"\n混淆矩阵(expected->actual: count):")
|
||||
for k, v in sorted(report["confusion_matrix"].items()):
|
||||
if not k.startswith(("green->green", "amber->amber", "orange->orange", "red->red")):
|
||||
print(f" {k}: {v}")
|
||||
print(f"\n详细报告已写入 eval_report.json")
|
||||
Reference in New Issue
Block a user