"""法务评测集批量测试:用 assess() 分诊器跑全部评测问题,输出准确率报告。

用法: python3 run_eval.py
输出: eval_report.json + 控制台统计
"""
import json
from collections import Counter
from bossai import assess


def run_eval(eval_file: str = "eval_set.json") -> dict:
    """跑全部评测问题,对比预期风险等级和实际分诊结果。"""
    with open(eval_file, encoding="utf-8") as f:
        items = json.load(f)

    results = []
    correct = 0
    total = len(items)

    for item in items:
        card = assess(item["question"])
        actual = card.level
        expected = item["expected_risk"]
        is_correct = actual == expected
        if is_correct:
            correct += 1
        results.append({
            "id": item["id"],
            "question": item["question"],
            "category": item["category"],
            "region": item["region"],
            "expected_risk": expected,
            "actual_risk": actual,
            "correct": is_correct,
        })

    # 按分类统计
    by_category = {}
    for r in results:
        cat = r["category"]
        if cat not in by_category:
            by_category[cat] = {"total": 0, "correct": 0}
        by_category[cat]["total"] += 1
        if r["correct"]:
            by_category[cat]["correct"] += 1

    # 按风险等级统计
    by_risk = {}
    for r in results:
        risk = r["expected_risk"]
        if risk not in by_risk:
            by_risk[risk] = {"total": 0, "correct": 0}
        by_risk[risk]["total"] += 1
        if r["correct"]:
            by_risk[risk]["correct"] += 1

    # 混淆矩阵
    confusion = Counter()
    for r in results:
        confusion[(r["expected_risk"], r["actual_risk"])] += 1

    accuracy = correct / total if total else 0
    report = {
        "total": total,
        "correct": correct,
        "accuracy": round(accuracy, 4),
        "by_category": {k: {**v, "accuracy": round(v["correct"] / v["total"], 4)} for k, v in by_category.items()},
        "by_risk": {k: {**v, "accuracy": round(v["correct"] / v["total"], 4)} for k, v in by_risk.items()},
        "confusion_matrix": {f"{k[0]}->{k[1]}": v for k, v in sorted(confusion.items())},
        "results": results,
    }

    with open("eval_report.json", "w", encoding="utf-8") as f:
        json.dump(report, f, ensure_ascii=False, indent=2)

    return report


if __name__ == "__main__":
    report = run_eval()
    print(f"\n===== 法务评测报告 =====")
    print(f"总数: {report['total']}  正确: {report['correct']}  准确率: {report['accuracy']:.1%}")
    print(f"\n按分类:")
    for cat, stats in sorted(report["by_category"].items()):
        print(f"  {cat:4} {stats['correct']}/{stats['total']} = {stats['accuracy']:.1%}")
    print(f"\n按风险等级:")
    for risk, stats in sorted(report["by_risk"].items()):
        print(f"  {risk:6} {stats['correct']}/{stats['total']} = {stats['accuracy']:.1%}")
    print(f"\n混淆矩阵(expected->actual: count):")
    for k, v in sorted(report["confusion_matrix"].items()):
        if not k.startswith(("green->green", "amber->amber", "orange->orange", "red->red")):
            print(f"  {k}: {v}")
    print(f"\n详细报告已写入 eval_report.json")