From 641e33b834188293fb45d822766756c6f4b06999 Mon Sep 17 00:00:00 2001 From: freedak Date: Fri, 7 Aug 2026 14:55:25 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20QYLAW=20=E6=B3=95=E5=BE=8B=E6=B3=95?= =?UTF-8?q?=E8=A7=84=E7=9F=A5=E8=AF=86=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 语义检索(FAISS + embedding)+ 精确查找(法规名+条号) - RAG 问答(SSE 流式,支持 thinking 折叠显示) - 法规浏览(原文阅读) - 历史记录(检索+对话持久化到 SQLite) - 设置页(系统提示词/模板/LLM 参数可配置) - 检索质量评估脚本 Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .gitignore | 38 ++++ Dockerfile | 39 ++++ README.md | 70 +++++++ app/__init__.py | 1 + app/config.py | 74 ++++++++ app/main.py | 101 ++++++++++ app/models.py | 107 +++++++++++ app/routers/__init__.py | 1 + app/routers/history.py | 76 ++++++++ app/routers/law.py | 51 +++++ app/routers/prompts.py | 88 +++++++++ app/routers/rag.py | 159 ++++++++++++++++ app/routers/search.py | 89 +++++++++ app/routers/stats.py | 20 ++ app/services/__init__.py | 1 + app/services/embedding.py | 79 ++++++++ app/services/history.py | 300 +++++++++++++++++++++++++++++ app/services/law_reader.py | 147 ++++++++++++++ app/services/llm.py | 140 ++++++++++++++ app/services/prompt_config.py | 190 +++++++++++++++++++ app/services/retriever.py | 264 ++++++++++++++++++++++++++ app/static/css/app.css | 2 + app/static/css/tailwind.css | 1 + app/static/index.html | 126 ++++++++++++ app/static/js/app.js | 264 ++++++++++++++++++++++++++ app/static/js/browse.js | 212 +++++++++++++++++++++ app/static/js/history.js | 332 ++++++++++++++++++++++++++++++++ app/static/js/rag.js | 207 ++++++++++++++++++++ app/static/js/search.js | 168 ++++++++++++++++ app/static/js/settings.js | 258 +++++++++++++++++++++++++ requirements.txt | 6 + run.md | 247 ++++++++++++++++++++++++ scripts/build_index.py | 329 ++++++++++++++++++++++++++++++++ scripts/deploy.sh | 107 +++++++++++ scripts/eval_retrieval.py | 348 ++++++++++++++++++++++++++++++++++ scripts/parse_clause.py | 266 ++++++++++++++++++++++++++ scripts/rebuild_faiss.py | 147 ++++++++++++++ scripts/verify_index.py | 101 ++++++++++ tests/test_parse_clause.py | 98 ++++++++++ 39 files changed, 5254 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 app/__init__.py create mode 100644 app/config.py create mode 100644 app/main.py create mode 100644 app/models.py create mode 100644 app/routers/__init__.py create mode 100644 app/routers/history.py create mode 100644 app/routers/law.py create mode 100644 app/routers/prompts.py create mode 100644 app/routers/rag.py create mode 100644 app/routers/search.py create mode 100644 app/routers/stats.py create mode 100644 app/services/__init__.py create mode 100644 app/services/embedding.py create mode 100644 app/services/history.py create mode 100644 app/services/law_reader.py create mode 100644 app/services/llm.py create mode 100644 app/services/prompt_config.py create mode 100644 app/services/retriever.py create mode 100644 app/static/css/app.css create mode 100644 app/static/css/tailwind.css create mode 100644 app/static/index.html create mode 100644 app/static/js/app.js create mode 100644 app/static/js/browse.js create mode 100644 app/static/js/history.js create mode 100644 app/static/js/rag.js create mode 100644 app/static/js/search.js create mode 100644 app/static/js/settings.js create mode 100644 requirements.txt create mode 100644 run.md create mode 100644 scripts/build_index.py create mode 100755 scripts/deploy.sh create mode 100644 scripts/eval_retrieval.py create mode 100644 scripts/parse_clause.py create mode 100644 scripts/rebuild_faiss.py create mode 100644 scripts/verify_index.py create mode 100644 tests/test_parse_clause.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f55a0c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.egg-info/ +dist/ +build/ +.eggs/ + +# 环境 +.env +.venv/ +venv/ +env/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# 数据(不提交) +*.db +*.sqlite +*.faiss +law-kb-data/ + +# 日志 +*.log +logs/ + +# OS +.DS_Store +Thumbs.db + +# 构建产物 +tailwind-input.css diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b1cbe48 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# QYLAW 法律法规知识库 Dockerfile +# 基于 python:3.11-slim,纯 CPU(FAISS 不需 GPU) +FROM python:3.11-slim + +LABEL maintainer="qy123" +LABEL description="QYLAW 法律法规知识库 — FAISS 语义检索 + RAG 问答" + +# 设置工作目录 +WORKDIR /app + +# 安装系统依赖(faiss-cpu 需要 libgomp) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgomp1 \ + && rm -rf /var/lib/apt/lists/* + +# 先复制 requirements 利用 Docker 缓存 +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# 复制应用代码 +COPY app/ ./app/ +COPY scripts/ ./scripts/ + +# 环境变量默认值(容器内通过 --network host 访问 localhost) +ENV LAW_PACK_DIR=/data/law-pack-2026-07-01-markdown \ + LAW_KB_DATA_DIR=/data/law-kb-data \ + EMBEDDING_URL=http://localhost:8003/v1 \ + LLM_URL=http://localhost:7000/v1 \ + APP_PORT=8090 + +# 暴露端口(--network host 模式下仅作记录) +EXPOSE 8090 + +# 健康检查 +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:8090/health')" || exit 1 + +# 启动命令 +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8090"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0f119a --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# QYLAW 法律法规知识库 + +基于 17291 篇现行有效法律法规的语义检索与 RAG 问答系统。 + +## 架构 + +``` +114 (192.168.110.114) +┌────────────────────────────────────┐ +│ law-kb (FastAPI :8090, Docker) │ +│ --network host │ +│ │ +│ 静态页(/) + API(/api/*) │ +│ FAISS + SQLite + 法规原文 │ +│ │ +│ → localhost:8003 embedding │ +│ → localhost:7000 qwen35 │ +└────────────────────────────────────┘ + ↑ + 本机浏览器 http://192.168.110.114:8090 +``` + +## 功能 + +- **语义检索**:自然语言查询,返回相关法规条文,支持按类别/省份过滤 +- **RAG 问答**:提问 → 检索相关条文 → LLM 生成带引用的答案(流式) +- **原文浏览**:按类别/省份浏览法规列表,查看完整原文 + +## 快速开始 + +```bash +# 1. 传输法规数据到 114 +rsync -avz law-pack-2026-07-01-markdown/ nvidia@192.168.110.114:/data/law-pack-2026-07-01-markdown/ + +# 2. 构建索引 +docker run --rm --network host \ + -v /data/law-pack-2026-07-01-markdown:/data/law-pack-2026-07-01-markdown:ro \ + -v /data/law-kb-data:/data/law-kb-data \ + law-kb:latest \ + python scripts/build_index.py --mode full + +# 3. 启动服务 +docker run -d --name law-kb --network host --restart unless-stopped \ + -v /data/law-pack-2026-07-01-markdown:/data/law-pack-2026-07-01-markdown:ro \ + -v /data/law-kb-data:/data/law-kb-data \ + law-kb:latest + +# 4. 访问 +open http://192.168.110.114:8090 +``` + +## 目录结构 + +``` +law-kb/ +├── app/ # FastAPI 应用 +│ ├── main.py # 入口 +│ ├── config.py # 配置 +│ ├── models.py # Pydantic 模型 +│ ├── routers/ # API 路由 +│ ├── services/ # 服务层 +│ └── static/ # 前端静态文件 +├── scripts/ # 索引构建脚本 +├── tests/ # 测试 +├── Dockerfile +├── requirements.txt +└── run.md # 运行手册 +``` + +详见 `run.md` 和 `pmdocs/`。 diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..03cb697 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""QYLAW 法律法规知识库应用包""" diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..7be1dfa --- /dev/null +++ b/app/config.py @@ -0,0 +1,74 @@ +"""配置模块 — 路径、端口、外部服务地址""" +import os +from pathlib import Path + +# ===== 路径配置 ===== +# 法规原文目录(容器内挂载点,只读) +LAW_PACK_DIR = Path(os.getenv("LAW_PACK_DIR", "/data/law-pack-2026-07-01-markdown")) + +# 索引数据目录(容器内挂载点,读写) +LAW_KB_DATA_DIR = Path(os.getenv("LAW_KB_DATA_DIR", "/data/law-kb-data")) + +# FAISS 索引文件路径 +FAISS_INDEX_PATH = LAW_KB_DATA_DIR / "faiss" / "index.faiss" + +# SQLite metadata 数据库路径 +SQLITE_PATH = LAW_KB_DATA_DIR / "metadata.db" + +# 构建日志目录 +LOGS_DIR = LAW_KB_DATA_DIR / "logs" + +# 地方性法规区域映射文件 +REGION_MAPPING_PATH = LAW_PACK_DIR / "地方性法规区域映射.json" + +# 法规类别与目录名映射 +CATEGORY_DIRS = { + "法律": "法律", + "行政法规": "行政法规", + "监察法规": "监察法规", + "司法解释": "司法解释", + "地方性法规": "地方性法规", +} + +# ===== 外部服务地址 ===== +# 容器内通过 localhost 访问(--network host 模式) +EMBEDDING_URL = os.getenv("EMBEDDING_URL", "http://localhost:8003/v1") +LLM_URL = os.getenv("LLM_URL", "http://localhost:7000/v1") + +# embedding 模型名(与 114 上 vLLM served-model-name 一致) +EMBEDDING_MODEL = os.getenv("EMBEDDING_MODEL", "qwen3-embedding-0.6b") + +# LLM 模型名 +LLM_MODEL = os.getenv("LLM_MODEL", "qwen3.5-35b") + +# ===== 索引参数 ===== +# embedding 批量大小 +EMBEDDING_BATCH_SIZE = int(os.getenv("EMBEDDING_BATCH_SIZE", "32")) + +# FAISS HNSW 参数 +HNSW_M = 32 +HNSW_EF_CONSTRUCTION = 200 +HNSW_EF_SEARCH = 64 + +# 检索默认 Top-K +DEFAULT_TOP_K = 20 + +# RAG 检索 Top-K +RAG_TOP_K = 10 + +# RAG 相似度阈值(低于此值不调用 LLM) +RAG_SIMILARITY_THRESHOLD = 0.3 + +# ===== API 参数 ===== +# 查询最大长度 +MAX_QUERY_LENGTH = 1000 + +# 分页默认值 +DEFAULT_PAGE_SIZE = 20 +MAX_PAGE_SIZE = 100 + +# 浏览页分页 +BROWSE_PAGE_SIZE = 50 + +# ===== 服务端口 ===== +APP_PORT = int(os.getenv("APP_PORT", "8090")) diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..253b855 --- /dev/null +++ b/app/main.py @@ -0,0 +1,101 @@ +"""FastAPI 应用入口 — 挂载静态页 + API 路由""" +import time +import uuid +from contextlib import asynccontextmanager +from fastapi import FastAPI, Request, HTTPException +from fastapi.staticfiles import StaticFiles +from fastapi.responses import JSONResponse, FileResponse, HTMLResponse +from fastapi.middleware.cors import CORSMiddleware +from pathlib import Path + +from app.config import APP_PORT, LAW_KB_DATA_DIR +from app.routers import search, rag, law, stats, prompts, history +from app.services.retriever import Retriever + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """应用生命周期:启动时加载索引,关闭时释放""" + # 启动:加载 FAISS 索引和 SQLite + retriever = Retriever.get_instance() + retriever.load() + yield + # 关闭:无需特殊处理 + + +app = FastAPI( + title="QYLAW 法律法规知识库", + version="1.0.0", + lifespan=lifespan, +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.middleware("http") +async def add_trace_id(request: Request, call_next): + """为每个请求添加 trace_id""" + trace_id = request.headers.get("X-Trace-Id") or str(uuid.uuid4())[:8] + request.state.trace_id = trace_id + start = time.time() + response = await call_next(request) + elapsed = (time.time() - start) * 1000 + response.headers["X-Trace-Id"] = trace_id + response.headers["X-Response-Time-ms"] = f"{elapsed:.0f}" + return response + + +# 注册 API 路由 +app.include_router(search.router, prefix="/api", tags=["检索"]) +app.include_router(rag.router, prefix="/api", tags=["RAG 问答"]) +app.include_router(law.router, prefix="/api", tags=["法规浏览"]) +app.include_router(stats.router, prefix="/api", tags=["统计"]) +app.include_router(prompts.router, prefix="/api", tags=["提示词配置"]) +app.include_router(history.router, prefix="/api", tags=["历史记录"]) + + +# 静态文件 +STATIC_DIR = Path(__file__).parent / "static" +app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") + + +def _serve_index() -> HTMLResponse: + """返回 index.html(带 no-cache 头,防止浏览器缓存旧版)""" + html = (STATIC_DIR / "index.html").read_text(encoding="utf-8") + return HTMLResponse(content=html, headers={ + "Cache-Control": "no-cache, no-store, must-revalidate", + "Pragma": "no-cache", + "Expires": "0", + }) + + +@app.get("/") +async def index(): + """主页""" + return _serve_index() + + +@app.get("/health") +async def health(): + """健康检查""" + return {"status": "ok"} + + +# 前端路由 catch-all:必须放在所有 API 路由和 /health 之后 +@app.get("/{path:path}") +async def spa_fallback(path: str): + """SPA 路由兜底:/search, /rag, /browse, /history, /settings 返回 index.html""" + # 排除 API 和静态文件路径(已被前面的路由和 mount 处理) + if path.startswith("api/") or path.startswith("static/"): + raise HTTPException(status_code=404) + return _serve_index() + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=APP_PORT) diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..ab93259 --- /dev/null +++ b/app/models.py @@ -0,0 +1,107 @@ +"""Pydantic 请求/响应模型""" +from typing import Optional, List +from pydantic import BaseModel, Field + + +# ===== 统一响应壳 ===== +class ApiResponse(BaseModel): + """统一 API 响应壳""" + code: int = 0 + message: str = "ok" + data: Optional[dict] = None + trace_id: Optional[str] = None + + +# ===== 检索 ===== +class SearchRequest(BaseModel): + """检索请求""" + query: str = Field(..., min_length=1, max_length=1000, description="查询文本") + category: Optional[str] = Field(None, description="法规类别过滤") + province: Optional[str] = Field(None, description="省份过滤(仅地方性法规)") + city: Optional[str] = Field(None, description="市级过滤(仅地方性法规)") + top_k: int = Field(20, ge=1, le=100, description="返回数量") + page: int = Field(1, ge=1, description="页码") + page_size: int = Field(20, ge=1, le=100, description="每页数量") + + +class SearchResultItem(BaseModel): + """单条检索结果""" + law_id: int + law_name: str + category: str + chapter: Optional[str] = None + clause_no: Optional[str] = None + content: str + score: float + file_path: str + province: Optional[str] = None + city: Optional[str] = None + publish_date: Optional[str] = None + + +class SearchResponseData(BaseModel): + """检索响应数据""" + results: List[SearchResultItem] + total: int + page: int + page_size: int + + +# ===== RAG 问答 ===== +class RagRequest(BaseModel): + """RAG 问答请求""" + question: str = Field(..., min_length=1, max_length=1000, description="用户问题") + template: str = Field("simple", description="prompt 模板: simple/professional/compare") + category: Optional[str] = Field(None, description="法规类别过滤") + province: Optional[str] = Field(None, description="省份过滤") + system_prompt: Optional[str] = Field(None, description="自定义系统提示词,覆盖默认") + template_text: Optional[str] = Field(None, description="自定义模板正文,覆盖预设模板。需含 {context} 和 {question} 占位符") + + +class CitationItem(BaseModel): + """RAG 引用条文""" + law_id: int + law_name: str + category: str + chapter: Optional[str] = None + clause_no: Optional[str] = None + content: str + score: float + file_path: str + province: Optional[str] = None + + +# ===== 法规浏览 ===== +class LawListItem(BaseModel): + """法规列表项""" + law_id: int + name: str + category: str + publish_date: Optional[str] = None + province: Optional[str] = None + city: Optional[str] = None + clause_count: int + file_path: str + + +class LawDetail(BaseModel): + """法规详情""" + law_id: int + name: str + category: str + publish_date: Optional[str] = None + province: Optional[str] = None + city: Optional[str] = None + content: str + file_path: str + clauses: List[dict] = [] + + +# ===== 统计 ===== +class StatsResponse(BaseModel): + """索引统计""" + total_laws: int + total_clauses: int + category_stats: dict + index_built_at: Optional[str] = None + faiss_dim: Optional[int] = None diff --git a/app/routers/__init__.py b/app/routers/__init__.py new file mode 100644 index 0000000..80ca775 --- /dev/null +++ b/app/routers/__init__.py @@ -0,0 +1 @@ +"""API 路由模块""" diff --git a/app/routers/history.py b/app/routers/history.py new file mode 100644 index 0000000..54b0620 --- /dev/null +++ b/app/routers/history.py @@ -0,0 +1,76 @@ +"""历史记录 API — /api/history""" +from fastapi import APIRouter, Request, HTTPException, Query + +from app.services import history as hist_service + +router = APIRouter() + + +@router.get("/history/search") +async def list_search_history( + request: Request, + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), +): + """获取检索历史列表(分页,最新在前)""" + trace_id = getattr(request.state, "trace_id", "") + data = hist_service.get_search_history(page, page_size) + return {"code": 0, "message": "ok", "data": data, "trace_id": trace_id} + + +@router.get("/history/chat") +async def list_chat_history( + request: Request, + page: int = Query(1, ge=1), + page_size: int = Query(20, ge=1, le=100), +): + """获取对话历史列表(分页,最新在前)""" + trace_id = getattr(request.state, "trace_id", "") + data = hist_service.get_chat_history(page, page_size) + return {"code": 0, "message": "ok", "data": data, "trace_id": trace_id} + + +@router.get("/history/chat/{chat_id}") +async def get_chat_detail(chat_id: int, request: Request): + """获取单条对话历史详情""" + trace_id = getattr(request.state, "trace_id", "") + detail = hist_service.get_chat_detail(chat_id) + if not detail: + raise HTTPException(status_code=404, detail="对话记录不存在") + return {"code": 0, "message": "ok", "data": detail, "trace_id": trace_id} + + +@router.delete("/history/search/{history_id}") +async def delete_search_history(history_id: int, request: Request): + """删除单条检索历史""" + trace_id = getattr(request.state, "trace_id", "") + if hist_service.delete_search_history(history_id): + return {"code": 0, "message": "ok", "trace_id": trace_id} + raise HTTPException(status_code=500, detail="删除失败") + + +@router.delete("/history/chat/{history_id}") +async def delete_chat_history(history_id: int, request: Request): + """删除单条对话历史""" + trace_id = getattr(request.state, "trace_id", "") + if hist_service.delete_chat_history(history_id): + return {"code": 0, "message": "ok", "trace_id": trace_id} + raise HTTPException(status_code=500, detail="删除失败") + + +@router.delete("/history/search") +async def clear_search_history(request: Request): + """清空全部检索历史""" + trace_id = getattr(request.state, "trace_id", "") + if hist_service.clear_search_history(): + return {"code": 0, "message": "ok", "trace_id": trace_id} + raise HTTPException(status_code=500, detail="清空失败") + + +@router.delete("/history/chat") +async def clear_chat_history(request: Request): + """清空全部对话历史""" + trace_id = getattr(request.state, "trace_id", "") + if hist_service.clear_chat_history(): + return {"code": 0, "message": "ok", "trace_id": trace_id} + raise HTTPException(status_code=500, detail="清空失败") diff --git a/app/routers/law.py b/app/routers/law.py new file mode 100644 index 0000000..8c87a7d --- /dev/null +++ b/app/routers/law.py @@ -0,0 +1,51 @@ +"""法规浏览 API — /api/laws, /api/law/{law_id}""" +from fastapi import APIRouter, Request, HTTPException, Query +from typing import Optional + +from app.services.law_reader import LawReader +from app.config import BROWSE_PAGE_SIZE + +router = APIRouter() + + +@router.get("/laws") +async def list_laws( + request: Request, + category: Optional[str] = Query(None, description="法规类别过滤"), + province: Optional[str] = Query(None, description="省份过滤"), + keyword: Optional[str] = Query(None, description="法规名关键词"), + page: int = Query(1, ge=1, description="页码"), + page_size: int = Query(BROWSE_PAGE_SIZE, ge=1, le=200, description="每页数量"), +): + """法规列表(分页 + 过滤)""" + trace_id = getattr(request.state, "trace_id", "") + reader = LawReader.get_instance() + result = reader.list_laws( + category=category, + province=province, + keyword=keyword, + page=page, + page_size=page_size, + ) + return { + "code": 0, + "message": "ok", + "data": result, + "trace_id": trace_id, + } + + +@router.get("/law/{law_id}") +async def get_law(law_id: int, request: Request): + """法规详情(含原文 + 条文列表)""" + trace_id = getattr(request.state, "trace_id", "") + reader = LawReader.get_instance() + detail = reader.get_law(law_id) + if detail is None: + raise HTTPException(status_code=404, detail="法规不存在") + return { + "code": 0, + "message": "ok", + "data": detail, + "trace_id": trace_id, + } diff --git a/app/routers/prompts.py b/app/routers/prompts.py new file mode 100644 index 0000000..5bd6e94 --- /dev/null +++ b/app/routers/prompts.py @@ -0,0 +1,88 @@ +"""提示词配置 API — /api/prompts""" +from fastapi import APIRouter, Request +from pydantic import BaseModel +from typing import Optional + +from app.services.prompt_config import ( + get_all_configs, + set_config, + get_llm_params, + DEFAULTS, + TEMPLATE_KEY_MAP, +) + +router = APIRouter() + + +class PromptUpdateRequest(BaseModel): + """提示词更新请求""" + system_prompt: Optional[str] = None + template_simple: Optional[str] = None + template_professional: Optional[str] = None + template_compare: Optional[str] = None + # LLM 参数 + llm_thinking_enabled: Optional[str] = None # "true" / "false" + llm_max_tokens: Optional[str] = None # 字符串,后端转 int + llm_temperature: Optional[str] = None # 字符串,后端转 float + llm_thinking_budget: Optional[str] = None # 字符串,后端转 int + + +@router.get("/prompts") +async def get_prompts(request: Request): + """获取所有提示词配置(含默认值)""" + trace_id = getattr(request.state, "trace_id", "") + configs = get_all_configs() + return { + "code": 0, + "message": "ok", + "data": { + "configs": configs, + "defaults": DEFAULTS, + "template_keys": list(TEMPLATE_KEY_MAP.keys()), + "llm_params": get_llm_params(), + }, + "trace_id": trace_id, + } + + +@router.put("/prompts") +async def update_prompts(req: PromptUpdateRequest, request: Request): + """更新提示词配置(空字符串表示恢复默认)""" + trace_id = getattr(request.state, "trace_id", "") + updated = [] + # 系统提示词 + if req.system_prompt is not None: + if set_config("system_prompt", req.system_prompt): + updated.append("system_prompt") + # 模板正文 + for tpl_key in ["template_simple", "template_professional", "template_compare"]: + val = getattr(req, tpl_key, None) + if val is not None: + if set_config(tpl_key, val): + updated.append(tpl_key) + # LLM 参数 + for llm_key in ["llm_thinking_enabled", "llm_max_tokens", "llm_temperature", "llm_thinking_budget"]: + val = getattr(req, llm_key, None) + if val is not None: + if set_config(llm_key, val): + updated.append(llm_key) + return { + "code": 0, + "message": "ok", + "data": {"updated": updated}, + "trace_id": trace_id, + } + + +@router.post("/prompts/reset") +async def reset_prompts(request: Request): + """重置所有提示词配置为默认值""" + trace_id = getattr(request.state, "trace_id", "") + for key in DEFAULTS.keys(): + set_config(key, "") + return { + "code": 0, + "message": "ok", + "data": {"reset": list(DEFAULTS.keys())}, + "trace_id": trace_id, + } diff --git a/app/routers/rag.py b/app/routers/rag.py new file mode 100644 index 0000000..d921ba3 --- /dev/null +++ b/app/routers/rag.py @@ -0,0 +1,159 @@ +"""RAG 问答 API — /api/rag(SSE 流式)""" +import json +from fastapi import APIRouter, Request, HTTPException +from fastapi.responses import StreamingResponse +from typing import Optional + +from app.services.retriever import Retriever +from app.services.embedding import embed_text +from app.services.llm import stream_chat +from app.services.prompt_config import get_template_text, get_system_prompt +from app.services.history import add_chat_history, update_chat_history +from app.config import RAG_TOP_K, RAG_SIMILARITY_THRESHOLD, MAX_QUERY_LENGTH +from app.models import RagRequest + +router = APIRouter() + + +def _build_context(results: list) -> str: + """将检索结果拼装为 context 文本""" + lines = [] + for i, r in enumerate(results, 1): + meta = f"《{r['law_name']}》" + if r.get("clause_no"): + meta += f" {r['clause_no']}" + if r.get("chapter"): + meta += f"({r['chapter']})" + if r.get("province"): + meta += f" [{r['province']}]" + lines.append(f"{i}. {meta}\n{r['content']}") + return "\n\n".join(lines) + + +@router.post("/rag") +async def rag(req: RagRequest, request: Request): + """RAG 问答(SSE 流式) + + 流程:向量化问题 → 检索 Top-10 → 检查相似度阈值 → 拼 prompt → 流式生成 + """ + trace_id = getattr(request.state, "trace_id", "") + + if len(req.question) > MAX_QUERY_LENGTH: + raise HTTPException(status_code=400, detail="问题过长") + + # 向量化问题 + try: + query_vec = await embed_text(req.question) + except Exception as e: + raise HTTPException( + status_code=503, + detail=f"embedding 服务不可用: {str(e)}", + ) + + # 检索 + retriever = Retriever.get_instance() + if not retriever.is_ready(): + raise HTTPException(status_code=503, detail="索引未加载") + + results = retriever.search( + query_vec=query_vec, + top_k=RAG_TOP_K, + category=req.category, + province=req.province, + ) + + # 相似度阈值检查 + if not results or results[0]["score"] < RAG_SIMILARITY_THRESHOLD: + async def no_result_stream(): + yield f"data: {json.dumps({'type': 'no_result', 'message': '未找到相关法规,无法回答'}, ensure_ascii=False)}\n\n" + return StreamingResponse(no_result_stream(), media_type="text/event-stream") + + # 拼 prompt(优先级:请求自定义 > 数据库配置 > 默认) + if req.template_text and "{context}" in req.template_text and "{question}" in req.template_text: + template = req.template_text + else: + template = get_template_text(req.template) # 从数据库读取 + context = _build_context(results) + prompt = template.format(context=context, question=req.question) + + # 系统提示词(优先级:请求自定义 > 数据库配置 > 默认) + system_prompt = req.system_prompt if req.system_prompt else get_system_prompt() + + # LLM 参数(从数据库读取) + from app.services.prompt_config import get_llm_params + llm_params = get_llm_params() + + # 引用条文(先发送) + citations = [ + { + "law_id": r["law_id"], + "law_name": r["law_name"], + "category": r["category"], + "chapter": r.get("chapter"), + "clause_no": r.get("clause_no"), + "content": r["content"], + "score": r["score"], + "file_path": r["file_path"], + "province": r.get("province"), + } + for r in results + ] + + async def event_stream(): + # 先创建对话历史记录(status=streaming) + chat_id = add_chat_history( + question=req.question, + template=req.template, + category=req.category, + province=req.province, + thinking_enabled=llm_params["thinking_enabled"], + status="streaming", + ) + # 发送 chat_id 给前端(用于关联历史记录) + yield f"data: {json.dumps({'type': 'chat_id', 'chat_id': chat_id}, ensure_ascii=False)}\n\n" + # 发送引用条文 + yield f"data: {json.dumps({'type': 'citations', 'clauses': citations}, ensure_ascii=False)}\n\n" + # 发送 LLM 参数(前端用于显示 thinking 状态等) + yield f"data: {json.dumps({'type': 'params', 'thinking_enabled': llm_params['thinking_enabled']}, ensure_ascii=False)}\n\n" + # 流式生成答案(区分 thinking 和 answer) + thinking_acc = "" + answer_acc = "" + try: + async for chunk_type, chunk_content in stream_chat( + prompt, + system=system_prompt, + temperature=llm_params["temperature"], + max_tokens=llm_params["max_tokens"], + thinking_enabled=llm_params["thinking_enabled"], + thinking_budget=llm_params["thinking_budget"], + ): + if chunk_type == "thinking": + thinking_acc += chunk_content + elif chunk_type == "answer": + answer_acc += chunk_content + yield f"data: {json.dumps({'type': chunk_type, 'content': chunk_content}, ensure_ascii=False)}\n\n" + yield f"data: {json.dumps({'type': 'done'})}\n\n" + # 流式完成,更新历史记录 + update_chat_history( + chat_id=chat_id, + thinking=thinking_acc, + answer=answer_acc, + citations=json.dumps(citations, ensure_ascii=False), + status="done", + ) + except Exception as e: + yield f"data: {json.dumps({'type': 'error', 'message': str(e)}, ensure_ascii=False)}\n\n" + # 记录错误状态 + update_chat_history( + chat_id=chat_id, + thinking=thinking_acc, + answer=answer_acc, + citations=json.dumps(citations, ensure_ascii=False), + status="error", + ) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) diff --git a/app/routers/search.py b/app/routers/search.py new file mode 100644 index 0000000..88a7bce --- /dev/null +++ b/app/routers/search.py @@ -0,0 +1,89 @@ +"""检索 API — /api/search""" +from fastapi import APIRouter, Request, HTTPException, Query +from typing import Optional + +from app.services.retriever import Retriever +from app.services.embedding import embed_text +from app.services.history import add_search_history +from app.config import MAX_QUERY_LENGTH + +router = APIRouter() + + +@router.get("/search") +async def search( + request: Request, + query: str = Query(..., min_length=1, max_length=MAX_QUERY_LENGTH, description="查询文本"), + mode: str = Query("semantic", description="检索模式:semantic(语义) / keyword(关键词精确)"), + category: Optional[str] = Query(None, description="法规类别过滤"), + province: Optional[str] = Query(None, description="省份过滤"), + city: Optional[str] = Query(None, description="市级过滤"), + top_k: int = Query(20, ge=1, le=100, description="返回数量"), + page: int = Query(1, ge=1, description="页码"), + page_size: int = Query(20, ge=1, le=100, description="每页数量"), +): + """检索法规条文(语义检索或关键词精确搜索) + + - mode=semantic: 向量化查询 → FAISS 检索 → metadata 过滤(默认) + - mode=keyword: 按法规名+条号精确匹配,支持"郑州市劳动用工条例 第三十二条" + """ + trace_id = getattr(request.state, "trace_id", "") + + retriever = Retriever.get_instance() + if not retriever.is_ready(): + raise HTTPException(status_code=503, detail="索引未加载,请等待或检查索引文件") + + if mode == "keyword": + # 关键词精确搜索 + results = retriever.keyword_search( + query=query, + top_k=top_k * 3 if (category or province or city) else top_k, + category=category, + province=province, + city=city, + ) + else: + # 语义检索 + try: + query_vec = await embed_text(query) + except Exception as e: + raise HTTPException( + status_code=503, + detail=f"embedding 服务不可用: {str(e)}", + ) + fetch_k = top_k * 3 if (category or province or city) else top_k + results = retriever.search( + query_vec=query_vec, + top_k=fetch_k, + category=category, + province=province, + city=city, + ) + + # 分页 + total = len(results) + start = (page - 1) * page_size + end = start + page_size + page_results = results[start:end] + + # 记录检索历史 + add_search_history( + query=query, + category=category, + province=province, + city=city, + result_count=total, + top_k=top_k, + ) + + return { + "code": 0, + "message": "ok", + "data": { + "results": page_results, + "total": total, + "page": page, + "page_size": page_size, + }, + "trace_id": trace_id, + } diff --git a/app/routers/stats.py b/app/routers/stats.py new file mode 100644 index 0000000..df6baa0 --- /dev/null +++ b/app/routers/stats.py @@ -0,0 +1,20 @@ +"""统计 API — /api/stats""" +from fastapi import APIRouter, Request + +from app.services.retriever import Retriever + +router = APIRouter() + + +@router.get("/stats") +async def stats(request: Request): + """返回索引统计信息""" + trace_id = getattr(request.state, "trace_id", "") + retriever = Retriever.get_instance() + data = retriever.get_stats() + return { + "code": 0, + "message": "ok", + "data": data, + "trace_id": trace_id, + } diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..17fe368 --- /dev/null +++ b/app/services/__init__.py @@ -0,0 +1 @@ +"""服务层模块""" diff --git a/app/services/embedding.py b/app/services/embedding.py new file mode 100644 index 0000000..7f48888 --- /dev/null +++ b/app/services/embedding.py @@ -0,0 +1,79 @@ +"""embedding 服务客户端 — 调用 114:8003 Qwen3-Embedding 服务""" +import asyncio +import logging +from typing import List + +import httpx + +from app.config import EMBEDDING_URL, EMBEDDING_MODEL, EMBEDDING_BATCH_SIZE + +logger = logging.getLogger(__name__) + +_client: httpx.AsyncClient | None = None + + +def _get_client() -> httpx.AsyncClient: + """复用 httpx 客户端""" + global _client + if _client is None or _client.is_closed: + _client = httpx.AsyncClient(timeout=60) + return _client + + +async def embed_batch(texts: List[str], batch_size: int = EMBEDDING_BATCH_SIZE) -> List[List[float]]: + """批量向量化 + + Args: + texts: 文本列表 + batch_size: 每批大小,默认 32 + + Returns: + 向量列表,顺序与输入一致 + + Raises: + RuntimeError: embedding 服务不可用或返回错误 + """ + if not texts: + return [] + + client = _get_client() + all_vecs: List[List[float]] = [] + + for i in range(0, len(texts), batch_size): + batch = texts[i : i + batch_size] + payload = { + "model": EMBEDDING_MODEL, + "input": batch, + } + + # 重试 3 次,指数退避 + last_err = None + for attempt in range(3): + try: + resp = await client.post( + f"{EMBEDDING_URL}/embeddings", + json=payload, + ) + resp.raise_for_status() + data = resp.json() + # vLLM embeddings 响应格式: {data: [{embedding: [...]}], ...} + vecs = [item["embedding"] for item in data["data"]] + all_vecs.extend(vecs) + last_err = None + break + except Exception as e: + last_err = e + wait = 2 ** attempt + logger.warning(f"embedding 批次 {i//batch_size} 第 {attempt+1} 次失败: {e}, {wait}s 后重试") + await asyncio.sleep(wait) + + if last_err is not None: + raise RuntimeError(f"embedding 服务调用失败(重试 3 次): {last_err}") + + return all_vecs + + +async def embed_text(text: str) -> List[float]: + """单条文本向量化""" + vecs = await embed_batch([text]) + return vecs[0] diff --git a/app/services/history.py b/app/services/history.py new file mode 100644 index 0000000..2c77308 --- /dev/null +++ b/app/services/history.py @@ -0,0 +1,300 @@ +"""历史记录服务 — 检索历史和对话历史持久化到 SQLite + +表结构: +- search_history: 检索历史(查询词、过滤条件、结果数) +- chat_history: 对话历史(问题、模板、thinking、答案、引用条文) +""" +import sqlite3 +import time +import json +import logging +from typing import Optional + +from app.config import SQLITE_PATH + +logger = logging.getLogger(__name__) + + +def _get_conn() -> sqlite3.Connection: + """获取 SQLite 连接(自动建表)""" + SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(SQLITE_PATH), check_same_thread=False) + conn.executescript(""" + CREATE TABLE IF NOT EXISTS search_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + query TEXT NOT NULL, + category TEXT, + province TEXT, + city TEXT, + result_count INTEGER DEFAULT 0, + top_k INTEGER DEFAULT 20, + created_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS chat_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + question TEXT NOT NULL, + template TEXT DEFAULT 'simple', + category TEXT, + province TEXT, + thinking TEXT, + answer TEXT, + citations TEXT, + thinking_enabled INTEGER DEFAULT 1, + status TEXT DEFAULT 'done', + created_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_search_history_created + ON search_history(created_at DESC); + CREATE INDEX IF NOT EXISTS idx_chat_history_created + ON chat_history(created_at DESC); + """) + return conn + + +def add_search_history( + query: str, + category: Optional[str] = None, + province: Optional[str] = None, + city: Optional[str] = None, + result_count: int = 0, + top_k: int = 20, +) -> int: + """记录检索历史 + + Returns: + 记录 ID,失败返回 0 + """ + try: + conn = _get_conn() + ts = time.strftime("%Y-%m-%d %H:%M:%S") + cursor = conn.execute( + """INSERT INTO search_history + (query, category, province, city, result_count, top_k, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)""", + (query[:200], category, province, city, result_count, top_k, ts), + ) + conn.commit() + rid = cursor.lastrowid + conn.close() + return rid + except Exception as e: + logger.error(f"记录检索历史失败: {e}") + return 0 + + +def add_chat_history( + question: str, + template: str = "simple", + category: Optional[str] = None, + province: Optional[str] = None, + thinking: str = "", + answer: str = "", + citations: str = "[]", + thinking_enabled: bool = True, + status: str = "done", +) -> int: + """记录对话历史 + + Args: + citations: JSON 字符串,引用条文列表 + + Returns: + 记录 ID,失败返回 0 + """ + try: + conn = _get_conn() + ts = time.strftime("%Y-%m-%d %H:%M:%S") + cursor = conn.execute( + """INSERT INTO chat_history + (question, template, category, province, thinking, answer, + citations, thinking_enabled, status, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + (question[:1000], template, category, province, + thinking, answer, citations, 1 if thinking_enabled else 0, + status, ts), + ) + conn.commit() + rid = cursor.lastrowid + conn.close() + return rid + except Exception as e: + logger.error(f"记录对话历史失败: {e}") + return 0 + + +def update_chat_history( + chat_id: int, + thinking: str = "", + answer: str = "", + citations: str = "[]", + status: str = "done", +) -> bool: + """更新对话历史(流式生成完成后更新完整内容) + + Returns: + True 成功 + """ + if not chat_id: + return False + try: + conn = _get_conn() + conn.execute( + """UPDATE chat_history + SET thinking = ?, answer = ?, citations = ?, status = ? + WHERE id = ?""", + (thinking, answer, citations, status, chat_id), + ) + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"更新对话历史失败: {e}") + return False + + +def get_search_history( + page: int = 1, + page_size: int = 20, +) -> dict: + """获取检索历史列表(分页,最新在前) + + Returns: + {"items": [...], "total": int, "page": int, "page_size": int} + """ + try: + conn = _get_conn() + total = conn.execute("SELECT COUNT(*) FROM search_history").fetchone()[0] + offset = (page - 1) * page_size + rows = conn.execute( + """SELECT id, query, category, province, city, result_count, top_k, created_at + FROM search_history + ORDER BY created_at DESC + LIMIT ? OFFSET ?""", + (page_size, offset), + ).fetchall() + conn.close() + items = [ + { + "id": r[0], "query": r[1], "category": r[2], "province": r[3], + "city": r[4], "result_count": r[5], "top_k": r[6], "created_at": r[7], + } + for r in rows + ] + return {"items": items, "total": total, "page": page, "page_size": page_size} + except Exception as e: + logger.error(f"获取检索历史失败: {e}") + return {"items": [], "total": 0, "page": page, "page_size": page_size} + + +def get_chat_history( + page: int = 1, + page_size: int = 20, +) -> dict: + """获取对话历史列表(分页,最新在前) + + Returns: + {"items": [...], "total": int, "page": int, "page_size": int} + """ + try: + conn = _get_conn() + total = conn.execute("SELECT COUNT(*) FROM chat_history").fetchone()[0] + offset = (page - 1) * page_size + rows = conn.execute( + """SELECT id, question, template, category, province, + thinking, answer, citations, thinking_enabled, status, created_at + FROM chat_history + ORDER BY created_at DESC + LIMIT ? OFFSET ?""", + (page_size, offset), + ).fetchall() + conn.close() + items = [] + for r in rows: + items.append({ + "id": r[0], "question": r[1], "template": r[2], "category": r[3], + "province": r[4], "thinking": r[5], "answer": r[6], + "citations": json.loads(r[7]) if r[7] else [], + "thinking_enabled": bool(r[8]), "status": r[9], "created_at": r[10], + }) + return {"items": items, "total": total, "page": page, "page_size": page_size} + except Exception as e: + logger.error(f"获取对话历史失败: {e}") + return {"items": [], "total": 0, "page": page, "page_size": page_size} + + +def get_chat_detail(chat_id: int) -> Optional[dict]: + """获取单条对话历史详情""" + try: + conn = _get_conn() + row = conn.execute( + """SELECT id, question, template, category, province, + thinking, answer, citations, thinking_enabled, status, created_at + FROM chat_history WHERE id = ?""", + (chat_id,), + ).fetchone() + conn.close() + if not row: + return None + return { + "id": row[0], "question": row[1], "template": row[2], "category": row[3], + "province": row[4], "thinking": row[5], "answer": row[6], + "citations": json.loads(row[7]) if row[7] else [], + "thinking_enabled": bool(row[8]), "status": row[9], "created_at": row[10], + } + except Exception as e: + logger.error(f"获取对话详情失败: {e}") + return None + + +def delete_search_history(history_id: int) -> bool: + """删除单条检索历史""" + try: + conn = _get_conn() + conn.execute("DELETE FROM search_history WHERE id = ?", (history_id,)) + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"删除检索历史失败: {e}") + return False + + +def delete_chat_history(history_id: int) -> bool: + """删除单条对话历史""" + try: + conn = _get_conn() + conn.execute("DELETE FROM chat_history WHERE id = ?", (history_id,)) + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"删除对话历史失败: {e}") + return False + + +def clear_search_history() -> bool: + """清空全部检索历史""" + try: + conn = _get_conn() + conn.execute("DELETE FROM search_history") + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"清空检索历史失败: {e}") + return False + + +def clear_chat_history() -> bool: + """清空全部对话历史""" + try: + conn = _get_conn() + conn.execute("DELETE FROM chat_history") + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"清空对话历史失败: {e}") + return False diff --git a/app/services/law_reader.py b/app/services/law_reader.py new file mode 100644 index 0000000..037f699 --- /dev/null +++ b/app/services/law_reader.py @@ -0,0 +1,147 @@ +"""法规原文读取服务 — 从 law-pack 目录读取 markdown,从 SQLite 取列表""" +import sqlite3 +import logging +from typing import Optional, Dict, Any, List +from pathlib import Path + +from app.config import SQLITE_PATH, LAW_PACK_DIR + +logger = logging.getLogger(__name__) + + +class LawReader: + """法规原文读取单例""" + + _instance: Optional["LawReader"] = None + + @classmethod + def get_instance(cls) -> "LawReader": + if cls._instance is None: + cls._instance = cls() + return cls._instance + + def __init__(self): + self._conn: Optional[sqlite3.Connection] = None + + def _ensure_conn(self): + if self._conn is None and SQLITE_PATH.exists(): + self._conn = sqlite3.connect(str(SQLITE_PATH), check_same_thread=False) + self._conn.row_factory = sqlite3.Row + + def list_laws( + self, + category: Optional[str] = None, + province: Optional[str] = None, + keyword: Optional[str] = None, + page: int = 1, + page_size: int = 50, + ) -> Dict[str, Any]: + """法规列表(分页 + 过滤)""" + self._ensure_conn() + if self._conn is None: + return {"results": [], "total": 0, "page": page, "page_size": page_size} + + where = [] + params: List[Any] = [] + if category: + where.append("category = ?") + params.append(category) + if province: + where.append("province = ?") + params.append(province) + if keyword: + where.append("name LIKE ?") + params.append(f"%{keyword}%") + + where_clause = f"WHERE {' AND '.join(where)}" if where else "" + + # 总数 + total = self._conn.execute( + f"SELECT COUNT(*) FROM laws {where_clause}", params + ).fetchone()[0] + + # 分页 + offset = (page - 1) * page_size + rows = self._conn.execute( + f""" + SELECT id, name, category, publish_date, province, city, clause_count, file_path + FROM laws {where_clause} + ORDER BY category, name + LIMIT ? OFFSET ? + """, + params + [page_size, offset], + ).fetchall() + + results = [ + { + "law_id": row["id"], + "name": row["name"], + "category": row["category"], + "publish_date": row["publish_date"], + "province": row["province"], + "city": row["city"], + "clause_count": row["clause_count"], + "file_path": row["file_path"], + } + for row in rows + ] + + return { + "results": results, + "total": total, + "page": page, + "page_size": page_size, + } + + def get_law(self, law_id: int) -> Optional[Dict[str, Any]]: + """法规详情(含原文 + 条文列表)""" + self._ensure_conn() + if self._conn is None: + return None + + row = self._conn.execute( + "SELECT * FROM laws WHERE id = ?", (law_id,) + ).fetchone() + if row is None: + return None + + # 读取原文(路径校验防穿越) + file_path = Path(row["file_path"]) + try: + # 确保在 law-pack 目录内 + abs_path = file_path.resolve() + pack_root = LAW_PACK_DIR.resolve() + if not str(abs_path).startswith(str(pack_root)): + logger.error(f"路径穿越尝试: {file_path}") + return None + content = abs_path.read_text(encoding="utf-8") + except Exception as e: + logger.error(f"读取原文失败 {file_path}: {e}") + content = f"[读取失败: {e}]" + + # 条文列表 + clauses = self._conn.execute( + "SELECT id, chapter, clause_no, content FROM clauses WHERE law_id = ? ORDER BY id", + (law_id,), + ).fetchall() + clause_list = [ + { + "clause_id": c["id"], + "chapter": c["chapter"], + "clause_no": c["clause_no"], + "content": c["content"], + } + for c in clauses + ] + + return { + "law_id": row["id"], + "name": row["name"], + "category": row["category"], + "publish_date": row["publish_date"], + "province": row["province"], + "city": row["city"], + "content": content, + "file_path": row["file_path"], + "clauses": clause_list, + } diff --git a/app/services/llm.py b/app/services/llm.py new file mode 100644 index 0000000..41c1ac7 --- /dev/null +++ b/app/services/llm.py @@ -0,0 +1,140 @@ +"""LLM 客户端 — 调用 114:7000 Qwen3.5-35B 服务(流式)""" +import json +import logging +from typing import AsyncGenerator + +import httpx + +from app.config import LLM_URL, LLM_MODEL + +logger = logging.getLogger(__name__) + +_client: httpx.AsyncClient | None = None + + +def _get_client() -> httpx.AsyncClient: + global _client + if _client is None or _client.is_closed: + _client = httpx.AsyncClient(timeout=120) + return _client + + +DEFAULT_SYSTEM_PROMPT = "你是法律助手,根据提供的法规条文回答问题,必须引用法规名和条号。" + + +async def stream_chat( + prompt: str, + system: str | None = None, + temperature: float = 0.3, + max_tokens: int = 2048, + thinking_enabled: bool = True, + thinking_budget: int = 2048, +) -> AsyncGenerator[tuple[str, str], None]: + """流式对话,区分 thinking 和 answer + + 模型输出格式:thinking 内容以 "Here's a thinking process:" 开头, + 以 结尾,之后是正式回答。 + + Args: + prompt: 用户 prompt(已含 context) + system: system prompt + temperature: 温度 + max_tokens: 最大生成 token(含 thinking) + thinking_enabled: 是否启用 thinking + thinking_budget: thinking 预算(thinking 最大 token 数) + + Yields: + (type, content) 元组,type 为 "thinking" 或 "answer" + + Raises: + RuntimeError: LLM 服务不可用 + """ + client = _get_client() + sys_content = system if system else DEFAULT_SYSTEM_PROMPT + + # thinking 关闭时,通过 system prompt 补充指示 + if not thinking_enabled: + sys_content += "\n\n注意:不要输出思考过程,直接给出答案。" + + # payload:使用 chat_template_kwargs.enable_thinking 控制 thinking 开关 + # 这是 Qwen3.5 vLLM 的原生参数,比 system prompt 更可靠 + payload = { + "model": LLM_MODEL, + "messages": [ + {"role": "system", "content": sys_content}, + {"role": "user", "content": prompt}, + ], + "temperature": temperature, + "max_tokens": max_tokens, + "stream": True, + "chat_template_kwargs": {"enable_thinking": thinking_enabled}, + } + + # 状态机:跟踪当前是否在 thinking 区域 + in_thinking = thinking_enabled # thinking 开启时,输出通常以 thinking 开头 + # 小缓冲:仅用于检测跨块的 thinking 结束标记 + tail_buffer = "" + # Qwen3.5 的 thinking 结束标记: + THINKING_END_TAG = "" + + last_err = None + for attempt in range(2): # 重试 1 次 + try: + async with client.stream( + "POST", + f"{LLM_URL}/chat/completions", + json=payload, + timeout=120, + ) as resp: + resp.raise_for_status() + async for line in resp.aiter_lines(): + if not line or not line.startswith("data: "): + continue + data_str = line[6:] + if data_str.strip() == "[DONE]": + return + try: + chunk = json.loads(data_str) + delta = chunk["choices"][0].get("delta", {}) + content = delta.get("content", "") + if not content: + continue + + if not thinking_enabled: + # thinking 关闭,全部作为 answer + yield ("answer", content) + else: + # thinking 开启,需要解析 THINKING_END_TAG 标记 + if in_thinking: + if THINKING_END_TAG in content: + # 分割:前面是 thinking,后面是 answer + parts = content.split(THINKING_END_TAG, 1) + thinking_part = parts[0] + answer_part = parts[1] if len(parts) > 1 else "" + if thinking_part: + yield ("thinking", thinking_part) + if answer_part: + yield ("answer", answer_part.lstrip()) + in_thinking = False + tail_buffer = "" + else: + # 检查是否是 THINKING_END_TAG 的前缀(跨块情况) + tail_buffer = (tail_buffer + content)[-12:] + # 检查 tail_buffer 是否是 THINKING_END_TAG 的前缀 + is_prefix = any( + tail_buffer.endswith(THINKING_END_TAG[:k]) + for k in range(1, len(THINKING_END_TAG)) + ) + if is_prefix: + continue + yield ("thinking", content) + else: + yield ("answer", content) + except (json.JSONDecodeError, KeyError, IndexError): + continue + return + except Exception as e: + last_err = e + logger.warning(f"LLM 流式调用第 {attempt+1} 次失败: {e}") + + raise RuntimeError(f"LLM 服务调用失败: {last_err}") diff --git a/app/services/prompt_config.py b/app/services/prompt_config.py new file mode 100644 index 0000000..c44ffff --- /dev/null +++ b/app/services/prompt_config.py @@ -0,0 +1,190 @@ +"""提示词配置服务 — 从 SQLite 读写系统提示词和模板正文 + +配置项(config_key): +- system_prompt: 系统提示词(覆盖默认) +- template_simple: 通俗解释模板正文 +- template_professional: 专业分析模板正文 +- template_compare: 对比条文模板正文 +""" +import sqlite3 +import time +import logging +from typing import Optional, Dict +from pathlib import Path + +from app.config import SQLITE_PATH + +logger = logging.getLogger(__name__) + +# 默认值(与 rag.py 中 PROMPT_TEMPLATES 保持一致) +DEFAULTS = { + "system_prompt": "你是法律助手,根据提供的法规条文回答问题,必须引用法规名和条号。", + "template_simple": ( + "你是一位耐心的法律科普助手。请根据以下法规条文,用通俗易懂的语言回答用户问题。\n" + "要求:\n" + "1. 避免法律术语,用日常语言解释\n" + '2. 必须引用依据的法规名和条号,格式如"根据《XX法》第X条"\n' + '3. 如果条文不足以回答,明确说明"根据现有法规无法完全回答"\n' + "4. 不要编造法规\n\n" + "【相关法规条文】\n{context}\n\n【用户问题】\n{question}" + ), + "template_professional": ( + "你是一位专业的法律分析助手。请根据以下法规条文,对用户问题进行专业分析。\n" + "要求:\n" + "1. 使用规范法律术语\n" + '2. 必须引用依据的法规名和条号,格式如"依据《XX法》第X条"\n' + "3. 分析条文的适用条件、法律后果\n" + "4. 如有多条相关,对比分析\n" + "5. 指出条文的适用边界和可能的争议点\n" + "6. 不要编造法规\n\n" + "【相关法规条文】\n{context}\n\n【用户问题】\n{question}" + ), + "template_compare": ( + "你是一位法规研究助手。请根据以下法规条文,对比分析不同法规对同一问题的规定。\n" + "要求:\n" + "1. 列出每条相关法规的具体规定\n" + "2. 对比规定的异同\n" + "3. 标注法规名、条号、发布日期\n" + "4. 指出适用范围差异(如全国性 vs 地方性)\n" + "5. 不要编造法规\n\n" + "【相关法规条文】\n{context}\n\n【用户问题】\n{question}" + ), + # LLM 参数默认值 + "llm_thinking_enabled": "true", # thinking 开关 + "llm_max_tokens": "4096", # 最大生成 token 数 + "llm_temperature": "0.3", # 温度 + "llm_thinking_budget": "2048", # thinking 预算(thinking 最大 token 数) +} + +# 模板 key 与 rag.py PROMPT_TEMPLATES 的映射 +TEMPLATE_KEY_MAP = { + "simple": "template_simple", + "professional": "template_professional", + "compare": "template_compare", +} + + +def _get_conn() -> sqlite3.Connection: + """获取 SQLite 连接(自动建表)""" + SQLITE_PATH.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(SQLITE_PATH), check_same_thread=False) + # 自动建表(若数据库不存在或表不存在) + conn.executescript(""" + CREATE TABLE IF NOT EXISTS prompt_configs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + config_key TEXT NOT NULL UNIQUE, + config_value TEXT, + updated_at TEXT NOT NULL + ); + """) + return conn + + +def get_all_configs() -> Dict[str, str]: + """读取所有提示词配置,未配置的用默认值填充 + + Returns: + {config_key: config_value} 所有配置项(含默认值) + """ + configs = dict(DEFAULTS) # 先用默认值 + try: + conn = _get_conn() + rows = conn.execute( + "SELECT config_key, config_value FROM prompt_configs" + ).fetchall() + for row in rows: + key, val = row[0], row[1] + if val: # 空值视为未配置,用默认 + configs[key] = val + conn.close() + except Exception as e: + logger.warning(f"读取提示词配置失败,使用默认值: {e}") + return configs + + +def get_config(key: str) -> str: + """读取单个配置,未配置返回默认值""" + try: + conn = _get_conn() + row = conn.execute( + "SELECT config_value FROM prompt_configs WHERE config_key = ?", (key,) + ).fetchone() + conn.close() + if row and row[0]: + return row[0] + except Exception as e: + logger.warning(f"读取配置 {key} 失败: {e}") + return DEFAULTS.get(key, "") + + +def set_config(key: str, value: str) -> bool: + """写入单个配置(空值删除记录,回退到默认) + + Returns: + True 成功 + """ + try: + conn = _get_conn() + ts = time.strftime("%Y-%m-%d %H:%M:%S") + if value: # 有值则 upsert + conn.execute( + """ + INSERT INTO prompt_configs (config_key, config_value, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(config_key) DO UPDATE SET + config_value = excluded.config_value, + updated_at = excluded.updated_at + """, + (key, value, ts), + ) + else: # 空值则删除(回退默认) + conn.execute( + "DELETE FROM prompt_configs WHERE config_key = ?", (key,) + ) + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"写入配置 {key} 失败: {e}") + return False + + +def get_template_text(template: str) -> str: + """获取指定模板的正文 + + Args: + template: 模板名(simple/professional/compare) + + Returns: + 模板正文(含 {context} 和 {question} 占位符) + """ + key = TEMPLATE_KEY_MAP.get(template, "template_simple") + return get_config(key) + + +def get_llm_params() -> dict: + """获取 LLM 参数配置 + + Returns: + {"thinking_enabled": bool, "max_tokens": int, "temperature": float, "thinking_budget": int} + """ + configs = get_all_configs() + try: + return { + "thinking_enabled": configs.get("llm_thinking_enabled", "true").lower() == "true", + "max_tokens": int(configs.get("llm_max_tokens", "4096")), + "temperature": float(configs.get("llm_temperature", "0.3")), + "thinking_budget": int(configs.get("llm_thinking_budget", "2048")), + } + except (ValueError, TypeError): + return { + "thinking_enabled": True, + "max_tokens": 4096, + "temperature": 0.3, + "thinking_budget": 2048, + } + + +def get_system_prompt() -> str: + """获取系统提示词""" + return get_config("system_prompt") diff --git a/app/services/retriever.py b/app/services/retriever.py new file mode 100644 index 0000000..7724073 --- /dev/null +++ b/app/services/retriever.py @@ -0,0 +1,264 @@ +"""FAISS 检索器 — 加载索引 + SQLite metadata,实现语义检索 + 过滤""" +import sqlite3 +import logging +from typing import List, Optional, Dict, Any + +import numpy as np + +from app.config import ( + FAISS_INDEX_PATH, + SQLITE_PATH, + HNSW_EF_SEARCH, +) + +logger = logging.getLogger(__name__) + + +class Retriever: + """检索器单例""" + + _instance: Optional["Retriever"] = None + + @classmethod + def get_instance(cls) -> "Retriever": + if cls._instance is None: + cls._instance = cls() + return cls._instance + + def __init__(self): + self._faiss = None + self._conn: Optional[sqlite3.Connection] = None + self._dim: Optional[int] = None + self._ready = False + + def is_ready(self) -> bool: + return self._ready + + def load(self): + """加载 FAISS 索引和 SQLite""" + if not FAISS_INDEX_PATH.exists(): + logger.warning(f"FAISS 索引不存在: {FAISS_INDEX_PATH}") + return + if not SQLITE_PATH.exists(): + logger.warning(f"SQLite 不存在: {SQLITE_PATH}") + return + + import faiss + + logger.info(f"加载 FAISS 索引: {FAISS_INDEX_PATH}") + self._faiss = faiss.read_index(str(FAISS_INDEX_PATH)) + self._dim = self._faiss.d + + # 设置 HNSW 搜索参数 + if hasattr(self._faiss, "hnsw"): + self._faiss.hnsw.efSearch = HNSW_EF_SEARCH + + logger.info(f"加载 SQLite: {SQLITE_PATH}") + self._conn = sqlite3.connect(str(SQLITE_PATH), check_same_thread=False) + self._conn.row_factory = sqlite3.Row + + self._ready = True + count = self._faiss.ntotal + logger.info(f"索引加载完成: {count} 向量, dim={self._dim}") + + def search( + self, + query_vec: List[float], + top_k: int = 20, + category: Optional[str] = None, + province: Optional[str] = None, + city: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """语义检索 + metadata 过滤 + + Args: + query_vec: 查询向量 + top_k: 返回数量 + category: 法规类别过滤 + province: 省份过滤 + city: 市级过滤 + + Returns: + 结果列表,每项含 law_id/law_name/category/chapter/clause_no/content/score/file_path/province + """ + if not self._ready: + return [] + + # FAISS 检索(取 top_k * 3 用于过滤后仍有足够结果) + vec = np.array([query_vec], dtype=np.float32) + fetch_k = min(top_k * 3 if (category or province or city) else top_k, self._faiss.ntotal) + scores, indices = self._faiss.search(vec, fetch_k) + + results = [] + for score, faiss_idx in zip(scores[0], indices[0]): + if faiss_idx < 0: + continue + # 从 SQLite 取 metadata + row = self._conn.execute( + """ + SELECT c.id, c.law_id, c.chapter, c.clause_no, c.content, c.faiss_idx, + l.name, l.category, l.publish_date, l.file_path, l.province, l.city + FROM clauses c JOIN laws l ON c.law_id = l.id + WHERE c.faiss_idx = ? + """, + (int(faiss_idx),), + ).fetchone() + if row is None: + continue + + # metadata 过滤 + if category and row["category"] != category: + continue + if province and row["province"] != province: + continue + if city and row["city"] != city: + continue + + results.append({ + "clause_id": row["id"], + "law_id": row["law_id"], + "law_name": row["name"], + "category": row["category"], + "chapter": row["chapter"], + "clause_no": row["clause_no"], + "content": row["content"], + "score": float(score), + "file_path": row["file_path"], + "province": row["province"], + "city": row["city"], + "publish_date": row["publish_date"], + }) + + if len(results) >= top_k: + break + + return results + + def keyword_search( + self, + query: str, + top_k: int = 20, + category: Optional[str] = None, + province: Optional[str] = None, + city: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """关键词精确搜索(按法规名 + 条号 + 条文内容匹配) + + 支持"法规名 条号"格式(如"郑州市劳动用工条例 第三十二条"), + 也支持纯关键词搜索条文内容。 + + Returns: + 结果列表,格式与 search() 一致,score 为匹配相关度(非 FAISS 距离) + """ + if not self._ready or not query.strip(): + return [] + + query = query.strip() + # 尝试解析"法规名 条号"格式 + # 中文条号模式:第X条 + import re + clause_match = re.search(r'(第[一二三四五六七八九十百千零\d]+条)', query) + clause_no = clause_match.group(1) if clause_match else None + # 法规名 = 去掉条号后的部分 + law_name_query = re.sub(r'\s*第[一二三四五六七八九十百千零\d]+条\s*', '', query).strip() + + results = [] + try: + if clause_no and law_name_query: + # 精确匹配:法规名 LIKE + 条号 = + rows = self._conn.execute( + """ + SELECT c.id, c.law_id, c.chapter, c.clause_no, c.content, c.faiss_idx, + l.name, l.category, l.publish_date, l.file_path, l.province, l.city + FROM clauses c JOIN laws l ON c.law_id = l.id + WHERE l.name LIKE ? AND c.clause_no = ? + """, + (f"%{law_name_query}%", clause_no), + ).fetchall() + elif clause_no: + # 只按条号搜索 + rows = self._conn.execute( + """ + SELECT c.id, c.law_id, c.chapter, c.clause_no, c.content, c.faiss_idx, + l.name, l.category, l.publish_date, l.file_path, l.province, l.city + FROM clauses c JOIN laws l ON c.law_id = l.id + WHERE c.clause_no = ? + """, + (clause_no,), + ).fetchall() + else: + # 纯关键词:搜索法规名或条文内容 + rows = self._conn.execute( + """ + SELECT c.id, c.law_id, c.chapter, c.clause_no, c.content, c.faiss_idx, + l.name, l.category, l.publish_date, l.file_path, l.province, l.city + FROM clauses c JOIN laws l ON c.law_id = l.id + WHERE l.name LIKE ? OR c.content LIKE ? + """, + (f"%{query}%", f"%{query}%"), + ).fetchall() + + for row in rows: + # metadata 过滤 + if category and row["category"] != category: + continue + if province and row["province"] != province: + continue + if city and row["city"] != city: + continue + + results.append({ + "clause_id": row["id"], + "law_id": row["law_id"], + "law_name": row["name"], + "category": row["category"], + "chapter": row["chapter"], + "clause_no": row["clause_no"], + "content": row["content"], + "score": 0.0, # 关键词搜索无相似度分数 + "file_path": row["file_path"], + "province": row["province"], + "city": row["city"], + "publish_date": row["publish_date"], + }) + + if len(results) >= top_k: + break + except Exception as e: + logger.error(f"关键词搜索失败: {e}") + + return results + + def get_stats(self) -> Dict[str, Any]: + """返回索引统计""" + if not self._ready: + return { + "total_laws": 0, + "total_clauses": 0, + "category_stats": {}, + "index_built_at": None, + "faiss_dim": None, + "ready": False, + } + + total_laws = self._conn.execute("SELECT COUNT(*) FROM laws").fetchone()[0] + total_clauses = self._conn.execute("SELECT COUNT(*) FROM clauses").fetchone()[0] + cat_rows = self._conn.execute( + "SELECT category, COUNT(*) as cnt FROM laws GROUP BY category" + ).fetchall() + category_stats = {row["category"]: row["cnt"] for row in cat_rows} + + # 索引构建时间(取最新一条 law 的 indexed_at) + built_at_row = self._conn.execute( + "SELECT indexed_at FROM laws ORDER BY indexed_at DESC LIMIT 1" + ).fetchone() + index_built_at = built_at_row["indexed_at"] if built_at_row else None + + return { + "total_laws": total_laws, + "total_clauses": total_clauses, + "category_stats": category_stats, + "index_built_at": index_built_at, + "faiss_dim": self._dim, + "ready": True, + } diff --git a/app/static/css/app.css b/app/static/css/app.css new file mode 100644 index 0000000..cbf3633 --- /dev/null +++ b/app/static/css/app.css @@ -0,0 +1,2 @@ +/* app.css — Tailwind 之外的补充样式 */ +/* 目前所有样式已在 index.html 的 + + + + +
+
+ +
法
+
+

千源法规知识库

+

17291 篇现行有效法律法规 · 语义检索 + RAG 问答

+
+
+
+ +
+
+ + +
+ + + + + +
+ +
+ +
+ + + + + + + + + diff --git a/app/static/js/app.js b/app/static/js/app.js new file mode 100644 index 0000000..07785eb --- /dev/null +++ b/app/static/js/app.js @@ -0,0 +1,264 @@ +/** + * app.js — 公共逻辑:路由、公共组件、API 封装 + */ + +// ===== 全局常量(供 rag.js / settings.js 共享) ===== +const PROMPT_DEFAULTS = { + system_prompt: '你是法律助手,根据提供的法规条文回答问题,必须引用法规名和条号。', + template_simple: '你是一位耐心的法律科普助手。请根据以下法规条文,用通俗易懂的语言回答用户问题。\n要求:\n1. 避免法律术语,用日常语言解释\n2. 必须引用依据的法规名和条号,格式如"根据《XX法》第X条"\n3. 如果条文不足以回答,明确说明"根据现有法规无法完全回答"\n4. 不要编造法规\n\n【相关法规条文】\n{context}\n\n【用户问题】\n{question}', + template_professional: '你是一位专业的法律分析助手。请根据以下法规条文,对用户问题进行专业分析。\n要求:\n1. 使用规范法律术语\n2. 必须引用依据的法规名和条号,格式如"依据《XX法》第X条"\n3. 分析条文的适用条件、法律后果\n4. 如有多条相关,对比分析\n5. 指出条文的适用边界和可能的争议点\n6. 不要编造法规\n\n【相关法规条文】\n{context}\n\n【用户问题】\n{question}', + template_compare: '你是一位法规研究助手。请根据以下法规条文,对比分析不同法规对同一问题的规定。\n要求:\n1. 列出每条相关法规的具体规定\n2. 对比规定的异同\n3. 标注法规名、条号、发布日期\n4. 指出适用范围差异(如全国性 vs 地方性)\n5. 不要编造法规\n\n【相关法规条文】\n{context}\n\n【用户问题】\n{question}', +}; + +const TEMPLATES = [ + { value: 'simple', label: '通俗解释' }, + { value: 'professional', label: '专业分析' }, + { value: 'compare', label: '对比条文' }, +]; + +// ===== 路由 ===== +const PAGES = ['rag', 'search', 'browse', 'history', 'settings']; +let currentPage = 'rag'; + +/** 渲染指定页面(不修改 URL) */ +function renderPage(page) { + if (!PAGES.includes(page)) page = 'search'; + currentPage = page; + // 更新导航激活态 + document.querySelectorAll('.nav-item').forEach(btn => { + if (btn.dataset.page === page) { + btn.classList.remove('nav-inactive'); + btn.classList.add('nav-active'); + } else { + btn.classList.remove('nav-active'); + btn.classList.add('nav-inactive'); + } + }); + // 渲染页面 + const content = document.getElementById('content'); + if (page === 'search') content.innerHTML = SearchPage.render(); + if (page === 'rag') content.innerHTML = RagPage.render(); + if (page === 'browse') content.innerHTML = BrowsePage.render(); + if (page === 'history') content.innerHTML = HistoryPage.render(); + if (page === 'settings') content.innerHTML = SettingsPage.render(); + // 页面初始化 + if (page === 'search') SearchPage.init(); + if (page === 'rag') RagPage.init(); + if (page === 'browse') BrowsePage.init(); + if (page === 'history') HistoryPage.init(); + if (page === 'settings') SettingsPage.init(); + // 关闭移动端菜单 + document.getElementById('sidebar').classList.remove('open'); +} + +/** 导航到指定页面(修改 URL + 渲染) */ +function navigate(page) { + if (!PAGES.includes(page)) return; + if (page === currentPage) return; + // 更新 URL(不触发 popstate) + history.pushState({ page }, '', `/${page}`); + renderPage(page); +} + +/** 从 URL 路径解析当前页面 */ +function pageFromPath() { + const path = window.location.pathname.replace(/^\//, '').replace(/\/$/, ''); + return PAGES.includes(path) ? path : 'rag'; +} + +// 监听浏览器前进/后退 +window.addEventListener('popstate', (e) => { + const page = (e.state && e.state.page) || pageFromPath(); + renderPage(page); +}); + +// ===== 公共组件 ===== +const UI = { + /** 加载中 spinner */ + spinner(size = 16) { + return ``; + }, + + /** 空状态 */ + emptyState(title, desc, icon = '📋') { + return ` +
+
${icon}
+
${title}
+
${desc}
+
`; + }, + + /** 错误状态 */ + errorState(msg, onRetry) { + const retryBtn = onRetry ? `` : ''; + return ` +
+
⚠️
+
${msg}
+ ${retryBtn} +
`; + }, + + /** 类别标签 */ + categoryBadge(category) { + const colors = { + '法律': 'bg-blue-50 text-blue-700', + '行政法规': 'bg-purple-50 text-purple-700', + '监察法规': 'bg-amber-50 text-amber-700', + '司法解释': 'bg-teal-50 text-teal-700', + '地方性法规': 'bg-emerald-50 text-emerald-700', + }; + const cls = colors[category] || 'bg-gray-100 text-gray-600'; + return `${category}`; + }, + + /** 分页 */ + pagination(page, pageSize, total, onChange) { + const totalPages = Math.ceil(total / pageSize); + if (totalPages <= 1) return ''; + const prev = page > 1 ? `` : ''; + const next = page < totalPages ? `` : ''; + return ` +
+
共 ${total} 条,第 ${page}/${totalPages} 页
+
${prev}${next}
+
`; + }, + + /** 条文卡片 */ + clauseCard(item, highlight = false) { + const badge = UI.categoryBadge(item.category); + const province = item.province ? `[${item.province}${item.city ? ' · ' + item.city : ''}]` : ''; + const chapter = item.chapter ? `${item.chapter}` : ''; + const clauseNo = item.clause_no ? `${item.clause_no}` : ''; + const score = item.score !== undefined ? `相似度 ${item.score.toFixed(4)}` : ''; + const cls = highlight ? 'clause-highlight' : 'border border-gray-200 bg-white'; + return ` +
+
+ ${item.law_name} + ${badge} + ${clauseNo} + ${chapter} + ${province} + ${score} +
+
${item.content.substring(0, 200)}${item.content.length > 200 ? '...' : ''}
+ ${item.law_id ? `` : ''} +
`; + }, + + /** 跳转查看原文 */ + viewLaw(lawId) { + navigate('browse'); + setTimeout(() => BrowsePage.viewLaw(lawId), 100); + }, +}; + +// ===== API 封装 ===== +const API = { + async get(url) { + const resp = await fetch(url); + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + return resp.json(); + }, + + async post(url, body) { + const resp = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + return resp; + }, + + async put(url, body) { + const resp = await fetch(url, { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + return resp.json(); + }, + + async delete(url) { + const resp = await fetch(url, { method: 'DELETE' }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + return resp.json(); + }, + + /** 检索 */ + search(params) { + const qs = new URLSearchParams(params).toString(); + return API.get(`/api/search?${qs}`); + }, + + /** 法规列表 */ + listLaws(params) { + const qs = new URLSearchParams(params).toString(); + return API.get(`/api/laws?${qs}`); + }, + + /** 法规详情 */ + getLaw(lawId) { + return API.get(`/api/law/${lawId}`); + }, + + /** 统计 */ + stats() { + return API.get('/api/stats'); + }, +}; + +// ===== 初始化 ===== +document.addEventListener('DOMContentLoaded', () => { + // 导航绑定 + document.querySelectorAll('.nav-item').forEach(btn => { + btn.addEventListener('click', () => navigate(btn.dataset.page)); + }); + // 移动端菜单 + document.getElementById('mobile-menu-btn').addEventListener('click', () => { + document.getElementById('sidebar').classList.toggle('open'); + }); + // 加载统计 + loadStats(); + // 从 URL 决定初始页 + const initialPage = pageFromPath(); + // 替换当前历史记录,确保有 state + history.replaceState({ page: initialPage }, '', `/${initialPage === 'rag' ? '' : initialPage}`); + renderPage(initialPage); +}); + +async function loadStats() { + try { + const resp = await API.stats(); + const s = resp.data; + if (s && s.ready) { + document.getElementById('status-dot').className = 'w-2 h-2 rounded-full bg-emerald-500'; + document.getElementById('status-text').textContent = '就绪'; + document.getElementById('stat-clauses').textContent = s.total_clauses.toLocaleString(); + document.getElementById('stat-laws').textContent = s.total_laws.toLocaleString(); + } else { + document.getElementById('status-dot').className = 'w-2 h-2 rounded-full bg-amber-400'; + document.getElementById('status-text').textContent = '索引未就绪'; + } + } catch (e) { + document.getElementById('status-dot').className = 'w-2 h-2 rounded-full bg-rose-500'; + document.getElementById('status-text').textContent = '服务不可用'; + } +} diff --git a/app/static/js/browse.js b/app/static/js/browse.js new file mode 100644 index 0000000..5005a14 --- /dev/null +++ b/app/static/js/browse.js @@ -0,0 +1,212 @@ +/** + * browse.js — 法规浏览页 + 原文详情 + */ +const BrowsePage = { + state: { view: 'list', laws: [], total: 0, page: 1, pageSize: 50, category: '', province: '', keyword: '', loading: false, error: null, currentLaw: null }, + + render() { + if (this.state.view === 'detail' && this.state.currentLaw) { + return this.renderDetail(); + } + return this.renderList(); + }, + + renderList() { + return ` +
+

法规浏览

+ + +
+
+ + + + +
+
+ + +
+ ${UI.emptyState('输入条件浏览法规', '可按类别、省份、关键词筛选')} +
+
`; + }, + + renderDetail() { + const law = this.state.currentLaw; + const badge = UI.categoryBadge(law.category); + const province = law.province ? `[${law.province}${law.city ? ' · ' + law.city : ''}]` : ''; + const date = law.publish_date ? `发布: ${law.publish_date}` : ''; + + // 目录树(章节 + 条号) + const toc = (law.clauses || []).map(c => { + const chapter = c.chapter ? c.chapter : ''; + return `
+ ${chapter} + ${c.clause_no || ''} +
`; + }).join(''); + + return ` +
+ + +
+

${law.name}

+ ${badge} + ${province} + ${date} +
+ +
+ + ${toc ? `` : ''} + + +
+
${marked.parse(law.content)}
+
+
+
`; + }, + + init() { + this.loadProvinces(); + if (this.state.category) document.getElementById('browse-category').value = this.state.category; + if (this.state.province) document.getElementById('browse-province').value = this.state.province; + if (this.state.laws.length === 0) this.doBrowse(1); + }, + + provinces: ['北京市','天津市','河北省','山西省','内蒙古自治区','辽宁省','吉林省','黑龙江省','上海市','江苏省','浙江省','安徽省','福建省','江西省','山东省','河南省','湖北省','湖南省','广东省','广西壮族自治区','海南省','重庆市','四川省','贵州省','云南省','西藏自治区','陕西省','甘肃省','青海省','宁夏回族自治区','新疆维吾尔自治区'], + + loadProvinces() { + const sel = document.getElementById('browse-province'); + if (!sel) return; + this.provinces.forEach(p => { + const opt = document.createElement('option'); + opt.value = p; opt.textContent = p; + sel.appendChild(opt); + }); + }, + + onFilterChange() { + this.state.category = document.getElementById('browse-category').value; + this.state.province = document.getElementById('browse-province').value; + }, + + async doBrowse(page) { + const kwInput = document.getElementById('browse-keyword'); + if (kwInput) this.state.keyword = kwInput.value.trim(); + this.state.page = page || 1; + this.state.view = 'list'; + this.state.loading = true; + this.state.error = null; + this.renderListState(); + + try { + const params = { page: this.state.page, page_size: this.state.pageSize }; + if (this.state.category) params.category = this.state.category; + if (this.state.province) params.province = this.state.province; + if (this.state.keyword) params.keyword = this.state.keyword; + + const resp = await API.listLaws(params); + this.state.laws = resp.data.results || []; + this.state.total = resp.data.total || 0; + this.state.loading = false; + this.renderListState(); + } catch (e) { + this.state.loading = false; + this.state.error = e.message; + this.renderListState(); + } + }, + + renderListState() { + const el = document.getElementById('browse-list'); + if (!el) return; + + if (this.state.loading) { + el.innerHTML = `
${UI.spinner(20)}加载中...
`; + return; + } + + if (this.state.error) { + el.innerHTML = UI.errorState(this.state.error, 'BrowsePage.doBrowse()'); + return; + } + + if (!this.state.laws.length) { + el.innerHTML = UI.emptyState('无法规', '试试调整筛选条件'); + return; + } + + const rows = this.state.laws.map(l => ` +
+
+ ${l.name} + ${UI.categoryBadge(l.category)} + ${l.province ? `[${l.province}${l.city ? ' · ' + l.city : ''}]` : ''} + ${l.publish_date ? `${l.publish_date}` : ''} + ${l.clause_count} 条 +
+
`).join(''); + + const pager = UI.pagination(this.state.page, this.state.pageSize, this.state.total, 'BrowsePage.doBrowse'); + el.innerHTML = `
共 ${this.state.total} 篇法规
${rows}${pager}`; + }, + + async viewLaw(lawId) { + this.state.view = 'detail'; + this.state.currentLaw = null; + const content = document.getElementById('content'); + content.innerHTML = `
${UI.spinner(20)}加载原文...
`; + + try { + const resp = await API.getLaw(lawId); + this.state.currentLaw = resp.data; + content.innerHTML = this.renderDetail(); + } catch (e) { + content.innerHTML = UI.errorState(e.message, `BrowsePage.viewLaw(${lawId})`); + } + }, + + backToList() { + this.state.view = 'list'; + this.state.currentLaw = null; + const content = document.getElementById('content'); + content.innerHTML = this.renderList(); + this.init(); + this.renderListState(); + }, + + scrollToClause(clauseId) { + // 简单实现:滚动到条文内容(原文中条文已渲染) + // TODO: 更精确的定位需要解析原文中的条号 + const el = document.getElementById(`clause-${clauseId}`); + if (el) { + el.scrollIntoView({ behavior: 'smooth', block: 'center' }); + el.classList.add('clause-highlight'); + setTimeout(() => el.classList.remove('clause-highlight'), 2000); + } + }, +}; diff --git a/app/static/js/history.js b/app/static/js/history.js new file mode 100644 index 0000000..17fa1af --- /dev/null +++ b/app/static/js/history.js @@ -0,0 +1,332 @@ +/** + * history.js — 历史记录页(检索历史 + 对话历史) + */ +const HistoryPage = { + state: { + tab: 'chat', // 'chat' 或 'search' + chatPage: 1, + chatPageSize: 20, + chatData: null, + searchPage: 1, + searchPageSize: 20, + searchData: null, + loading: false, + error: null, + detailChat: null, // 查看详情的对话 + }, + + render() { + const tab = this.state.tab; + return ` +
+

历史记录

+ + +
+ + +
+ +
+ + +
+ ${this._renderLoading()} +
+
+ `; + }, + + init() { + this.loadList(); + }, + + _renderLoading() { + return `
${UI.spinner(20)}加载中...
`; + }, + + setTab(tab) { + this.state.tab = tab; + this.state.error = null; + this.state.detailChat = null; + const content = document.getElementById('content'); + content.innerHTML = this.render(); + this.loadList(); + }, + + async loadList() { + this.state.loading = true; + this.state.error = null; + const el = document.getElementById('history-content'); + if (el) el.innerHTML = this._renderLoading(); + + try { + if (this.state.tab === 'chat') { + const resp = await API.get(`/api/history/chat?page=${this.state.chatPage}&page_size=${this.state.chatPageSize}`); + this.state.chatData = resp.data; + } else { + const resp = await API.get(`/api/history/search?page=${this.state.searchPage}&page_size=${this.state.searchPageSize}`); + this.state.searchData = resp.data; + } + this.state.loading = false; + this.renderList(); + } catch (e) { + this.state.loading = false; + this.state.error = e.message; + this.renderList(); + } + }, + + renderList() { + const el = document.getElementById('history-content'); + if (!el) return; + + if (this.state.error) { + el.innerHTML = UI.errorState(this.state.error, 'HistoryPage.loadList()'); + return; + } + + if (this.state.tab === 'chat') { + el.innerHTML = this._renderChatList(); + } else { + el.innerHTML = this._renderSearchList(); + } + }, + + _renderChatList() { + const data = this.state.chatData; + if (!data || !data.items || data.items.length === 0) { + return UI.emptyState('暂无对话历史', '在智能问答页提问后会自动记录'); + } + + const items = data.items.map(item => { + const tplLabel = { simple: '通俗解释', professional: '专业分析', compare: '对比条文' }[item.template] || item.template; + const statusBadge = item.status === 'error' + ? '错误' + : item.status === 'streaming' + ? '生成中' + : ''; + const answerPreview = item.answer + ? item.answer.substring(0, 120).replace(/\n/g, ' ') + (item.answer.length > 120 ? '...' : '') + : '无答案'; + const citationCount = item.citations ? item.citations.length : 0; + + return ` +
+
+
+
+ ${item.created_at} + ${tplLabel} + ${item.thinking_enabled ? 'Thinking' : ''} + ${statusBadge} + ${citationCount ? `引用 ${citationCount} 条` : ''} +
+
${this._escapeHtml(item.question)}
+
${this._escapeHtml(answerPreview)}
+
+
+ + +
+
+
+ `; + }).join(''); + + const pagination = UI.pagination( + data.page, data.page_size, data.total, + 'HistoryPage.changeChatPage' + ); + + return items + pagination; + }, + + _renderSearchList() { + const data = this.state.searchData; + if (!data || !data.items || data.items.length === 0) { + return UI.emptyState('暂无检索历史', '在语义检索页查询后会自动记录'); + } + + const items = data.items.map(item => { + const filters = []; + if (item.category) filters.push(UI.categoryBadge(item.category)); + if (item.province) filters.push(`[${item.province}]`); + return ` +
+
+
+
+ ${item.created_at} + ${filters.join('')} + ${item.result_count} 条结果 +
+
${this._escapeHtml(item.query)}
+
+
+ + +
+
+
+ `; + }).join(''); + + const pagination = UI.pagination( + data.page, data.page_size, data.total, + 'HistoryPage.changeSearchPage' + ); + + return items + pagination; + }, + + changeChatPage(page) { + this.state.chatPage = page; + this.loadList(); + }, + + changeSearchPage(page) { + this.state.searchPage = page; + this.loadList(); + }, + + async viewChat(id) { + try { + const resp = await API.get(`/api/history/chat/${id}`); + this.state.detailChat = resp.data; + this._renderChatDetail(); + } catch (e) { + this.state.error = e.message; + this.renderList(); + } + }, + + _renderChatDetail() { + const chat = this.state.detailChat; + if (!chat) return; + + const tplLabel = { simple: '通俗解释', professional: '专业分析', compare: '对比条文' }[chat.template] || chat.template; + const thinkingHtml = chat.thinking + ? `
+ + + Thinking 思考过程 + ${chat.thinking.length} 字 + +
+ 
${this._escapeHtml(chat.thinking)}
+
+
` + : ''; + const answerHtml = chat.answer + ? `
${marked.parse(chat.answer)}
` + : '
无答案
'; + const citationsHtml = chat.citations && chat.citations.length + ? `
引用条文(${chat.citations.length})
${chat.citations.map(c => UI.clauseCard(c)).join('')}
` + : ''; + + const el = document.getElementById('history-content'); + el.innerHTML = ` +
+ +
+
+
+ ${chat.created_at} + ${tplLabel} + ${chat.thinking_enabled ? 'Thinking' : ''} +
+
${this._escapeHtml(chat.question)}
+
AI 回答
+ ${thinkingHtml} + ${answerHtml} + ${citationsHtml} +
+ `; + }, + + closeDetail() { + this.state.detailChat = null; + this.renderList(); + }, + + async deleteChat(id) { + if (!confirm('确认删除这条对话历史?')) return; + try { + await fetch(`/api/history/chat/${id}`, { method: 'DELETE' }); + this.loadList(); + } catch (e) { + alert('删除失败: ' + e.message); + } + }, + + async deleteSearch(id) { + if (!confirm('确认删除这条检索历史?')) return; + try { + await fetch(`/api/history/search/${id}`, { method: 'DELETE' }); + this.loadList(); + } catch (e) { + alert('删除失败: ' + e.message); + } + }, + + async clearAll() { + const type = this.state.tab === 'chat' ? '对话' : '检索'; + if (!confirm(`确认清空全部${type}历史?此操作不可恢复。`)) return; + try { + const url = this.state.tab === 'chat' ? '/api/history/chat' : '/api/history/search'; + await fetch(url, { method: 'DELETE' }); + this.loadList(); + } catch (e) { + alert('清空失败: ' + e.message); + } + }, + + replaySearch(item) { + // 跳转到检索页并填充查询条件 + navigate('search'); + setTimeout(() => { + const input = document.getElementById('search-input'); + if (input) input.value = item.query; + if (SearchPage.state) { + SearchPage.state.query = item.query; + SearchPage.state.category = item.category || ''; + SearchPage.state.province = item.province || ''; + if (item.category) { + const sel = document.getElementById('search-category'); + if (sel) sel.value = item.category; + } + if (item.province) { + const sel = document.getElementById('search-province'); + if (sel) sel.value = item.province; + } + } + // 自动触发检索 + if (SearchPage.submit) SearchPage.submit(); + }, 100); + }, + + _escapeHtml(text) { + if (!text) return ''; + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + }, +}; diff --git a/app/static/js/rag.js b/app/static/js/rag.js new file mode 100644 index 0000000..70110a0 --- /dev/null +++ b/app/static/js/rag.js @@ -0,0 +1,207 @@ +/** + * rag.js — 智能问答页(RAG + SSE 流式) + */ + +const RagPage = { + state: { + question: '', template: 'simple', answer: '', citations: [], streaming: false, error: null, + thinking: '', // thinking 内容 + thinkingEnabled: true, // thinking 开关(从 API 加载) + }, + + render() { + const templates = [ + { value: 'simple', label: '通俗解释' }, + { value: 'professional', label: '专业分析' }, + { value: 'compare', label: '对比条文' }, + ]; + return ` +
+

智能问答

+ + +
+ +
+
+ ${templates.map(t => ` + `).join('')} +
+
+ ${this.state.question.length}/1000 + +
+
+
+ + +
+ ${UI.emptyState('提问开始对话', '系统将检索相关法规条文并由 AI 生成带引用的答案')} +
+
`; + }, + + init() { + // 加载 LLM 参数(用于 thinking 状态显示) + this.loadLLMParams(); + }, + + async loadLLMParams() { + try { + const resp = await API.get('/api/prompts'); + this.state.thinkingEnabled = resp.data.llm_params.thinking_enabled; + } catch (e) { + console.error('加载 LLM 参数失败:', e); + } + }, + + onInput() { + const input = document.getElementById('rag-input'); + if (input) this.state.question = input.value; + }, + + setTemplate(t) { + this.state.template = t; + const content = document.getElementById('content'); + content.innerHTML = this.render(); + }, + + async submit() { + const input = document.getElementById('rag-input'); + if (input) this.state.question = input.value.trim(); + if (!this.state.question || this.state.streaming) return; + + this.state.answer = ''; + this.state.thinking = ''; + this.state.citations = []; + this.state.streaming = true; + this.state.error = null; + this.renderAnswer(); + + try { + const resp = await fetch('/api/rag', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + question: this.state.question, + template: this.state.template, + }), + }); + + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + + // SSE 流式读取 + const reader = resp.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop(); // 保留不完整的行 + + for (const line of lines) { + if (!line.startsWith('data: ')) continue; + try { + const evt = JSON.parse(line.slice(6)); + if (evt.type === 'answer') { + this.state.answer += evt.content; + this.renderAnswer(); + } else if (evt.type === 'thinking') { + this.state.thinking += evt.content; + this.renderAnswer(); + } else if (evt.type === 'params') { + this.state.thinkingEnabled = evt.thinking_enabled; + } else if (evt.type === 'citations') { + this.state.citations = evt.clauses || []; + this.renderAnswer(); + } else if (evt.type === 'no_result') { + this.state.error = evt.message; + this.renderAnswer(); + } else if (evt.type === 'error') { + this.state.error = evt.message; + this.renderAnswer(); + } else if (evt.type === 'done') { + // 完成 + } + } catch (e) { /* 忽略解析错误 */ } + } + } + this.state.streaming = false; + this.renderAnswer(); + } catch (e) { + this.state.streaming = false; + this.state.error = e.message; + this.renderAnswer(); + } + }, + + renderAnswer() { + const el = document.getElementById('rag-answer'); + if (!el) return; + + if (this.state.error) { + el.innerHTML = UI.errorState(this.state.error, 'RagPage.submit()'); + return; + } + + if (!this.state.answer && !this.state.citations.length && this.state.streaming) { + el.innerHTML = `
${UI.spinner(20)}检索法规并生成答案中...
`; + return; + } + + if (!this.state.answer && !this.state.thinking && !this.state.citations.length) { + el.innerHTML = UI.emptyState('提问开始对话', '系统将检索相关法规条文并由 AI 生成带引用的答案'); + return; + } + + // thinking 折叠区(仅 thinking 开启且有内容时显示) + const thinkingHtml = this.state.thinking + ? `
+ + + Thinking 思考过程 + ${this.state.thinking.length} 字 + ${this.state.streaming && !this.state.answer ? '思考中...' : ''} + +
+ 
${this._escapeHtml(this.state.thinking)}
+
+
` + : ''; + + const answerHtml = this.state.answer + ? `
${marked.parse(this.state.answer)}
` + : (this.state.streaming && this.state.thinking ? '
等待生成答案...
' : ''); + const citationsHtml = this.state.citations.length + ? `
引用条文(${this.state.citations.length})
${this.state.citations.map(c => UI.clauseCard(c)).join('')}
` + : ''; + + el.innerHTML = ` +
+
AI 回答
+ ${thinkingHtml} + ${answerHtml} + ${this.state.streaming ? `
${UI.spinner(14)} 生成中...
` : ''} + ${citationsHtml} +
`; + }, + + _escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + }, +}; diff --git a/app/static/js/search.js b/app/static/js/search.js new file mode 100644 index 0000000..940e5c3 --- /dev/null +++ b/app/static/js/search.js @@ -0,0 +1,168 @@ +/** + * search.js — 语义检索页 + */ +const SearchPage = { + state: { query: '', mode: 'semantic', category: '', province: '', city: '', results: [], total: 0, page: 1, pageSize: 20, loading: false, error: null }, + + render() { + const isKeyword = this.state.mode === 'keyword'; + return ` +
+

法规检索

+ + +
+ +
+ + + + ${isKeyword ? '按法规名+条号精确匹配,如"郑州市劳动用工条例 第三十二条"' : '自然语言语义搜索,如"加班工资计算基数"'} + +
+
+ + +
+
+ + + ${this.state.query.length}/1000 +
+
+ + +
+ ${UI.emptyState('输入查询开始检索', isKeyword ? '按法规名+条号精确匹配' : '支持自然语言查询,如"个人信息跨境传输的规定"')} +
+
`; + }, + + setMode(mode) { + this.state.mode = mode; + this.state.results = []; + this.state.total = 0; + const content = document.getElementById('content'); + content.innerHTML = this.render(); + this.init(); + }, + + init() { + // 加载省份列表(从统计或固定列表) + this.loadProvinces(); + // 恢复筛选状态 + if (this.state.category) document.getElementById('filter-category').value = this.state.category; + if (this.state.province) document.getElementById('filter-province').value = this.state.province; + }, + + provinces: ['北京市','天津市','河北省','山西省','内蒙古自治区','辽宁省','吉林省','黑龙江省','上海市','江苏省','浙江省','安徽省','福建省','江西省','山东省','河南省','湖北省','湖南省','广东省','广西壮族自治区','海南省','重庆市','四川省','贵州省','云南省','西藏自治区','陕西省','甘肃省','青海省','宁夏回族自治区','新疆维吾尔自治区'], + + loadProvinces() { + const sel = document.getElementById('filter-province'); + if (!sel) return; + this.provinces.forEach(p => { + const opt = document.createElement('option'); + opt.value = p; opt.textContent = p; + sel.appendChild(opt); + }); + }, + + onFilterChange() { + this.state.category = document.getElementById('filter-category').value; + this.state.province = document.getElementById('filter-province').value; + if (this.state.query) this.doSearch(); + }, + + async doSearch(page) { + const input = document.getElementById('search-input'); + if (input) this.state.query = input.value.trim(); + if (!this.state.query) return; + + // 自动判定模式:包含"第X条"且看起来像法规名+条号时,自动用精确查找 + const hasClauseNo = /第[一二三四五六七八九十百千零\d]+条/.test(this.state.query); + const autoMode = hasClauseNo ? 'keyword' : this.state.mode; + + this.state.page = page || 1; + this.state.loading = true; + this.state.error = null; + this.renderResults(); + + try { + const params = { + query: this.state.query, + mode: autoMode, + top_k: 20, + page: this.state.page, + page_size: this.state.pageSize, + }; + if (this.state.category) params.category = this.state.category; + if (this.state.province) params.province = this.state.province; + + const resp = await API.search(params); + this.state.results = resp.data.results || []; + this.state.total = resp.data.total || 0; + this.state.loading = false; + this.renderResults(); + } catch (e) { + this.state.loading = false; + this.state.error = e.message; + this.renderResults(); + } + }, + + renderResults() { + const el = document.getElementById('search-results'); + if (!el) return; + + if (this.state.loading) { + el.innerHTML = `
${UI.spinner(20)}检索中...
`; + return; + } + + if (this.state.error) { + el.innerHTML = UI.errorState(this.state.error, 'SearchPage.doSearch()'); + return; + } + + if (!this.state.results.length && this.state.query) { + el.innerHTML = UI.emptyState('无检索结果', '试试调整查询词或筛选条件'); + return; + } + + if (!this.state.results.length) { + el.innerHTML = UI.emptyState('输入查询开始检索', '支持自然语言查询'); + return; + } + + const cards = this.state.results.map(r => UI.clauseCard(r)).join(''); + const pager = UI.pagination(this.state.page, this.state.pageSize, this.state.total, 'SearchPage.doSearch'); + // 显示当前检索模式 + const hasClauseNo = /第[一二三四五六七八九十百千零\d]+条/.test(this.state.query); + const autoMode = hasClauseNo ? 'keyword' : this.state.mode; + const modeLabel = autoMode === 'keyword' ? '精确查找' : '语义检索'; + const autoHint = hasClauseNo && this.state.mode === 'semantic' ? ' (检测到条号,自动切换精确查找)' : ''; + el.innerHTML = `
找到 ${this.state.total} 条结果 [${modeLabel}]${autoHint}
${cards}${pager}`; + }, +}; diff --git a/app/static/js/settings.js b/app/static/js/settings.js new file mode 100644 index 0000000..eefb7f7 --- /dev/null +++ b/app/static/js/settings.js @@ -0,0 +1,258 @@ +/** + * settings.js — 设置页(系统提示词 + 模板正文 + LLM 参数) + * 所有配置持久化到 SQLite,通过 /api/prompts 读写 + */ + +const SettingsPage = { + state: { + template: 'simple', + configs: null, // 从 API 加载的配置(含自定义值) + llmParams: null, // LLM 参数 + systemPrompt: '', // 系统提示词(编辑框值,空=使用默认) + templateText: '', // 模板正文(编辑框值,空=使用默认) + saveStatus: '', // 保存状态提示 + loading: true, + error: null, + }, + + render() { + if (this.state.loading) { + return `
+

设置

+
${UI.spinner(20)}加载配置中...
+
`; + } + if (this.state.error) { + return `
+

设置

+ ${UI.errorState(this.state.error, 'SettingsPage.init()')} +
`; + } + + const tpl = TEMPLATES.find(t => t.value === this.state.template); + const llm = this.state.llmParams || {}; + + return ` +
+

设置

+

配置系统提示词、问答模板和 LLM 参数,所有设置持久化到数据库。

+ + +
+
+ + +
+ +

定义 AI 的角色和基本行为。留空使用默认值。

+
+ + +
+
+ + +
+ +
+ ${TEMPLATES.map(t => ` + `).join('')} +
+ +

自定义模板正文,必须包含 {context} 和 {question} 占位符。留空使用预设模板。

+ + +
+ 查看预设模板参考(当前: ${tpl?.label || ''}) + 
${this._escapeHtml(PROMPT_DEFAULTS['template_' + this.state.template] || '')}
+
+
+ + +
+
LLM 参数
+
+ +
+ + +
+ +
+
+ + ${llm.temperature ?? 0.3} +
+ +
+ +
+ + +
+ +
+ + +
+
+

Thinking 开启时,AI 会先思考再回答(可折叠查看)。Max Tokens 是总 token 上限(含 thinking)。

+
+ + +
+
+ `; + }, + + async init() { + await this.loadConfigs(); + }, + + async loadConfigs() { + this.state.loading = true; + const content = document.getElementById('content'); + if (content) content.innerHTML = this.render(); + + try { + const resp = await API.get('/api/prompts'); + this.state.configs = resp.data.configs; + this.state.llmParams = resp.data.llm_params; + // 系统提示词:直接显示数据库值(空则用默认值填充) + this.state.systemPrompt = this.state.configs.system_prompt || PROMPT_DEFAULTS.system_prompt; + // 当前模板正文:直接显示数据库值(空则用默认值填充) + this._loadTemplateText(); + this.state.loading = false; + this.state.error = null; + if (content) content.innerHTML = this.render(); + } catch (e) { + this.state.loading = false; + this.state.error = e.message; + if (content) content.innerHTML = this.render(); + } + }, + + _loadTemplateText() { + const tplKey = `template_${this.state.template}`; + this.state.templateText = this.state.configs[tplKey] || PROMPT_DEFAULTS[tplKey]; + }, + + setTemplate(t) { + this.state.template = t; + if (this.state.configs) this._loadTemplateText(); + const content = document.getElementById('content'); + content.innerHTML = this.render(); + }, + + // ===== 系统提示词 ===== + async saveSystemPrompt(val) { + this.state.systemPrompt = val; + this._showSaveStatus('保存中...'); + try { + await API.put('/api/prompts', { system_prompt: val }); + this._showSaveStatus('已保存'); + } catch (e) { + this._showSaveStatus('保存失败'); + } + }, + + async resetSystemPrompt() { + this.state.systemPrompt = ''; + this._showSaveStatus('重置中...'); + try { + await API.put('/api/prompts', { system_prompt: '' }); + const el = document.getElementById('settings-system-prompt'); + if (el) el.value = ''; + this._showSaveStatus('已恢复默认'); + } catch (e) { + this._showSaveStatus('重置失败'); + } + }, + + // ===== 模板正文 ===== + async saveTemplateText(val) { + this.state.templateText = val; + this._showSaveStatus('保存中...'); + try { + const body = {}; + body[`template_${this.state.template}`] = val; + await API.put('/api/prompts', body); + this._showSaveStatus('已保存'); + } catch (e) { + this._showSaveStatus('保存失败'); + } + }, + + async resetTemplateText() { + this.state.templateText = ''; + this._showSaveStatus('重置中...'); + try { + const body = {}; + body[`template_${this.state.template}`] = ''; + await API.put('/api/prompts', body); + const el = document.getElementById('settings-template-text'); + if (el) el.value = ''; + this._showSaveStatus('已恢复默认'); + } catch (e) { + this._showSaveStatus('重置失败'); + } + }, + + // ===== LLM 参数 ===== + async saveLLMParam(key, value) { + this._showSaveStatus('保存中...'); + try { + const body = {}; + body[key] = String(value); + await API.put('/api/prompts', body); + this._showSaveStatus('已保存'); + // 重新加载参数 + const resp = await API.get('/api/prompts'); + this.state.llmParams = resp.data.llm_params; + } catch (e) { + this._showSaveStatus('保存失败'); + } + }, + + // ===== 工具方法 ===== + _showSaveStatus(msg) { + this.state.saveStatus = msg; + const el = document.getElementById('settings-save-status'); + if (el) { + el.textContent = msg; + el.style.opacity = '1'; + setTimeout(() => { el.style.opacity = '0'; }, 2000); + } + }, + + _escapeHtml(text) { + if (!text) return ''; + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + }, +}; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..675a04a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi>=0.110.0 +uvicorn[standard]>=0.29.0 +httpx>=0.27.0 +faiss-cpu>=1.7.4 +pydantic>=2.0.0 +python-multipart>=0.0.9 diff --git a/run.md b/run.md new file mode 100644 index 0000000..053f211 --- /dev/null +++ b/run.md @@ -0,0 +1,247 @@ +# run.md — QYLAW 法律法规知识库 运行手册 + +> 项目:QYLAW 法律法规知识库 +> 部署:192.168.110.114 (Docker, --network host) +> 更新:2026-08-06 + +--- + +## 一、技术栈 + +| 项目 | 版本/说明 | +|------|------| +| 后端 | Python 3.11 + FastAPI + Uvicorn | +| 向量库 | FAISS (faiss-cpu, HNSW 索引) | +| metadata | SQLite (Python 内置 sqlite3) | +| 向量化 | Qwen3-Embedding-0.6B (114:8003, 复用) | +| RAG 生成 | Qwen3.5-35B (114:7000, 复用) | +| 前端 | HTML + Vanilla JS + Tailwind CDN | +| 部署 | Docker (--network host) | +| 端口 | 8090 | + +--- + +## 二、首次准备 + +### 2.1 数据准备 + +```bash +ssh nvidia@192.168.110.114 +# 密码: qy123. + +# 确认 law-pack 已传输到 114 +ls /data/law-pack-2026-07-01-markdown/ +# 预期: 法律/ 行政法规/ 监察法规/ 司法解释/ 地方性法规/ 地方性法规区域映射.json + +# 若不存在,从本机 rsync 传输(310MB) +# 在本机执行: +rsync -avz --progress /Users/freedak/Documents/AIDashboard/qy123/law-pack-2026-07-01-markdown/ \ + nvidia@192.168.110.114:/data/law-pack-2026-07-01-markdown/ + +# 创建索引数据目录 +mkdir -p /data/law-kb-data/faiss /data/law-kb-data/logs +``` + +### 2.2 确认依赖服务 + +```bash +# embedding 服务(8003) +curl -s http://localhost:8003/v1/models | python3 -m json.tool + +# qwen35 服务(7000) +curl -s http://localhost:7000/v1/models | python3 -m json.tool +``` + +--- + +## 三、索引构建 + +### 3.1 全量构建 + +```bash +# [Docker] 在容器内执行(构建脚本需要访问 embedding 服务) +docker run --rm --network host \ + -v /data/law-pack-2026-07-01-markdown:/data/law-pack-2026-07-01-markdown:ro \ + -v /data/law-kb-data:/data/law-kb-data \ + law-kb:latest \ + python scripts/build_index.py --mode full + +# [Native] 本地直接执行(需已安装依赖) +cd law-kb +python scripts/build_index.py --mode full +``` + +**输出示例**: +``` +============================================ + QYLAW 索引构建 — 全量模式 + 时间: 2026-08-06 14:00:00 +============================================ + 扫描目录: /data/law-pack-2026-07-01-markdown + [法律] 349 文件, 切片 12500 条 + [行政法规] 606 文件, 切片 28000 条 + ... + 跳过文件(null 映射): 23 篇 + 总切片: 520000 条 + 向量维度: 1024 + FAISS 索引大小: 1.2 GB + 耗时: 2.5 小时 + 索引文件: /data/law-kb-data/faiss/index.faiss + SQLite: /data/law-kb-data/metadata.db +============================================ +``` + +### 3.2 增量构建 + +```bash +# [Docker] +docker run --rm --network host \ + -v /data/law-pack-2026-07-01-markdown:/data/law-pack-2026-07-01-markdown:ro \ + -v /data/law-kb-data:/data/law-kb-data \ + law-kb:latest \ + python scripts/build_index.py --mode incremental +``` + +### 3.3 索引验证 + +```bash +# [Docker] +docker run --rm --network host \ + -v /data/law-pack-2026-07-01-markdown:/data/law-pack-2026-07-01-markdown:ro \ + -v /data/law-kb-data:/data/law-kb-data \ + law-kb:latest \ + python scripts/verify_index.py +``` + +--- + +## 四、应用启停 + +### 4.1 启动服务 + +```bash +# [Docker] 启动 law-kb 容器 +docker run -d --name law-kb --network host \ + --restart unless-stopped \ + -v /data/law-pack-2026-07-01-markdown:/data/law-pack-2026-07-01-markdown:ro \ + -v /data/law-kb-data:/data/law-kb-data \ + law-kb:latest + +# 验证 +curl -s http://localhost:8090/health +curl -s http://localhost:8090/api/stats | python3 -m json.tool +``` + +### 4.2 停止服务 + +```bash +docker stop law-kb +``` + +### 4.3 重启服务 + +```bash +docker restart law-kb +``` + +### 4.4 查看日志 + +```bash +docker logs law-kb --tail 50 +docker logs -f law-kb +``` + +--- + +## 五、构建与部署 + +### 5.1 构建镜像 + +```bash +# 在 114 上构建 +cd /data/project/law-kb # 或项目实际路径 +docker build -t law-kb:latest . +``` + +### 5.2 一键部署 + +```bash +# 使用部署脚本(本机执行,自动 rsync + 构建 + 启动) +bash scripts/deploy.sh +``` + +--- + +## 六、备份与恢复 + +### 6.1 备份索引 + +```bash +# 索引文件可独立备份(FAISS + SQLite) +ssh nvidia@192.168.110.114 +tar -czf /data/backup/law-kb-index-$(date +%Y%m%d).tar.gz \ + -C /data/law-kb-data faiss/ metadata.db +``` + +### 6.2 恢复索引 + +```bash +ssh nvidia@192.168.110.114 +docker stop law-kb +tar -xzf /data/backup/law-kb-index-YYYYMMDD.tar.gz -C /data/law-kb-data/ +docker start law-kb +``` + +--- + +## 七、排错 + +| 问题 | 原因 | 解决方案 | +|------|------|------| +| 启动后 stats 返回 ready=false | 索引未构建 | 执行全量构建(三、3.1) | +| 检索返回 503 | embedding 服务未启动 | `docker ps` 确认 embedding 容器运行 | +| RAG 返回 503 | qwen35 服务未启动 | `docker ps` 确认 qwen35 容器运行 | +| RAG 返回 no_result | 相似度低于阈值(0.3) | 检查索引质量或调整问题表述 | +| 构建脚本 OOM | 切片数过多内存不足 | 减小 EMBEDDING_BATCH_SIZE | +| 容器无法访问 8003/7000 | 未用 --network host | 确认启动命令含 `--network host` | + +--- + +## 八、端口表 + +| 端口 | 服务 | 类型 | 说明 | +|------|------|------|------| +| 8090 | law-kb | HTTP | QYLAW 知识库 Web + API | +| 8003 | embedding | HTTP | 向量化服务(复用) | +| 7000 | qwen35 | HTTP | RAG 生成服务(复用) | + +--- + +## 九、环境变量 + +| 变量 | 默认值 | 说明 | +|------|------|------| +| `LAW_PACK_DIR` | /data/law-pack-2026-07-01-markdown | 法规原文目录 | +| `LAW_KB_DATA_DIR` | /data/law-kb-data | 索引数据目录 | +| `EMBEDDING_URL` | http://localhost:8003/v1 | embedding 服务地址 | +| `LLM_URL` | http://localhost:7000/v1 | LLM 服务地址 | +| `EMBEDDING_MODEL` | qwen3-embedding-0.6b | embedding 模型名 | +| `LLM_MODEL` | qwen3.5-35b | LLM 模型名 | +| `EMBEDDING_BATCH_SIZE` | 32 | embedding 批量大小 | +| `APP_PORT` | 8090 | 应用端口 | + +--- + +## 十、FAQ + +**Q: 为什么用 --network host?** +A: 容器需要访问 114 本地的 8003(embedding)和 7000(qwen35)端口。host 网络模式下容器直接用宿主网络,无需 --add-host 或端口映射,最简配置。内网环境无安全顾虑。 + +**Q: 索引构建中断了怎么办?** +A: 构建脚本先写临时文件,成功后原子替换。中断不会损坏已有索引。直接重跑即可。 + +**Q: 地方性法规 null 映射文件如何补?** +A: 编辑 `地方性法规区域映射.json` 补充映射,然后执行增量构建。 + +**Q: 如何更新法规数据?** +A: 替换 `/data/law-pack-2026-07-01-markdown/` 下的文件,执行增量构建(三、3.2)。 diff --git a/scripts/build_index.py b/scripts/build_index.py new file mode 100644 index 0000000..6c9a904 --- /dev/null +++ b/scripts/build_index.py @@ -0,0 +1,329 @@ +"""索引构建脚本 — 全量/增量构建 FAISS 索引 + SQLite metadata + +用法: + python scripts/build_index.py --mode full # 全量构建 + python scripts/build_index.py --mode incremental # 增量构建 + +流程: + 1. 扫描 law-pack 目录,按类别切片 + 2. 写入 SQLite metadata(laws + clauses 表) + 3. 批量调用 embedding 服务向量化所有条文 + 4. 构建 FAISS HNSW 索引并持久化 + 5. 输出统计信息 + +安全: + - 先写临时文件,成功后原子替换 + - 中断不损坏已有索引 +""" +import argparse +import asyncio +import json +import logging +import os +import sqlite3 +import sys +import time +from pathlib import Path +from typing import List, Dict, Any, Optional + +# 添加项目根目录到 path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import numpy as np +from app.config import ( + LAW_PACK_DIR, + LAW_KB_DATA_DIR, + FAISS_INDEX_PATH, + SQLITE_PATH, + LOGS_DIR, + REGION_MAPPING_PATH, + CATEGORY_DIRS, + EMBEDDING_BATCH_SIZE, + HNSW_M, + HNSW_EF_CONSTRUCTION, +) +from scripts.parse_clause import ( + Clause, + load_region_mapping, + scan_category_dir, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", +) +logger = logging.getLogger(__name__) + + +# ===== SQLite Schema ===== +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS laws ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + category TEXT NOT NULL, + publish_date TEXT, + province TEXT, + city TEXT, + region_level TEXT, + file_path TEXT NOT NULL, + clause_count INTEGER DEFAULT 0, + file_mtime REAL DEFAULT 0, + indexed_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS clauses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + law_id INTEGER NOT NULL, + chapter TEXT, + clause_no TEXT, + content TEXT NOT NULL, + faiss_idx INTEGER, + FOREIGN KEY (law_id) REFERENCES laws(id) +); + +CREATE INDEX IF NOT EXISTS idx_clauses_law_id ON clauses(law_id); +CREATE INDEX IF NOT EXISTS idx_clauses_faiss_idx ON clauses(faiss_idx); +CREATE INDEX IF NOT EXISTS idx_laws_category ON laws(category); +CREATE INDEX IF NOT EXISTS idx_laws_province ON laws(province); + +CREATE TABLE IF NOT EXISTS prompt_configs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + config_key TEXT NOT NULL UNIQUE, + config_value TEXT, + updated_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_prompt_configs_key ON prompt_configs(config_key); +""" + + +def init_sqlite(db_path: Path): + """初始化 SQLite(创建表)""" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(str(db_path)) + conn.executescript(SCHEMA_SQL) + conn.commit() + return conn + + +def reset_sqlite(db_path: Path): + """重置 SQLite(全量构建时用)""" + if db_path.exists(): + db_path.unlink() + return init_sqlite(db_path) + + +async def build_index(mode: str = "full"): + """构建索引""" + start_time = time.time() + timestamp = time.strftime("%Y-%m-%d %H:%M:%S") + + logger.info("=" * 60) + logger.info(f"QYLAW 索引构建 — {mode} 模式") + logger.info(f"时间: {timestamp}") + logger.info(f"法规目录: {LAW_PACK_DIR}") + logger.info(f"数据目录: {LAW_KB_DATA_DIR}") + logger.info("=" * 60) + + # 确保目录存在 + LAW_KB_DATA_DIR.mkdir(parents=True, exist_ok=True) + (LAW_KB_DATA_DIR / "faiss").mkdir(parents=True, exist_ok=True) + LOGS_DIR.mkdir(parents=True, exist_ok=True) + + # 加载区域映射 + region_mapping = load_region_mapping(REGION_MAPPING_PATH) + logger.info(f"加载区域映射: {len(region_mapping)} 条") + + # 扫描所有类别 + all_clauses: List[Clause] = [] + all_skipped: List[str] = [] + category_stats: Dict[str, Dict] = {} + + for category in CATEGORY_DIRS: + logger.info(f"扫描 [{category}]...") + clauses, skipped = scan_category_dir(LAW_PACK_DIR, category, region_mapping) + all_clauses.extend(clauses) + all_skipped.extend(skipped) + law_count = len(set(c.law_name for c in clauses)) + category_stats[category] = { + "laws": law_count, + "clauses": len(clauses), + "skipped": len(skipped), + } + logger.info(f" [{category}] {law_count} 法规, {len(clauses)} 条文, {len(skipped)} 跳过") + + logger.info(f"总计: {len(all_clauses)} 条文, {len(all_skipped)} 跳过文件") + + if not all_clauses: + logger.error("无有效条文,终止构建") + return + + # 写入 SQLite + if mode == "full": + conn = reset_sqlite(SQLITE_PATH) + else: + conn = init_sqlite(SQLITE_PATH) + + # 按法规分组 + laws_map: Dict[str, Clause] = {} # law_name -> 首个 clause(取 metadata) + law_clauses: Dict[str, List[Clause]] = {} + for c in all_clauses: + if c.law_name not in laws_map: + laws_map[c.law_name] = c + law_clauses[c.law_name] = [] + law_clauses[c.law_name].append(c) + + # 写 laws 表 + law_id_map: Dict[str, int] = {} + for law_name, first_clause in laws_map.items(): + file_path = Path(first_clause.file_path) + file_mtime = file_path.stat().st_mtime if file_path.exists() else 0 + cur = conn.execute( + """ + INSERT INTO laws (name, category, publish_date, province, city, region_level, + file_path, clause_count, file_mtime, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + law_name, + first_clause.category, + first_clause.publish_date, + first_clause.province, + first_clause.city, + first_clause.region_level, + first_clause.file_path, + len(law_clauses[law_name]), + file_mtime, + timestamp, + ), + ) + law_id_map[law_name] = cur.lastrowid + + # 写 clauses 表(暂不写 faiss_idx,向量化后更新) + clause_rows: List[tuple] = [] + for law_name, clauses in law_clauses.items(): + law_id = law_id_map[law_name] + for c in clauses: + clause_rows.append((law_id, c.chapter, c.clause_no, c.content)) + + conn.executemany( + "INSERT INTO clauses (law_id, chapter, clause_no, content) VALUES (?, ?, ?, ?)", + clause_rows, + ) + conn.commit() + + # 获取 clause id 顺序(与写入顺序一致) + clause_ids = [row[0] for row in conn.execute("SELECT id FROM clauses ORDER BY id").fetchall()] + clause_texts = [row[0] for row in conn.execute("SELECT content FROM clauses ORDER BY id").fetchall()] + logger.info(f"SQLite 写入完成: {len(clause_ids)} 条文, {len(law_id_map)} 法规") + + # 批量向量化 + logger.info(f"开始向量化(批量大小 {EMBEDDING_BATCH_SIZE})...") + + # 延迟导入 embedding 服务(避免循环依赖) + from app.services.embedding import embed_batch + + embed_start = time.time() + try: + all_vecs = await embed_batch(clause_texts, batch_size=EMBEDDING_BATCH_SIZE) + except Exception as e: + logger.error(f"向量化失败: {e}") + conn.close() + return + + embed_time = time.time() - embed_start + logger.info(f"向量化完成: {len(all_vecs)} 向量, 耗时 {embed_time:.0f}s") + + if len(all_vecs) != len(clause_ids): + logger.error(f"向量数 {len(all_vecs)} != 条文数 {len(clause_ids)}") + conn.close() + return + + vec_dim = len(all_vecs[0]) if all_vecs else 0 + logger.info(f"向量维度: {vec_dim}") + + # 更新 clauses 表的 faiss_idx + for i, cid in enumerate(clause_ids): + conn.execute("UPDATE clauses SET faiss_idx = ? WHERE id = ?", (i, cid)) + conn.commit() + + # 构建 FAISS 索引 + # 注:HNSW 构建内存峰值高(68万向量约 30GB),114 内存紧张时 OOM + # 改用 IndexFlatL2(暴力检索,零额外内存,68万向量检索 <100ms) + # 若内存充裕可改回 HNSW:faiss.IndexHNSWFlat(vec_dim, HNSW_M) + logger.info("构建 FAISS Flat 索引(暴力检索,省内存)...") + import faiss + + vecs_array = np.array(all_vecs, dtype=np.float32) + index = faiss.IndexFlatL2(vec_dim) + index.add(vecs_array) + + logger.info(f"FAISS 索引构建完成: {index.ntotal} 向量") + + # 原子写入(先写临时文件,成功后替换) + tmp_faiss = FAISS_INDEX_PATH.with_suffix(".faiss.tmp") + faiss.write_index(index, str(tmp_faiss)) + + # 替换 + if FAISS_INDEX_PATH.exists(): + FAISS_INDEX_PATH.unlink() + tmp_faiss.rename(FAISS_INDEX_PATH) + logger.info(f"FAISS 索引持久化: {FAISS_INDEX_PATH}") + + conn.close() + + # 输出统计 + total_time = time.time() - start_time + index_size = FAISS_INDEX_PATH.stat().st_size / 1024 / 1024 + + stats = { + "mode": mode, + "timestamp": timestamp, + "total_clauses": len(all_clauses), + "total_laws": len(law_id_map), + "category_stats": category_stats, + "skipped_files": all_skipped, + "vector_dim": vec_dim, + "faiss_index_size_mb": round(index_size, 1), + "embed_time_seconds": round(embed_time, 0), + "total_time_seconds": round(total_time, 0), + } + + # 写统计到日志文件 + stats_path = LOGS_DIR / f"build_{time.strftime('%Y%m%d_%H%M%S')}.json" + with open(stats_path, "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + + # 控制台输出 + print("\n" + "=" * 60) + print(f" QYLAW 索引构建完成 — {mode} 模式") + print(f" 耗时: {total_time:.0f}s ({total_time/3600:.1f}h)") + print("=" * 60) + for cat, s in category_stats.items(): + print(f" [{cat}] {s['laws']} 法规, {s['clauses']} 条文, {s['skipped']} 跳过") + print(f" 总切片: {len(all_clauses)} 条") + print(f" 向量维度: {vec_dim}") + print(f" FAISS 索引: {index_size:.1f} MB") + print(f" 向量化耗时: {embed_time:.0f}s") + print(f" 跳过文件: {len(all_skipped)} 篇") + print(f" 索引文件: {FAISS_INDEX_PATH}") + print(f" SQLite: {SQLITE_PATH}") + print(f" 统计日志: {stats_path}") + print("=" * 60) + + +def main(): + parser = argparse.ArgumentParser(description="QYLAW 索引构建") + parser.add_argument( + "--mode", + choices=["full", "incremental"], + default="full", + help="构建模式: full=全量重建, incremental=增量更新", + ) + args = parser.parse_args() + + asyncio.run(build_index(args.mode)) + + +if __name__ == "__main__": + main() diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..f3ae93a --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# QYLAW 部署脚本 — 传输数据 + 构建镜像 + 启动容器 +# 用法: bash scripts/deploy.sh +set -e + +# ===== 配置 ===== +REMOTE_HOST="nvidia@192.168.110.114" +REMOTE_LAW_PACK="/data/law-pack-2026-07-01-markdown" +REMOTE_LAW_KB_DATA="/data/law-kb-data" +LOCAL_LAW_PACK="/Users/freedak/Documents/AIDashboard/qy123/law-pack-2026-07-01-markdown" +PROJECT_DIR="$(cd "$(dirname "$0")/.." && pwd)" + +echo "============================================" +echo " QYLAW 法律法规知识库部署" +echo " 时间: $(date '+%Y-%m-%d %H:%M:%S')" +echo "============================================" + +# ===== Step 1: 传输法规数据(若 114 上不存在)===== +echo "" +echo "[Step 1] 检查并传输法规数据..." +ssh "$REMOTE_HOST" "test -d $REMOTE_LAW_PACK && echo EXISTS || echo MISSING" +LAW_PACK_STATUS=$(ssh "$REMOTE_HOST" "test -d $REMOTE_LAW_PACK && echo EXISTS || echo MISSING") + +if [ "$LAW_PACK_STATUS" = "MISSING" ]; then + echo " 法规数据不存在,开始 rsync 传输(310MB)..." + if [ ! -d "$LOCAL_LAW_PACK" ]; then + echo " [错误] 本地法规数据不存在: $LOCAL_LAW_PACK" + exit 1 + fi + rsync -avz --progress "$LOCAL_LAW_PACK/" "$REMOTE_HOST:$REMOTE_LAW_PACK/" + echo " [OK] 法规数据传输完成" +else + echo " [OK] 法规数据已存在,跳过传输" +fi + +# ===== Step 2: 创建索引数据目录 ===== +echo "" +echo "[Step 2] 创建索引数据目录..." +ssh "$REMOTE_HOST" "mkdir -p $REMOTE_LAW_KB_DATA/faiss $REMOTE_LAW_KB_DATA/logs" +echo " [OK] 目录就绪" + +# ===== Step 3: 传输项目代码并构建镜像 ===== +echo "" +echo "[Step 3] 传输代码并构建 Docker 镜像..." +ssh "$REMOTE_HOST" "mkdir -p /data/project/law-kb" +rsync -avz --exclude '__pycache__' --exclude '.git' --exclude 'law-kb-data' \ + "$PROJECT_DIR/" "$REMOTE_HOST:/data/project/law-kb/" + +echo " 构建 Docker 镜像..." +ssh "$REMOTE_HOST" "cd /data/project/law-kb && docker build -t law-kb:latest ." +echo " [OK] 镜像构建完成" + +# ===== Step 4: 停止旧容器(若存在)===== +echo "" +echo "[Step 4] 停止旧容器..." +ssh "$REMOTE_HOST" "docker rm -f law-kb 2>/dev/null || true" +echo " [OK] 旧容器已清理" + +# ===== Step 5: 启动新容器 ===== +echo "" +echo "[Step 5] 启动 law-kb 容器..." +ssh "$REMOTE_HOST" " +docker run -d --name law-kb --network host --restart unless-stopped \ + -v $REMOTE_LAW_PACK:$REMOTE_LAW_PACK:ro \ + -v $REMOTE_LAW_KB_DATA:$REMOTE_LAW_KB_DATA \ + -e LAW_PACK_DIR=$REMOTE_LAW_PACK \ + -e LAW_KB_DATA_DIR=$REMOTE_LAW_KB_DATA \ + -e EMBEDDING_URL=http://localhost:8003/v1 \ + -e LLM_URL=http://localhost:7000/v1 \ + law-kb:latest +" +echo " [OK] 容器已启动" + +# ===== Step 6: 健康检查 ===== +echo "" +echo "[Step 6] 健康检查..." +sleep 3 +echo " 等待服务启动..." +for i in $(seq 1 10); do + HEALTH=$(ssh "$REMOTE_HOST" "curl -s -o /dev/null -w '%{http_code}' http://localhost:8090/health 2>/dev/null || echo 000") + if [ "$HEALTH" = "200" ]; then + echo " [OK] 服务健康(尝试 $i)" + break + fi + echo " 等待中...($i/10, HTTP $HEALTH)" + sleep 2 +done + +# 统计 +echo "" +echo " 索引状态:" +ssh "$REMOTE_HOST" "curl -s http://localhost:8090/api/stats 2>/dev/null | python3 -m json.tool 2>/dev/null || echo ' 统计接口未就绪(可能需要先构建索引)'" + +echo "" +echo "============================================" +echo " 部署完成!" +echo " 访问: http://192.168.110.114:8090" +echo "" +echo " 下一步:" +echo " 1. 若索引未构建,执行:" +echo " ssh $REMOTE_HOST" +echo " docker run --rm --network host \\" +echo " -v $REMOTE_LAW_PACK:$REMOTE_LAW_PACK:ro \\" +echo " -v $REMOTE_LAW_KB_DATA:$REMOTE_LAW_KB_DATA \\" +echo " law-kb:latest python scripts/build_index.py --mode full" +echo " 2. 构建完成后重启容器: docker restart law-kb" +echo "============================================" diff --git a/scripts/eval_retrieval.py b/scripts/eval_retrieval.py new file mode 100644 index 0000000..7842b9e --- /dev/null +++ b/scripts/eval_retrieval.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python3 +"""检索质量评估脚本 + +从 metadata.db 随机抽取条款,自动生成测试 query, +调用 /api/search 接口,计算 Recall@K、MRR、NDCG 等指标。 + +用法: + python3 scripts/eval_retrieval.py [--sample 500] [--top-k 10] [--api http://localhost:8090] + +两种测试模式: +1. 结构化 query:"法规名 条号"(如"郑州市劳动用工条例 第三十二条") + → 测试精确查找模式(keyword)和语义检索模式(semantic)的命中差异 +2. 内容 query:条款内容前 50 字 + → 测试语义检索是否能通过条文片段找到原文 +""" +import argparse +import asyncio +import json +import math +import random +import sqlite3 +import sys +import time +from pathlib import Path +from typing import List, Dict, Tuple + +# 添加项目根目录到 path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import httpx + + +def build_eval_dataset(db_path: str, sample_size: int = 500) -> List[Dict]: + """从数据库随机抽取条款,构建评估数据集 + + Returns: + [{"clause_id", "law_name", "clause_no", "content", "law_id"}, ...] + """ + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + + # 随机抽取条款(排除内容过短的) + rows = conn.execute( + """ + SELECT c.id, c.law_id, c.clause_no, c.content, + l.name as law_name, l.category, l.province + FROM clauses c JOIN laws l ON c.law_id = l.id + WHERE length(c.content) > 20 AND c.clause_no IS NOT NULL + ORDER BY RANDOM() LIMIT ? + """, + (sample_size,), + ).fetchall() + conn.close() + + dataset = [] + for r in rows: + dataset.append({ + "clause_id": r["id"], + "law_id": r["law_id"], + "law_name": r["law_name"], + "clause_no": r["clause_no"], + "content": r["content"], + "category": r["category"], + "province": r["province"], + }) + return dataset + + +def generate_queries(item: Dict) -> Dict[str, str]: + """为一个条款生成多种 query + + Returns: + {"structured": "法规名 条号", "content_snippet": "条款前50字"} + """ + # 结构化 query:法规名 + 条号 + structured = f"{item['law_name']} {item['clause_no']}" + + # 内容 query:条款内容前 50 字(去掉换行) + content_clean = item["content"].replace("\n", " ").strip() + snippet = content_clean[:50] + + return { + "structured": structured, + "content_snippet": snippet, + } + + +async def search_api( + client: httpx.AsyncClient, + api_base: str, + query: str, + mode: str = "semantic", + top_k: int = 10, +) -> List[Dict]: + """调用检索 API + + Returns: + 结果列表,每项含 clause_id/law_name/clause_no/score 等 + """ + params = { + "query": query, + "mode": mode, + "top_k": top_k, + "page": 1, + "page_size": top_k, + } + try: + resp = await client.get(f"{api_base}/api/search", params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + results = data.get("data", {}).get("results", []) + # 补充 clause_id(检索 API 返回的字段名) + for r in results: + if "clause_id" not in r: + r["clause_id"] = r.get("id") + return results + except Exception as e: + print(f" [ERROR] 检索失败: {e}") + return [] + + +def compute_recall_at_k(results: List[Dict], gt_clause_id: int, k: int) -> float: + """Recall@K:ground truth 是否在 Top-K 结果中""" + top_k = results[:k] + for r in top_k: + if r.get("clause_id") == gt_clause_id: + return 1.0 + return 0.0 + + +def compute_mrr(results: List[Dict], gt_clause_id: int) -> float: + """MRR:第一个相关文档的排名倒数""" + for i, r in enumerate(results, 1): + if r.get("clause_id") == gt_clause_id: + return 1.0 / i + return 0.0 + + +def compute_ndcg_at_k(results: List[Dict], gt_clause_id: int, k: int) -> float: + """NDCG@K:二值相关性(命中=1,未命中=0)""" + dcg = 0.0 + for i, r in enumerate(results[:k], 1): + if r.get("clause_id") == gt_clause_id: + dcg = 1.0 / math.log2(i + 1) + break + # IDCG:理想情况下相关文档排第 1 + idcg = 1.0 / math.log2(2) # = 1.0 + return dcg / idcg if idcg > 0 else 0.0 + + +def compute_hit_rate(results: List[Dict], gt_clause_id: int) -> float: + """Hit Rate:是否至少命中一个相关文档""" + for r in results: + if r.get("clause_id") == gt_clause_id: + return 1.0 + return 0.0 + + +async def run_eval( + dataset: List[Dict], + api_base: str, + top_k: int = 10, + modes: List[str] = None, + query_types: List[str] = None, +) -> Dict: + """运行评估 + + Args: + dataset: 评估数据集 + api_base: API 地址 + top_k: Top-K + modes: 检索模式列表 ["semantic", "keyword"] + query_types: query 类型列表 ["structured", "content_snippet"] + + Returns: + 评估结果报告 + """ + if modes is None: + modes = ["semantic", "keyword"] + if query_types is None: + query_types = ["structured", "content_snippet"] + + report = { + "sample_size": len(dataset), + "top_k": top_k, + "modes": modes, + "query_types": query_types, + "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"), + "results": {}, + } + + async with httpx.AsyncClient() as client: + for mode in modes: + for qtype in query_types: + key = f"{mode}_{qtype}" + print(f"\n=== 评估: mode={mode}, query_type={qtype} ===") + + recalls = [] + mrrs = [] + ndcgs = [] + hit_rates = [] + latencies = [] + errors = 0 + + for i, item in enumerate(dataset): + queries = generate_queries(item) + query = queries.get(qtype, "") + if not query: + continue + + gt_id = item["clause_id"] + + t0 = time.time() + results = await search_api(client, api_base, query, mode, top_k) + latency = (time.time() - t0) * 1000 + latencies.append(latency) + + if not results: + errors += 1 + recalls.append(0.0) + mrrs.append(0.0) + ndcgs.append(0.0) + hit_rates.append(0.0) + else: + recalls.append(compute_recall_at_k(results, gt_id, top_k)) + mrrs.append(compute_mrr(results, gt_id)) + ndcgs.append(compute_ndcg_at_k(results, gt_id, top_k)) + hit_rates.append(compute_hit_rate(results, gt_id)) + + # 进度 + if (i + 1) % 50 == 0: + avg_recall = sum(recalls) / len(recalls) + print(f" 进度: {i+1}/{len(dataset)} | Recall@{top_k}={avg_recall:.3f}") + + n = len(recalls) + report["results"][key] = { + "mode": mode, + "query_type": qtype, + "count": n, + "errors": errors, + "recall_at_k": sum(recalls) / n if n else 0, + "mrr": sum(mrrs) / n if n else 0, + "ndcg_at_k": sum(ndcgs) / n if n else 0, + "hit_rate": sum(hit_rates) / n if n else 0, + "avg_latency_ms": sum(latencies) / len(latencies) if latencies else 0, + "p95_latency_ms": sorted(latencies)[int(len(latencies) * 0.95)] if latencies else 0, + } + + r = report["results"][key] + print(f" 结果: Recall@{top_k}={r['recall_at_k']:.3f} | MRR={r['mrr']:.3f} | " + f"NDCG@{top_k}={r['ndcg_at_k']:.3f} | HitRate={r['hit_rate']:.3f} | " + f"avg={r['avg_latency_ms']:.0f}ms | errors={errors}") + + return report + + +def print_report(report: Dict): + """打印评估报告""" + print("\n" + "=" * 80) + print("检索质量评估报告") + print("=" * 80) + print(f"时间: {report['timestamp']}") + print(f"样本数: {report['sample_size']}") + print(f"Top-K: {report['top_k']}") + print() + + # 表格输出 + print(f"{'模式':<12} {'Query类型':<18} {'Recall@K':>10} {'MRR':>10} {'NDCG@K':>10} {'HitRate':>10} {'avg(ms)':>10} {'错误':>6}") + print("-" * 96) + for key, r in report["results"].items(): + print(f"{r['mode']:<12} {r['query_type']:<18} " + f"{r['recall_at_k']:>10.3f} {r['mrr']:>10.3f} {r['ndcg_at_k']:>10.3f} " + f"{r['hit_rate']:>10.3f} {r['avg_latency_ms']:>10.0f} {r['errors']:>6}") + + print() + # 分析建议 + sem_struct = report["results"].get("semantic_structured", {}) + kw_struct = report["results"].get("keyword_structured", {}) + sem_content = report["results"].get("semantic_content_snippet", {}) + + print("分析:") + if sem_struct and kw_struct: + if kw_struct["recall_at_k"] > sem_struct["recall_at_k"] + 0.1: + print(f" - 结构化查询(法规名+条号):精确查找(Recall={kw_struct['recall_at_k']:.3f})" + f" 显著优于语义检索(Recall={sem_struct['recall_at_k']:.3f})") + print(f" → 建议:用户输入法规名+条号时,自动切换到精确查找模式") + else: + print(f" - 结构化查询:语义检索(Recall={sem_struct['recall_at_k']:.3f})" + f" 与精确查找(Recall={kw_struct['recall_at_k']:.3f})相当") + + if sem_content: + r = sem_content + if r["recall_at_k"] > 0.8: + print(f" - 内容片段查询:语义检索表现优秀(Recall={r['recall_at_k']:.3f})") + elif r["recall_at_k"] > 0.5: + print(f" - 内容片段查询:语义检索表现一般(Recall={r['recall_at_k']:.3f}),有提升空间") + else: + print(f" - 内容片段查询:语义检索表现较差(Recall={r['recall_at_k']:.3f}),需检查 embedding 模型") + + print() + + +def main(): + parser = argparse.ArgumentParser(description="检索质量评估") + parser.add_argument("--sample", type=int, default=500, help="抽样数量(默认 500)") + parser.add_argument("--top-k", type=int, default=10, help="Top-K(默认 10)") + parser.add_argument("--api", type=str, default="http://localhost:8090", help="API 地址") + parser.add_argument("--db", type=str, default="/data/law-kb-data/metadata.db", help="SQLite 路径") + parser.add_argument("--seed", type=int, default=42, help="随机种子(默认 42)") + parser.add_argument("--output", type=str, default=None, help="结果输出 JSON 文件路径") + parser.add_argument("--modes", type=str, nargs="+", default=["semantic", "keyword"], + help="检索模式(默认 semantic keyword)") + parser.add_argument("--query-types", type=str, nargs="+", + default=["structured", "content_snippet"], + help="query 类型(默认 structured content_snippet)") + args = parser.parse_args() + + random.seed(args.seed) + + print(f"构建评估数据集(抽样 {args.sample} 条)...") + dataset = build_eval_dataset(args.db, args.sample) + print(f"数据集大小: {len(dataset)}") + + if not dataset: + print("错误:数据集为空,检查数据库路径") + sys.exit(1) + + # 运行评估 + report = asyncio.run(run_eval( + dataset=dataset, + api_base=args.api, + top_k=args.top_k, + modes=args.modes, + query_types=args.query_types, + )) + + # 打印报告 + print_report(report) + + # 保存结果 + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + print(f"结果已保存到: {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/parse_clause.py b/scripts/parse_clause.py new file mode 100644 index 0000000..3aa121b --- /dev/null +++ b/scripts/parse_clause.py @@ -0,0 +1,266 @@ +"""法规切片解析器 — 按"第X条"切片,提取 metadata + +解析 markdown 法规文件: +- 法规名:文件名去 .md 后缀,去末尾 _YYYYMMDD +- 发布日期:文件名末尾 _YYYYMMDD +- 章节:跟踪"第X章"上下文 +- 条文:按"第X条"切片,内容聚合到下一个"第X条"/"第X章"/"第X节" +- 地方性法规:从 地方性法规区域映射.json 附加省份/市级 + +用法: + from scripts.parse_clause import parse_law_file + clauses = parse_law_file("/data/law-pack/法律/中华人民共和国民法典_20200528.md", "法律") +""" +import json +import re +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional, Dict, Any + +logger = logging.getLogger(__name__) + +# 中文数字正则(支持"第一""第二十二""第一百二十六"等) +CN_NUM = r"[一二三四五六七八九十百千零〇两]+" +# 条文开头:第X条 + 全角空格/普通空格 +CLAUSE_RE = re.compile(rf"^第({CN_NUM})条[\s\u3000]+(.*)") +# 章节开头:第X章 + 标题 +CHAPTER_RE = re.compile(rf"^第{CN_NUM}章[\s\u3000]+(.+)") +# 节开头:第X节 +SECTION_RE = re.compile(rf"^第{CN_NUM}节[\s\u3000]+(.+)") +# 编开头:第X编 +PART_RE = re.compile(rf"^第{CN_NUM}编[\s\u3000]+(.+)") +# 文件名日期后缀 +DATE_SUFFIX_RE = re.compile(r"_(\d{8})$") + + +@dataclass +class Clause: + """法规条文""" + law_name: str + category: str + chapter: Optional[str] = None # 当前章节,如"第一章 基本规定" + clause_no: str = "" # 条号,如"第一条" / "第143条" + content: str = "" # 条文内容 + publish_date: Optional[str] = None # 发布日期 YYYY-MM-DD + file_path: str = "" # 原文路径 + province: Optional[str] = None # 省份(地方性法规) + city: Optional[str] = None # 市级(地方性法规) + region_level: Optional[str] = None # 区域级别(省级/市级) + + def to_dict(self) -> Dict[str, Any]: + return { + "law_name": self.law_name, + "category": self.category, + "chapter": self.chapter, + "clause_no": self.clause_no, + "content": self.content, + "publish_date": self.publish_date, + "file_path": self.file_path, + "province": self.province, + "city": self.city, + "region_level": self.region_level, + } + + +def extract_name_and_date(filename: str) -> tuple[str, Optional[str]]: + """从文件名提取法规名和发布日期 + + Args: + filename: 文件名(含或不含 .md 后缀) + + Returns: + (法规名, 发布日期 YYYY-MM-DD 或 None) + """ + stem = Path(filename).stem + match = DATE_SUFFIX_RE.search(stem) + if match: + date_str = match.group(1) + name = stem[: match.start()] + # 格式化日期 YYYY-MM-DD + date_formatted = f"{date_str[:4]}-{date_str[4:6]}-{date_str[6:8]}" + return name, date_formatted + return stem, None + + +def load_region_mapping(mapping_path: Path) -> Dict[str, Optional[Dict]]: + """加载地方性法规区域映射 + + Returns: + {文件名: {region_name, province, level, region_id, province_id} | None} + """ + if not mapping_path.exists(): + logger.warning(f"区域映射文件不存在: {mapping_path}") + return {} + with open(mapping_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def parse_law_file( + filepath: str | Path, + category: str, + region_mapping: Optional[Dict] = None, +) -> List[Clause]: + """解析单个法规文件,按"第X条"切片 + + Args: + filepath: 法规 markdown 文件路径 + category: 法规类别(法律/行政法规/监察法规/司法解释/地方性法规) + region_mapping: 地方性法规区域映射(文件名 -> {province, city, ...}) + + Returns: + 条文列表;若文件无法识别"第X条"则返回空列表 + """ + filepath = Path(filepath) + filename = filepath.name + law_name, publish_date = extract_name_and_date(filename) + + # 地方性法规附加区域信息 + province = None + city = None + region_level = None + if category == "地方性法规" and region_mapping is not None: + info = region_mapping.get(filename) + if info is None: + # 映射为 null,跳过(调用方负责记录) + return [] + province = info.get("province") + region_level = info.get("level") + # 市级:region_name 如果不是省份名,则视为市级 + region_name = info.get("region_name") + if region_name and province and region_name != province: + city = region_name + + # 读取文件 + try: + text = filepath.read_text(encoding="utf-8") + except Exception as e: + logger.error(f"读取失败 {filepath}: {e}") + return [] + + lines = text.split("\n") + clauses: List[Clause] = [] + current_chapter: Optional[str] = None + current_clause: Optional[Clause] = None + current_content_lines: List[str] = [] + + def _flush_current(): + """将当前条文写入列表""" + nonlocal current_clause, current_content_lines + if current_clause is not None: + content = "\n".join(current_content_lines).strip() + if content: + current_clause.content = content + current_clause.chapter = current_chapter + clauses.append(current_clause) + current_clause = None + current_content_lines = [] + + for line in lines: + stripped = line.strip() + + # 跳过空行(但保留在内容中,后续 strip 处理) + if not stripped: + if current_clause is not None: + current_content_lines.append("") + continue + + # 检测章节 + chap_match = CHAPTER_RE.match(stripped) + if chap_match: + _flush_current() + current_chapter = stripped + continue + + # 检测节(不切片,只更新上下文) + if SECTION_RE.match(stripped): + _flush_current() + continue + + # 检测编(不切片) + if PART_RE.match(stripped): + _flush_current() + continue + + # 检测条文 + clause_match = CLAUSE_RE.match(stripped) + if clause_match: + _flush_current() + clause_no = f"第{clause_match.group(1)}条" + first_line = clause_match.group(2).strip() + current_clause = Clause( + law_name=law_name, + category=category, + clause_no=clause_no, + publish_date=publish_date, + file_path=str(filepath), + province=province, + city=city, + region_level=region_level, + ) + current_content_lines = [first_line] if first_line else [] + continue + + # 普通行:若在条文中,追加到内容 + if current_clause is not None: + current_content_lines.append(stripped) + + _flush_current() + return clauses + + +def scan_category_dir( + law_pack_dir: Path, + category: str, + region_mapping: Optional[Dict] = None, +) -> tuple[List[Clause], List[str]]: + """扫描某类别目录下所有法规文件 + + Args: + law_pack_dir: 法规根目录 + category: 类别(法律/行政法规/...) + region_mapping: 区域映射(仅地方性法规需要) + + Returns: + (条文列表, 跳过文件列表[null 映射或解析失败]) + """ + cat_dir = law_pack_dir / category + if not cat_dir.exists(): + logger.warning(f"类别目录不存在: {cat_dir}") + return [], [] + + all_clauses: List[Clause] = [] + skipped: List[str] = [] + + md_files = sorted(cat_dir.glob("*.md")) + for md_file in md_files: + clauses = parse_law_file(md_file, category, region_mapping) + if not clauses: + # 区分 null 映射跳过 vs 解析失败 + if category == "地方性法规" and region_mapping is not None: + info = region_mapping.get(md_file.name) + if info is None: + skipped.append(f"[null映射] {md_file.name}") + else: + skipped.append(f"[无条文] {md_file.name}") + else: + skipped.append(f"[无条文] {md_file.name}") + else: + all_clauses.extend(clauses) + + return all_clauses, skipped + + +if __name__ == "__main__": + # 自测:解析民法典 + import sys + + logging.basicConfig(level=logging.INFO) + test_file = sys.argv[1] if len(sys.argv) > 1 else None + if test_file: + clauses = parse_law_file(test_file, "法律") + print(f"解析 {test_file}: {len(clauses)} 条") + for c in clauses[:3]: + print(f" {c.clause_no} [{c.chapter}] {c.content[:50]}...") + else: + print("用法: python parse_clause.py <法规文件路径>") diff --git a/scripts/rebuild_faiss.py b/scripts/rebuild_faiss.py new file mode 100644 index 0000000..877b156 --- /dev/null +++ b/scripts/rebuild_faiss.py @@ -0,0 +1,147 @@ +"""FAISS 索引重建脚本 — 从 SQLite 读取已有条文,重新向量化 + 构建 FAISS + +用途:索引构建因 OOM/中断失败,但 SQLite metadata 已完好时, +跳过切片解析,仅重新向量化 + 构建 FAISS 索引。 + +用法: + python scripts/rebuild_faiss.py +""" +import asyncio +import logging +import sqlite3 +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import numpy as np +from app.config import ( + SQLITE_PATH, + FAISS_INDEX_PATH, + LAW_KB_DATA_DIR, + EMBEDDING_BATCH_SIZE, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", +) +logger = logging.getLogger(__name__) + + +async def rebuild(): + """从 SQLite 读取条文,重新向量化 + 构建 FAISS Flat 索引""" + start_time = time.time() + timestamp = time.strftime("%Y-%m-%d %H:%M:%S") + + logger.info("=" * 60) + logger.info(f"QYLAW FAISS 索引重建(跳过切片,从 SQLite 读取)") + logger.info(f"时间: {timestamp}") + logger.info("=" * 60) + + if not SQLITE_PATH.exists(): + logger.error(f"SQLite 不存在: {SQLITE_PATH}") + return + + conn = sqlite3.connect(str(SQLITE_PATH)) + conn.row_factory = sqlite3.Row + + # 读取所有条文(按 faiss_idx 排序,保证顺序一致) + rows = conn.execute( + "SELECT id, faiss_idx, content FROM clauses WHERE faiss_idx IS NOT NULL ORDER BY faiss_idx" + ).fetchall() + total = len(rows) + logger.info(f"从 SQLite 读取: {total} 条文") + + if total == 0: + logger.error("无条文,终止") + conn.close() + return + + clause_texts = [row["content"] for row in rows] + clause_ids = [row["id"] for row in rows] + + # 向量化(分批写入预分配的 numpy 数组,避免 Python list 内存爆炸) + logger.info(f"开始向量化(批量大小 {EMBEDDING_BATCH_SIZE})...") + from app.services.embedding import embed_batch + + embed_start = time.time() + + # 先用第一批获取向量维度,然后预分配 numpy 数组 + first_batch = clause_texts[:EMBEDDING_BATCH_SIZE] + try: + first_vecs = await embed_batch(first_batch, batch_size=EMBEDDING_BATCH_SIZE) + except Exception as e: + logger.error(f"向量化失败: {e}") + conn.close() + return + + vec_dim = len(first_vecs[0]) + logger.info(f"向量维度: {vec_dim}") + + # 预分配 numpy 数组(68万 × 1024 × 4字节 ≈ 2.6GB,一次性分配) + vecs_array = np.zeros((total, vec_dim), dtype=np.float32) + vecs_array[:len(first_vecs)] = np.array(first_vecs, dtype=np.float32) + logger.info(f"预分配 numpy 数组: {total} × {vec_dim} ({vecs_array.nbytes / 1024**3:.1f} GB)") + + # 分批向量化剩余条文,直接写入 numpy 数组 + for i in range(EMBEDDING_BATCH_SIZE, total, EMBEDDING_BATCH_SIZE): + batch = clause_texts[i : i + EMBEDDING_BATCH_SIZE] + try: + batch_vecs = await embed_batch(batch, batch_size=EMBEDDING_BATCH_SIZE) + vecs_array[i : i + len(batch_vecs)] = np.array(batch_vecs, dtype=np.float32) + except Exception as e: + logger.error(f"向量化批次 {i} 失败: {e}") + conn.close() + return + if (i // EMBEDDING_BATCH_SIZE) % 1000 == 0: + logger.info(f"进度: {i}/{total} ({i*100//total}%)") + + embed_time = time.time() - embed_start + logger.info(f"向量化完成: {total} 向量, 耗时 {embed_time:.0f}s") + + # 释放 clause_texts 内存(不再需要) + del clause_texts + + # 构建 FAISS Flat 索引(省内存,暴力检索) + logger.info("构建 FAISS Flat 索引...") + import faiss + + index = faiss.IndexFlatL2(vec_dim) + index.add(vecs_array) + logger.info(f"FAISS 索引构建完成: {index.ntotal} 向量") + + # 释放 numpy 数组(已写入 FAISS 索引) + del vecs_array + + # 原子写入 + FAISS_INDEX_PATH.parent.mkdir(parents=True, exist_ok=True) + tmp_faiss = FAISS_INDEX_PATH.with_suffix(".faiss.tmp") + faiss.write_index(index, str(tmp_faiss)) + if FAISS_INDEX_PATH.exists(): + FAISS_INDEX_PATH.unlink() + tmp_faiss.rename(FAISS_INDEX_PATH) + + index_size = FAISS_INDEX_PATH.stat().st_size / 1024 / 1024 + logger.info(f"FAISS 索引持久化: {FAISS_INDEX_PATH} ({index_size:.1f} MB)") + + conn.close() + + total_time = time.time() - start_time + print("\n" + "=" * 60) + print(f" FAISS 索引重建完成") + print(f" 总耗时: {total_time:.0f}s ({total_time/60:.1f}min)") + print(f" 向量数: {total}") + print(f" 向量维度: {vec_dim}") + print(f" 索引大小: {index_size:.1f} MB") + print(f" 向量化耗时: {embed_time:.0f}s") + print("=" * 60) + + +def main(): + asyncio.run(rebuild()) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_index.py b/scripts/verify_index.py new file mode 100644 index 0000000..da25b72 --- /dev/null +++ b/scripts/verify_index.py @@ -0,0 +1,101 @@ +"""索引验证脚本 — 预置典型查询,抽样验证检索质量 + +用法: + python scripts/verify_index.py + +输出每个查询的 Top-5 结果,供人工评估命中率。 +""" +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from app.services.retriever import Retriever +from app.services.embedding import embed_text + + +# 预置典型查询(20 个,覆盖各类别) +TEST_QUERIES = [ + # 法律 + "个人信息保护", + "个人信息跨境传输", + "竞业协议补偿金", + "民法典婚姻家庭", + "合同违约责任", + "知识产权侵权赔偿", + "公司股东权利", + # 行政法规 + "不动产登记流程", + "专利申请条件", + # 司法解释 + "公益诉讼办案规则", + "刑事诉讼证据规则", + # 监察法规 + "监察工作信息公开", + # 地方性法规 + "垃圾分类管理", + "烟花爆竹禁放", + "物业管理规定", + "生态环境保护", + "城市市容管理", + "食品安全监管", + "道路交通管理", + "未成年人保护", +] + + +async def verify(): + """执行验证""" + # 加载索引 + retriever = Retriever.get_instance() + retriever.load() + + if not retriever.is_ready(): + print("索引未加载,请先执行构建") + return + + print("=" * 70) + print(" QYLAW 索引验证") + print("=" * 70) + + hit_count = 0 + total = len(TEST_QUERIES) + + for i, query in enumerate(TEST_QUERIES, 1): + print(f"\n[{i}/{total}] 查询: {query}") + try: + query_vec = await embed_text(query) + results = retriever.search(query_vec, top_k=5) + if results: + print(f" Top-5 结果:") + for j, r in enumerate(results, 1): + score = r["score"] + law = r["law_name"] + clause = r.get("clause_no", "") + chapter = r.get("chapter", "") + content_preview = r["content"][:80].replace("\n", " ") + print(f" {j}. [{score:.4f}] {law} {clause} ({chapter})") + print(f" {content_preview}...") + # 简单命中率判断:Top-1 分数 > 0.5 视为命中 + if results[0]["score"] > 0.5: + hit_count += 1 + print(f" ✅ 命中") + else: + print(f" ⚠️ 分数偏低") + else: + print(f" ❌ 无结果") + except Exception as e: + print(f" ❌ 错误: {e}") + + print("\n" + "=" * 70) + print(f" 验证完成: {hit_count}/{total} 命中 (Top-1 score > 0.5)") + print("=" * 70) + + +def main(): + asyncio.run(verify()) + + +if __name__ == "__main__": + main() diff --git a/tests/test_parse_clause.py b/tests/test_parse_clause.py new file mode 100644 index 0000000..01b8c13 --- /dev/null +++ b/tests/test_parse_clause.py @@ -0,0 +1,98 @@ +"""切片解析单测""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from scripts.parse_clause import ( + parse_law_file, + extract_name_and_date, + load_region_mapping, + scan_category_dir, +) + + +# 测试数据路径(相对于项目根) +PACK_DIR = Path(__file__).parent.parent.parent / "law-pack-2026-07-01-markdown" + + +def test_extract_name_and_date(): + """测试文件名解析""" + name, date = extract_name_and_date("中华人民共和国民法典_20200528.md") + assert name == "中华人民共和国民法典" + assert date == "2020-05-28" + + name, date = extract_name_and_date("不动产登记暂行条例_20240310.md") + assert name == "不动产登记暂行条例" + assert date == "2024-03-10" + + # 无日期后缀 + name, date = extract_name_and_date("某法规.md") + assert name == "某法规" + assert date is None + + +def test_parse_civil_code(): + """测试民法典切片""" + f = PACK_DIR / "法律" / "中华人民共和国民法典_20200528.md" + if not f.exists(): + return # 数据不存在时跳过 + clauses = parse_law_file(f, "法律") + assert len(clauses) > 1000 # 民法典 1260 条 + assert clauses[0].clause_no == "第一条" + assert clauses[0].law_name == "中华人民共和国民法典" + assert clauses[0].category == "法律" + assert clauses[0].publish_date == "2020-05-28" + assert "保护民事主体" in clauses[0].content + + +def test_parse_admin_regulation(): + """测试行政法规切片""" + f = PACK_DIR / "行政法规" / "不动产登记暂行条例_20240310.md" + if not f.exists(): + return + clauses = parse_law_file(f, "行政法规") + assert len(clauses) > 0 + assert clauses[0].category == "行政法规" + + +def test_parse_local_regulation_with_mapping(): + """测试地方性法规 + 区域映射""" + mapping = load_region_mapping(PACK_DIR / "地方性法规区域映射.json") + f = PACK_DIR / "地方性法规" / "七台河市倭肯河流域水环境保护条例_20211224.md" + if not f.exists(): + return + clauses = parse_law_file(f, "地方性法规", mapping) + assert len(clauses) > 0 + assert clauses[0].province == "黑龙江省" + assert clauses[0].city == "七台河市" + assert clauses[0].region_level == "市级" + + +def test_null_mapping_skipped(): + """测试 null 映射文件跳过""" + mapping = load_region_mapping(PACK_DIR / "地方性法规区域映射.json") + f = PACK_DIR / "地方性法规" / "~$壮族自治区巴马盘阳河流域生态环境保护条例_20150527.md" + if not f.exists(): + return + clauses = parse_law_file(f, "地方性法规", mapping) + assert len(clauses) == 0 # null 映射应跳过 + + +if __name__ == "__main__": + # 手动运行(无 pytest 时) + tests = [ + test_extract_name_and_date, + test_parse_civil_code, + test_parse_admin_regulation, + test_parse_local_regulation_with_mapping, + test_null_mapping_skipped, + ] + for t in tests: + try: + t() + print(f"✅ {t.__name__} 通过") + except AssertionError as e: + print(f"❌ {t.__name__} 失败: {e}") + except Exception as e: + print(f"⚠️ {t.__name__} 跳过: {e}")