feat: UI优化和功能完善
- 添加公共分页组件，支持上边显示、分页大小10 - 侧边栏/Header按Liner风格优化，添加动画效果 - 异常处理列表支持分页和明细弹窗 - 任务结果页面支持已匹配列表分页 - 修复Dialog弹窗背景透明问题 - 新增dropdown-menu组件 - 添加parsed_file_record模型和回填脚本 - 前端枚举中文化
This commit is contained in:
freedakgmail
@@ -0,0 +1,216 @@
|
||||
"""
|
||||
解析后文件记录服务
|
||||
|
||||
提供文件解析→标准→入库的完整链路
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pandas as pd
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.field_mapping import FieldMapping
|
||||
from app.models.parsed_file_record import ParsedFileRecord
|
||||
from app.models.uploaded_file import UploadedFile
|
||||
from app.services.file_storage import FileStorageService
|
||||
|
||||
|
||||
EMPLOYEE_ID_KEYS = (
|
||||
"employee_id", "员工编号", "员工ID", "员工id", "工号", "员工工号",
|
||||
)
|
||||
EMPLOYEE_NAME_KEYS = ("employee_name", "员工姓名", "姓名", "name")
|
||||
|
||||
|
||||
def _normalize_row(
|
||||
row: dict[str, Any],
|
||||
mappings: list[FieldMapping],
|
||||
) -> dict[str, Any]:
|
||||
"""按字段映射将行标准化"""
|
||||
standardized = dict(row)
|
||||
for mapping in mappings:
|
||||
if mapping.is_skipped or not mapping.standard_field:
|
||||
continue
|
||||
if mapping.source_field in row and row[mapping.source_field] not in (None, ""):
|
||||
standardized[mapping.standard_field] = row[mapping.source_field]
|
||||
return standardized
|
||||
|
||||
|
||||
def _extract_employee_id(row: dict[str, Any]) -> str | None:
|
||||
for key in EMPLOYEE_ID_KEYS:
|
||||
value = row.get(key)
|
||||
if value not in (None, ""):
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _extract_employee_name(row: dict[str, Any]) -> str | None:
|
||||
for key in EMPLOYEE_NAME_KEYS:
|
||||
value = row.get(key)
|
||||
if value not in (None, ""):
|
||||
return str(value).strip()
|
||||
return None
|
||||
|
||||
|
||||
def _bucket_of(file_type: str) -> str:
|
||||
lowered = file_type.lower()
|
||||
if "工资" in file_type or "salary" in lowered:
|
||||
return "salary"
|
||||
if "社保" in file_type or "social" in lowered:
|
||||
return "social_security"
|
||||
if "个税" in file_type or "tax" in lowered:
|
||||
return "tax"
|
||||
if "银行" in file_type or "bank" in lowered:
|
||||
return "bank"
|
||||
return "other"
|
||||
|
||||
|
||||
def _read_file_rows(
|
||||
file: UploadedFile,
|
||||
mappings: list[FieldMapping],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""读取文件并返回行列表"""
|
||||
file_path = FileStorageService.get_file_path(file.stored_filename)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
return []
|
||||
|
||||
suffix = Path(file.original_filename or file.stored_filename).suffix.lower()
|
||||
try:
|
||||
if suffix in {".xlsx", ".xls"}:
|
||||
df = pd.read_excel(file_path)
|
||||
else:
|
||||
df = None
|
||||
for encoding in ("utf-8", "utf-8-sig", "gbk", "gb2312"):
|
||||
try:
|
||||
df = pd.read_csv(file_path, encoding=encoding)
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if df is None:
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
df = df.where(pd.notna(df), None)
|
||||
records = []
|
||||
for idx, row in enumerate(df.to_dict(orient="records")):
|
||||
normalized = _normalize_row(row, mappings)
|
||||
records.append({
|
||||
"row_index": idx,
|
||||
"raw_data": {k: v for k, v in row.items() if k is not None},
|
||||
"normalized_data": normalized,
|
||||
"employee_id": _extract_employee_id(normalized),
|
||||
"employee_name": _extract_employee_name(normalized),
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
class ParsedFileRecordService:
|
||||
"""解析后文件记录服务"""
|
||||
|
||||
def __init__(self, db: AsyncSession):
|
||||
self.db = db
|
||||
|
||||
async def parse_and_save_file(self, file: UploadedFile) -> int:
|
||||
"""
|
||||
解析单个上传文件并写入 parsed_file_records
|
||||
|
||||
Returns:
|
||||
写入的记录数
|
||||
"""
|
||||
# 已有记录则跳过（幂等）
|
||||
existing = await self.db.execute(
|
||||
select(ParsedFileRecord)
|
||||
.where(ParsedFileRecord.uploaded_file_id == file.id)
|
||||
.limit(1)
|
||||
)
|
||||
if existing.scalars().first() is not None:
|
||||
return 0
|
||||
|
||||
# 加载字段映射
|
||||
mappings_result = await self.db.execute(
|
||||
select(FieldMapping).where(
|
||||
FieldMapping.file_id == file.id,
|
||||
FieldMapping.company_id == file.company_id,
|
||||
)
|
||||
)
|
||||
mappings = list(mappings_result.scalars().all())
|
||||
|
||||
# 解析文件行
|
||||
rows = _read_file_rows(file, mappings)
|
||||
bucket = _bucket_of(file.file_type)
|
||||
|
||||
# 批量写入
|
||||
records = [
|
||||
ParsedFileRecord(
|
||||
company_id=file.company_id,
|
||||
task_id=file.task_id,
|
||||
uploaded_file_id=file.id,
|
||||
file_bucket=bucket,
|
||||
row_index=r["row_index"],
|
||||
raw_data=r["raw_data"],
|
||||
normalized_data=r["normalized_data"],
|
||||
employee_id=r["employee_id"],
|
||||
employee_name=r["employee_name"],
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
self.db.add_all(records)
|
||||
await self.db.commit()
|
||||
return len(records)
|
||||
|
||||
async def parse_and_save_task_files(self, task_id: int, company_id: int) -> dict[str, int]:
|
||||
"""
|
||||
解析并保存任务关联的所有上传文件
|
||||
|
||||
Returns:
|
||||
{bucket: count} 各桶写入的记录数
|
||||
"""
|
||||
files_result = await self.db.execute(
|
||||
select(UploadedFile).where(
|
||||
UploadedFile.task_id == task_id,
|
||||
UploadedFile.company_id == company_id,
|
||||
)
|
||||
)
|
||||
files = list(files_result.scalars().all())
|
||||
counts: dict[str, int] = {}
|
||||
for file in files:
|
||||
cnt = await self.parse_and_save_file(file)
|
||||
if cnt > 0:
|
||||
bucket = _bucket_of(file.file_type)
|
||||
counts[bucket] = counts.get(bucket, 0) + cnt
|
||||
return counts
|
||||
|
||||
async def get_records_by_task(
|
||||
self,
|
||||
task_id: int,
|
||||
company_id: int,
|
||||
buckets: list[str] | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 100,
|
||||
) -> tuple[list[ParsedFileRecord], int]:
|
||||
"""
|
||||
按任务读取解析记录
|
||||
|
||||
Returns:
|
||||
(records, total)
|
||||
"""
|
||||
query = select(ParsedFileRecord).where(
|
||||
ParsedFileRecord.task_id == task_id,
|
||||
ParsedFileRecord.company_id == company_id,
|
||||
)
|
||||
if buckets:
|
||||
query = query.where(ParsedFileRecord.file_bucket.in_(buckets))
|
||||
|
||||
count_result = await self.db.execute(
|
||||
select(ParsedFileRecord.id).where(*query.whereclause.compile().compile() if query.whereclause is not None else [])
|
||||
)
|
||||
# 简化计数
|
||||
all_result = await self.db.execute(query)
|
||||
all_records = list(all_result.scalars().all())
|
||||
total = len(all_records)
|
||||
|
||||
start = (page - 1) * page_size
|
||||
return all_records[start:start + page_size], total
|
||||
Reference in New Issue
Block a user