feat: 完善前后端核心功能模块
后端: - 新增认证(auth)、任务(tasks)、映射(mappings)、对账(reconciliation)、异常(exceptions)、导出(exports) API - 新增核心模块: database, security, permissions, tenant, exceptions, error_handlers - 新增数据模型: user, company, reconciliation_task, field_mapping, uploaded_file 等 - 新增服务层: ai_recognizer, file_parser, file_storage, mapping, reconciliation 等 - 添加数据库迁移脚本 前端: - 新增登录页面和仪表盘页面 - 新增任务列表、任务详情、字段映射页面 - 新增异常处理页面和规则设置页面 - 新增 API 代理路由 /api/[...path] - 新增 UI 组件库 (button, card, dialog, input, table 等) - 新增 auth 组件 (ProtectedRoute, PermissionGate) - 新增 layout 组件 (Header, Sidebar) - 新增 mapping 组件 (FieldMappingTable, AISuggestionPanel) - 新增 API 客户端和 hooks (useAsync, useToast, usePermission 等) - 新增状态管理 (auth-store, company-store, ui-store) - 集成 Tailwind CSS 和 shadcn/ui 组件库 其他: - 添加 Alembic 数据库迁移配置 - 添加初始化示例数据脚本 - 更新项目文档
This commit is contained in:
freedakgmail
@@ -0,0 +1,399 @@
|
||||
"""
|
||||
数据清洗服务
|
||||
|
||||
清洗和标准化工资表数据，为对账做准备
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class CleaningResult:
|
||||
"""清洗结果"""
|
||||
original_value: Any
|
||||
cleaned_value: Any
|
||||
is_valid: bool
|
||||
error_message: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationRule:
|
||||
"""验证规则"""
|
||||
name: str
|
||||
validate: Callable[[Any], bool]
|
||||
error_message: str
|
||||
|
||||
|
||||
class DataCleaner:
|
||||
"""
|
||||
数据清洗器
|
||||
|
||||
用于清洗和验证工资表数据
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.validation_rules: Dict[str, List[ValidationRule]] = {}
|
||||
self._init_default_rules()
|
||||
|
||||
def _init_default_rules(self):
|
||||
"""初始化默认验证规则"""
|
||||
# 姓名验证：2-20个中文字符或英文字母
|
||||
self.add_rule(
|
||||
"employee_name",
|
||||
ValidationRule(
|
||||
name="name_length",
|
||||
validate=lambda v: bool(v) and 1 < len(str(v).strip()) <= 50,
|
||||
error_message="姓名长度应在2-50个字符之间"
|
||||
)
|
||||
)
|
||||
|
||||
# 工号验证：数字或字母组合
|
||||
self.add_rule(
|
||||
"employee_id",
|
||||
ValidationRule(
|
||||
name="id_format",
|
||||
validate=lambda v: bool(v) and bool(re.match(r'^[\w\-]+$', str(v))),
|
||||
error_message="工号格式不正确"
|
||||
)
|
||||
)
|
||||
|
||||
# 金额验证：正数且在合理范围内
|
||||
self.add_rule(
|
||||
"amount",
|
||||
ValidationRule(
|
||||
name="amount_positive",
|
||||
validate=lambda v: self._parse_number(v) is not None and self._parse_number(v) >= 0,
|
||||
error_message="金额必须为非负数"
|
||||
)
|
||||
)
|
||||
|
||||
# 身份证号验证
|
||||
self.add_rule(
|
||||
"id_card",
|
||||
ValidationRule(
|
||||
name="id_card_length",
|
||||
validate=lambda v: bool(v) and len(str(v).strip()) in [15, 18],
|
||||
error_message="身份证号长度应为15或18位"
|
||||
)
|
||||
)
|
||||
|
||||
# 银行账号验证
|
||||
self.add_rule(
|
||||
"bank_card",
|
||||
ValidationRule(
|
||||
name="bank_card_digits",
|
||||
validate=lambda v: bool(v) and str(v).isdigit() and len(str(v)) >= 10,
|
||||
error_message="银行账号应为至少10位数字"
|
||||
)
|
||||
)
|
||||
|
||||
def add_rule(self, field_type: str, rule: ValidationRule):
|
||||
"""添加验证规则"""
|
||||
if field_type not in self.validation_rules:
|
||||
self.validation_rules[field_type] = []
|
||||
self.validation_rules[field_type].append(rule)
|
||||
|
||||
def _parse_number(self, value: Any) -> Optional[Decimal]:
|
||||
"""解析数字"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return Decimal(str(value))
|
||||
|
||||
if isinstance(value, str):
|
||||
# 移除常见的货币符号、逗号和空格
|
||||
cleaned = value.strip()
|
||||
cleaned = re.sub(r'[¥$,，\s]', '', cleaned)
|
||||
|
||||
# 处理负数
|
||||
is_negative = cleaned.startswith('-')
|
||||
cleaned = cleaned.lstrip('-')
|
||||
|
||||
# 检查是否包含有效数字
|
||||
if not cleaned or not re.match(r'^\d+(\.\d+)?$', cleaned):
|
||||
return None
|
||||
|
||||
try:
|
||||
result = Decimal(cleaned)
|
||||
return -result if is_negative else result
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def _parse_date(self, value: Any) -> Optional[datetime]:
|
||||
"""解析日期"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
|
||||
if isinstance(value, str):
|
||||
# 尝试多种日期格式
|
||||
formats = [
|
||||
'%Y-%m-%d',
|
||||
'%Y/%m/%d',
|
||||
'%Y年%m月%d日',
|
||||
'%Y%m%d',
|
||||
'%Y-%m-%d %H:%M:%S',
|
||||
]
|
||||
|
||||
for fmt in formats:
|
||||
try:
|
||||
return datetime.strptime(value.strip(), fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
def clean_string(self, value: Any, strip: bool = True) -> str:
|
||||
"""清洗字符串"""
|
||||
if value is None:
|
||||
return ""
|
||||
|
||||
result = str(value)
|
||||
if strip:
|
||||
result = result.strip()
|
||||
return result
|
||||
|
||||
def clean_amount(self, value: Any) -> CleaningResult:
|
||||
"""清洗金额字段"""
|
||||
original = value
|
||||
parsed = self._parse_number(value)
|
||||
|
||||
if parsed is None:
|
||||
return CleaningResult(
|
||||
original_value=original,
|
||||
cleaned_value=None,
|
||||
is_valid=False,
|
||||
error_message="无法解析为数字"
|
||||
)
|
||||
|
||||
return CleaningResult(
|
||||
original_value=original,
|
||||
cleaned_value=float(parsed),
|
||||
is_valid=True
|
||||
)
|
||||
|
||||
def clean_employee_name(self, value: Any) -> CleaningResult:
|
||||
"""清洗员工姓名"""
|
||||
cleaned = self.clean_string(value)
|
||||
|
||||
if not cleaned:
|
||||
return CleaningResult(
|
||||
original_value=value,
|
||||
cleaned_value=None,
|
||||
is_valid=False,
|
||||
error_message="姓名为空"
|
||||
)
|
||||
|
||||
# 移除多余空格
|
||||
cleaned = re.sub(r'\s+', ' ', cleaned)
|
||||
|
||||
return CleaningResult(
|
||||
original_value=value,
|
||||
cleaned_value=cleaned,
|
||||
is_valid=True
|
||||
)
|
||||
|
||||
def clean_employee_id(self, value: Any) -> CleaningResult:
|
||||
"""清洗工号"""
|
||||
cleaned = self.clean_string(value)
|
||||
|
||||
if not cleaned:
|
||||
return CleaningResult(
|
||||
original_value=value,
|
||||
cleaned_value=None,
|
||||
is_valid=False,
|
||||
error_message="工号为空"
|
||||
)
|
||||
|
||||
return CleaningResult(
|
||||
original_value=value,
|
||||
cleaned_value=cleaned.upper(), # 统一大写
|
||||
is_valid=True
|
||||
)
|
||||
|
||||
def clean_bank_card(self, value: Any) -> CleaningResult:
|
||||
"""清洗银行账号"""
|
||||
original = value
|
||||
cleaned = self.clean_string(value)
|
||||
|
||||
# 只保留数字
|
||||
cleaned = re.sub(r'\D', '', cleaned)
|
||||
|
||||
if len(cleaned) < 10:
|
||||
return CleaningResult(
|
||||
original_value=original,
|
||||
cleaned_value=None,
|
||||
is_valid=False,
|
||||
error_message="银行账号长度不足"
|
||||
)
|
||||
|
||||
return CleaningResult(
|
||||
original_value=original,
|
||||
cleaned_value=cleaned,
|
||||
is_valid=True
|
||||
)
|
||||
|
||||
def clean_id_card(self, value: Any) -> CleaningResult:
|
||||
"""清洗身份证号"""
|
||||
original = value
|
||||
cleaned = self.clean_string(value)
|
||||
|
||||
# 统一大写
|
||||
cleaned = cleaned.upper()
|
||||
# 只保留数字和X
|
||||
cleaned = re.sub(r'[^0-9X]', '', cleaned)
|
||||
|
||||
if len(cleaned) not in [15, 18]:
|
||||
return CleaningResult(
|
||||
original_value=original,
|
||||
cleaned_value=None,
|
||||
is_valid=False,
|
||||
error_message="身份证号长度应为15或18位"
|
||||
)
|
||||
|
||||
return CleaningResult(
|
||||
original_value=original,
|
||||
cleaned_value=cleaned,
|
||||
is_valid=True
|
||||
)
|
||||
|
||||
def validate_field(self, field_type: str, value: Any) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
验证字段
|
||||
|
||||
Returns:
|
||||
(是否通过, 错误信息)
|
||||
"""
|
||||
rules = self.validation_rules.get(field_type, [])
|
||||
|
||||
for rule in rules:
|
||||
if not rule.validate(value):
|
||||
return False, rule.error_message
|
||||
|
||||
return True, None
|
||||
|
||||
def clean_row(self, row: Dict[str, Any], field_mappings: Dict[str, str]) -> Dict[str, Any]:
|
||||
"""
|
||||
清洗一行数据
|
||||
|
||||
Args:
|
||||
row: 原始行数据 {源字段名: 值}
|
||||
field_mappings: 字段映射 {标准字段名: 源字段名}
|
||||
|
||||
Returns:
|
||||
清洗后的数据 {标准字段名: 清洗后的值}
|
||||
"""
|
||||
cleaned = {}
|
||||
errors = {}
|
||||
|
||||
# 反转映射：标准字段 -> 源字段
|
||||
standard_to_source = {v: k for k, v in field_mappings.items()}
|
||||
|
||||
for standard_field, source_field in standard_to_source.items():
|
||||
raw_value = row.get(source_field)
|
||||
|
||||
# 根据字段类型选择清洗方法
|
||||
if standard_field in ["员工姓名", "姓名"]:
|
||||
result = self.clean_employee_name(raw_value)
|
||||
elif standard_field in ["工号", "员工号"]:
|
||||
result = self.clean_employee_id(raw_value)
|
||||
elif standard_field in ["银行账号", "卡号"]:
|
||||
result = self.clean_bank_card(raw_value)
|
||||
elif standard_field in ["身份证号", "证件号"]:
|
||||
result = self.clean_id_card(raw_value)
|
||||
elif standard_field in ["基本工资", "奖金", "补贴", "加班费", "扣款",
|
||||
"应发工资", "实发工资", "社保基数",
|
||||
"养老保险", "医疗保险", "失业保险", "公积金",
|
||||
"个税", "应税收入", "税后收入"]:
|
||||
result = self.clean_amount(raw_value)
|
||||
else:
|
||||
result = CleaningResult(
|
||||
original_value=raw_value,
|
||||
cleaned_value=self.clean_string(raw_value),
|
||||
is_valid=bool(raw_value)
|
||||
)
|
||||
|
||||
cleaned[standard_field] = result.cleaned_value
|
||||
|
||||
if not result.is_valid:
|
||||
errors[standard_field] = result.error_message
|
||||
|
||||
return cleaned
|
||||
|
||||
def batch_clean(
|
||||
self,
|
||||
data: List[Dict[str, Any]],
|
||||
field_mappings: Dict[str, str]
|
||||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||||
"""
|
||||
批量清洗数据
|
||||
|
||||
Returns:
|
||||
(有效数据, 无效数据)
|
||||
"""
|
||||
valid_data = []
|
||||
invalid_data = []
|
||||
|
||||
for row in data:
|
||||
cleaned_row = self.clean_row(row, field_mappings)
|
||||
|
||||
# 检查是否有无效字段
|
||||
has_invalid = any(v is None for v in cleaned_row.values())
|
||||
|
||||
if has_invalid:
|
||||
invalid_data.append({
|
||||
"original": row,
|
||||
"cleaned": cleaned_row
|
||||
})
|
||||
else:
|
||||
valid_data.append(cleaned_row)
|
||||
|
||||
return valid_data, invalid_data
|
||||
|
||||
|
||||
# 单例实例
|
||||
_cleaner_instance: Optional[DataCleaner] = None
|
||||
|
||||
|
||||
def get_cleaner() -> DataCleaner:
|
||||
"""获取数据清洗器单例"""
|
||||
global _cleaner_instance
|
||||
if _cleaner_instance is None:
|
||||
_cleaner_instance = DataCleaner()
|
||||
return _cleaner_instance
|
||||
|
||||
|
||||
async def clean_salary_data(
|
||||
data: List[Dict[str, Any]],
|
||||
field_mappings: Dict[str, str]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
清洗工资数据
|
||||
|
||||
Args:
|
||||
data: 原始工资数据
|
||||
field_mappings: 字段映射
|
||||
|
||||
Returns:
|
||||
清洗结果
|
||||
"""
|
||||
cleaner = get_cleaner()
|
||||
valid_data, invalid_data = cleaner.batch_clean(data, field_mappings)
|
||||
|
||||
return {
|
||||
"total_count": len(data),
|
||||
"valid_count": len(valid_data),
|
||||
"invalid_count": len(invalid_data),
|
||||
"valid_data": valid_data,
|
||||
"invalid_data": invalid_data,
|
||||
"cleaning_time": datetime.utcnow().isoformat(),
|
||||
}
|
||||
Reference in New Issue
Block a user