freedakgmail
33b4c734aa
后端: - 新增认证(auth)、任务(tasks)、映射(mappings)、对账(reconciliation)、异常(exceptions)、导出(exports) API - 新增核心模块: database, security, permissions, tenant, exceptions, error_handlers - 新增数据模型: user, company, reconciliation_task, field_mapping, uploaded_file 等 - 新增服务层: ai_recognizer, file_parser, file_storage, mapping, reconciliation 等 - 添加数据库迁移脚本 前端: - 新增登录页面和仪表盘页面 - 新增任务列表、任务详情、字段映射页面 - 新增异常处理页面和规则设置页面 - 新增 API 代理路由 /api/[...path] - 新增 UI 组件库 (button, card, dialog, input, table 等) - 新增 auth 组件 (ProtectedRoute, PermissionGate) - 新增 layout 组件 (Header, Sidebar) - 新增 mapping 组件 (FieldMappingTable, AISuggestionPanel) - 新增 API 客户端和 hooks (useAsync, useToast, usePermission 等) - 新增状态管理 (auth-store, company-store, ui-store) - 集成 Tailwind CSS 和 shadcn/ui 组件库 其他: - 添加 Alembic 数据库迁移配置 - 添加初始化示例数据脚本 - 更新项目文档
399 lines
12 KiB
Python
399 lines
12 KiB
Python
"""
|
||
数据清洗服务
|
||
|
||
清洗和标准化工资表数据，为对账做准备
|
||
"""
|
||
|
||
import re
|
||
from datetime import datetime
|
||
from decimal import Decimal, InvalidOperation
|
||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||
from dataclasses import dataclass
|
||
|
||
|
||
@dataclass
|
||
class CleaningResult:
|
||
"""清洗结果"""
|
||
original_value: Any
|
||
cleaned_value: Any
|
||
is_valid: bool
|
||
error_message: Optional[str] = None
|
||
|
||
|
||
@dataclass
|
||
class ValidationRule:
|
||
"""验证规则"""
|
||
name: str
|
||
validate: Callable[[Any], bool]
|
||
error_message: str
|
||
|
||
|
||
class DataCleaner:
|
||
"""
|
||
数据清洗器
|
||
|
||
用于清洗和验证工资表数据
|
||
"""
|
||
|
||
def __init__(self):
|
||
self.validation_rules: Dict[str, List[ValidationRule]] = {}
|
||
self._init_default_rules()
|
||
|
||
def _init_default_rules(self):
|
||
"""初始化默认验证规则"""
|
||
# 姓名验证：2-20个中文字符或英文字母
|
||
self.add_rule(
|
||
"employee_name",
|
||
ValidationRule(
|
||
name="name_length",
|
||
validate=lambda v: bool(v) and 1 < len(str(v).strip()) <= 50,
|
||
error_message="姓名长度应在2-50个字符之间"
|
||
)
|
||
)
|
||
|
||
# 工号验证：数字或字母组合
|
||
self.add_rule(
|
||
"employee_id",
|
||
ValidationRule(
|
||
name="id_format",
|
||
validate=lambda v: bool(v) and bool(re.match(r'^[\w\-]+$', str(v))),
|
||
error_message="工号格式不正确"
|
||
)
|
||
)
|
||
|
||
# 金额验证：正数且在合理范围内
|
||
self.add_rule(
|
||
"amount",
|
||
ValidationRule(
|
||
name="amount_positive",
|
||
validate=lambda v: self._parse_number(v) is not None and self._parse_number(v) >= 0,
|
||
error_message="金额必须为非负数"
|
||
)
|
||
)
|
||
|
||
# 身份证号验证
|
||
self.add_rule(
|
||
"id_card",
|
||
ValidationRule(
|
||
name="id_card_length",
|
||
validate=lambda v: bool(v) and len(str(v).strip()) in [15, 18],
|
||
error_message="身份证号长度应为15或18位"
|
||
)
|
||
)
|
||
|
||
# 银行账号验证
|
||
self.add_rule(
|
||
"bank_card",
|
||
ValidationRule(
|
||
name="bank_card_digits",
|
||
validate=lambda v: bool(v) and str(v).isdigit() and len(str(v)) >= 10,
|
||
error_message="银行账号应为至少10位数字"
|
||
)
|
||
)
|
||
|
||
def add_rule(self, field_type: str, rule: ValidationRule):
|
||
"""添加验证规则"""
|
||
if field_type not in self.validation_rules:
|
||
self.validation_rules[field_type] = []
|
||
self.validation_rules[field_type].append(rule)
|
||
|
||
def _parse_number(self, value: Any) -> Optional[Decimal]:
|
||
"""解析数字"""
|
||
if value is None:
|
||
return None
|
||
|
||
if isinstance(value, (int, float, Decimal)):
|
||
return Decimal(str(value))
|
||
|
||
if isinstance(value, str):
|
||
# 移除常见的货币符号、逗号和空格
|
||
cleaned = value.strip()
|
||
cleaned = re.sub(r'[¥$,，\s]', '', cleaned)
|
||
|
||
# 处理负数
|
||
is_negative = cleaned.startswith('-')
|
||
cleaned = cleaned.lstrip('-')
|
||
|
||
# 检查是否包含有效数字
|
||
if not cleaned or not re.match(r'^\d+(\.\d+)?$', cleaned):
|
||
return None
|
||
|
||
try:
|
||
result = Decimal(cleaned)
|
||
return -result if is_negative else result
|
||
except InvalidOperation:
|
||
return None
|
||
|
||
return None
|
||
|
||
def _parse_date(self, value: Any) -> Optional[datetime]:
|
||
"""解析日期"""
|
||
if value is None:
|
||
return None
|
||
|
||
if isinstance(value, datetime):
|
||
return value
|
||
|
||
if isinstance(value, str):
|
||
# 尝试多种日期格式
|
||
formats = [
|
||
'%Y-%m-%d',
|
||
'%Y/%m/%d',
|
||
'%Y年%m月%d日',
|
||
'%Y%m%d',
|
||
'%Y-%m-%d %H:%M:%S',
|
||
]
|
||
|
||
for fmt in formats:
|
||
try:
|
||
return datetime.strptime(value.strip(), fmt)
|
||
except ValueError:
|
||
continue
|
||
|
||
return None
|
||
|
||
def clean_string(self, value: Any, strip: bool = True) -> str:
|
||
"""清洗字符串"""
|
||
if value is None:
|
||
return ""
|
||
|
||
result = str(value)
|
||
if strip:
|
||
result = result.strip()
|
||
return result
|
||
|
||
def clean_amount(self, value: Any) -> CleaningResult:
|
||
"""清洗金额字段"""
|
||
original = value
|
||
parsed = self._parse_number(value)
|
||
|
||
if parsed is None:
|
||
return CleaningResult(
|
||
original_value=original,
|
||
cleaned_value=None,
|
||
is_valid=False,
|
||
error_message="无法解析为数字"
|
||
)
|
||
|
||
return CleaningResult(
|
||
original_value=original,
|
||
cleaned_value=float(parsed),
|
||
is_valid=True
|
||
)
|
||
|
||
def clean_employee_name(self, value: Any) -> CleaningResult:
|
||
"""清洗员工姓名"""
|
||
cleaned = self.clean_string(value)
|
||
|
||
if not cleaned:
|
||
return CleaningResult(
|
||
original_value=value,
|
||
cleaned_value=None,
|
||
is_valid=False,
|
||
error_message="姓名为空"
|
||
)
|
||
|
||
# 移除多余空格
|
||
cleaned = re.sub(r'\s+', ' ', cleaned)
|
||
|
||
return CleaningResult(
|
||
original_value=value,
|
||
cleaned_value=cleaned,
|
||
is_valid=True
|
||
)
|
||
|
||
def clean_employee_id(self, value: Any) -> CleaningResult:
|
||
"""清洗工号"""
|
||
cleaned = self.clean_string(value)
|
||
|
||
if not cleaned:
|
||
return CleaningResult(
|
||
original_value=value,
|
||
cleaned_value=None,
|
||
is_valid=False,
|
||
error_message="工号为空"
|
||
)
|
||
|
||
return CleaningResult(
|
||
original_value=value,
|
||
cleaned_value=cleaned.upper(), # 统一大写
|
||
is_valid=True
|
||
)
|
||
|
||
def clean_bank_card(self, value: Any) -> CleaningResult:
|
||
"""清洗银行账号"""
|
||
original = value
|
||
cleaned = self.clean_string(value)
|
||
|
||
# 只保留数字
|
||
cleaned = re.sub(r'\D', '', cleaned)
|
||
|
||
if len(cleaned) < 10:
|
||
return CleaningResult(
|
||
original_value=original,
|
||
cleaned_value=None,
|
||
is_valid=False,
|
||
error_message="银行账号长度不足"
|
||
)
|
||
|
||
return CleaningResult(
|
||
original_value=original,
|
||
cleaned_value=cleaned,
|
||
is_valid=True
|
||
)
|
||
|
||
def clean_id_card(self, value: Any) -> CleaningResult:
|
||
"""清洗身份证号"""
|
||
original = value
|
||
cleaned = self.clean_string(value)
|
||
|
||
# 统一大写
|
||
cleaned = cleaned.upper()
|
||
# 只保留数字和X
|
||
cleaned = re.sub(r'[^0-9X]', '', cleaned)
|
||
|
||
if len(cleaned) not in [15, 18]:
|
||
return CleaningResult(
|
||
original_value=original,
|
||
cleaned_value=None,
|
||
is_valid=False,
|
||
error_message="身份证号长度应为15或18位"
|
||
)
|
||
|
||
return CleaningResult(
|
||
original_value=original,
|
||
cleaned_value=cleaned,
|
||
is_valid=True
|
||
)
|
||
|
||
def validate_field(self, field_type: str, value: Any) -> Tuple[bool, Optional[str]]:
|
||
"""
|
||
验证字段
|
||
|
||
Returns:
|
||
(是否通过, 错误信息)
|
||
"""
|
||
rules = self.validation_rules.get(field_type, [])
|
||
|
||
for rule in rules:
|
||
if not rule.validate(value):
|
||
return False, rule.error_message
|
||
|
||
return True, None
|
||
|
||
def clean_row(self, row: Dict[str, Any], field_mappings: Dict[str, str]) -> Dict[str, Any]:
|
||
"""
|
||
清洗一行数据
|
||
|
||
Args:
|
||
row: 原始行数据 {源字段名: 值}
|
||
field_mappings: 字段映射 {标准字段名: 源字段名}
|
||
|
||
Returns:
|
||
清洗后的数据 {标准字段名: 清洗后的值}
|
||
"""
|
||
cleaned = {}
|
||
errors = {}
|
||
|
||
# 反转映射：标准字段 -> 源字段
|
||
standard_to_source = {v: k for k, v in field_mappings.items()}
|
||
|
||
for standard_field, source_field in standard_to_source.items():
|
||
raw_value = row.get(source_field)
|
||
|
||
# 根据字段类型选择清洗方法
|
||
if standard_field in ["员工姓名", "姓名"]:
|
||
result = self.clean_employee_name(raw_value)
|
||
elif standard_field in ["工号", "员工号"]:
|
||
result = self.clean_employee_id(raw_value)
|
||
elif standard_field in ["银行账号", "卡号"]:
|
||
result = self.clean_bank_card(raw_value)
|
||
elif standard_field in ["身份证号", "证件号"]:
|
||
result = self.clean_id_card(raw_value)
|
||
elif standard_field in ["基本工资", "奖金", "补贴", "加班费", "扣款",
|
||
"应发工资", "实发工资", "社保基数",
|
||
"养老保险", "医疗保险", "失业保险", "公积金",
|
||
"个税", "应税收入", "税后收入"]:
|
||
result = self.clean_amount(raw_value)
|
||
else:
|
||
result = CleaningResult(
|
||
original_value=raw_value,
|
||
cleaned_value=self.clean_string(raw_value),
|
||
is_valid=bool(raw_value)
|
||
)
|
||
|
||
cleaned[standard_field] = result.cleaned_value
|
||
|
||
if not result.is_valid:
|
||
errors[standard_field] = result.error_message
|
||
|
||
return cleaned
|
||
|
||
def batch_clean(
|
||
self,
|
||
data: List[Dict[str, Any]],
|
||
field_mappings: Dict[str, str]
|
||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
||
"""
|
||
批量清洗数据
|
||
|
||
Returns:
|
||
(有效数据, 无效数据)
|
||
"""
|
||
valid_data = []
|
||
invalid_data = []
|
||
|
||
for row in data:
|
||
cleaned_row = self.clean_row(row, field_mappings)
|
||
|
||
# 检查是否有无效字段
|
||
has_invalid = any(v is None for v in cleaned_row.values())
|
||
|
||
if has_invalid:
|
||
invalid_data.append({
|
||
"original": row,
|
||
"cleaned": cleaned_row
|
||
})
|
||
else:
|
||
valid_data.append(cleaned_row)
|
||
|
||
return valid_data, invalid_data
|
||
|
||
|
||
# 单例实例
|
||
_cleaner_instance: Optional[DataCleaner] = None
|
||
|
||
|
||
def get_cleaner() -> DataCleaner:
|
||
"""获取数据清洗器单例"""
|
||
global _cleaner_instance
|
||
if _cleaner_instance is None:
|
||
_cleaner_instance = DataCleaner()
|
||
return _cleaner_instance
|
||
|
||
|
||
async def clean_salary_data(
|
||
data: List[Dict[str, Any]],
|
||
field_mappings: Dict[str, str]
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
清洗工资数据
|
||
|
||
Args:
|
||
data: 原始工资数据
|
||
field_mappings: 字段映射
|
||
|
||
Returns:
|
||
清洗结果
|
||
"""
|
||
cleaner = get_cleaner()
|
||
valid_data, invalid_data = cleaner.batch_clean(data, field_mappings)
|
||
|
||
return {
|
||
"total_count": len(data),
|
||
"valid_count": len(valid_data),
|
||
"invalid_count": len(invalid_data),
|
||
"valid_data": valid_data,
|
||
"invalid_data": invalid_data,
|
||
"cleaning_time": datetime.utcnow().isoformat(),
|
||
} |