""" 数据清洗服务 清洗和标准化工资表数据,为对账做准备 """ import re from datetime import datetime from decimal import Decimal, InvalidOperation from typing import Any, Callable, Dict, List, Optional, Tuple, Union from dataclasses import dataclass @dataclass class CleaningResult: """清洗结果""" original_value: Any cleaned_value: Any is_valid: bool error_message: Optional[str] = None @dataclass class ValidationRule: """验证规则""" name: str validate: Callable[[Any], bool] error_message: str class DataCleaner: """ 数据清洗器 用于清洗和验证工资表数据 """ def __init__(self): self.validation_rules: Dict[str, List[ValidationRule]] = {} self._init_default_rules() def _init_default_rules(self): """初始化默认验证规则""" # 姓名验证:2-20个中文字符或英文字母 self.add_rule( "employee_name", ValidationRule( name="name_length", validate=lambda v: bool(v) and 1 < len(str(v).strip()) <= 50, error_message="姓名长度应在2-50个字符之间" ) ) # 工号验证:数字或字母组合 self.add_rule( "employee_id", ValidationRule( name="id_format", validate=lambda v: bool(v) and bool(re.match(r'^[\w\-]+$', str(v))), error_message="工号格式不正确" ) ) # 金额验证:正数且在合理范围内 self.add_rule( "amount", ValidationRule( name="amount_positive", validate=lambda v: self._parse_number(v) is not None and self._parse_number(v) >= 0, error_message="金额必须为非负数" ) ) # 身份证号验证 self.add_rule( "id_card", ValidationRule( name="id_card_length", validate=lambda v: bool(v) and len(str(v).strip()) in [15, 18], error_message="身份证号长度应为15或18位" ) ) # 银行账号验证 self.add_rule( "bank_card", ValidationRule( name="bank_card_digits", validate=lambda v: bool(v) and str(v).isdigit() and len(str(v)) >= 10, error_message="银行账号应为至少10位数字" ) ) def add_rule(self, field_type: str, rule: ValidationRule): """添加验证规则""" if field_type not in self.validation_rules: self.validation_rules[field_type] = [] self.validation_rules[field_type].append(rule) def _parse_number(self, value: Any) -> Optional[Decimal]: """解析数字""" if value is None: return None if isinstance(value, (int, float, Decimal)): return Decimal(str(value)) if isinstance(value, str): # 移除常见的货币符号、逗号和空格 cleaned = value.strip() cleaned = re.sub(r'[¥$,,\s]', '', cleaned) # 处理负数 is_negative = cleaned.startswith('-') cleaned = cleaned.lstrip('-') # 检查是否包含有效数字 if not cleaned or not re.match(r'^\d+(\.\d+)?$', cleaned): return None try: result = Decimal(cleaned) return -result if is_negative else result except InvalidOperation: return None return None def _parse_date(self, value: Any) -> Optional[datetime]: """解析日期""" if value is None: return None if isinstance(value, datetime): return value if isinstance(value, str): # 尝试多种日期格式 formats = [ '%Y-%m-%d', '%Y/%m/%d', '%Y年%m月%d日', '%Y%m%d', '%Y-%m-%d %H:%M:%S', ] for fmt in formats: try: return datetime.strptime(value.strip(), fmt) except ValueError: continue return None def clean_string(self, value: Any, strip: bool = True) -> str: """清洗字符串""" if value is None: return "" result = str(value) if strip: result = result.strip() return result def clean_amount(self, value: Any) -> CleaningResult: """清洗金额字段""" original = value parsed = self._parse_number(value) if parsed is None: return CleaningResult( original_value=original, cleaned_value=None, is_valid=False, error_message="无法解析为数字" ) return CleaningResult( original_value=original, cleaned_value=float(parsed), is_valid=True ) def clean_employee_name(self, value: Any) -> CleaningResult: """清洗员工姓名""" cleaned = self.clean_string(value) if not cleaned: return CleaningResult( original_value=value, cleaned_value=None, is_valid=False, error_message="姓名为空" ) # 移除多余空格 cleaned = re.sub(r'\s+', ' ', cleaned) return CleaningResult( original_value=value, cleaned_value=cleaned, is_valid=True ) def clean_employee_id(self, value: Any) -> CleaningResult: """清洗工号""" cleaned = self.clean_string(value) if not cleaned: return CleaningResult( original_value=value, cleaned_value=None, is_valid=False, error_message="工号为空" ) return CleaningResult( original_value=value, cleaned_value=cleaned.upper(), # 统一大写 is_valid=True ) def clean_bank_card(self, value: Any) -> CleaningResult: """清洗银行账号""" original = value cleaned = self.clean_string(value) # 只保留数字 cleaned = re.sub(r'\D', '', cleaned) if len(cleaned) < 10: return CleaningResult( original_value=original, cleaned_value=None, is_valid=False, error_message="银行账号长度不足" ) return CleaningResult( original_value=original, cleaned_value=cleaned, is_valid=True ) def clean_id_card(self, value: Any) -> CleaningResult: """清洗身份证号""" original = value cleaned = self.clean_string(value) # 统一大写 cleaned = cleaned.upper() # 只保留数字和X cleaned = re.sub(r'[^0-9X]', '', cleaned) if len(cleaned) not in [15, 18]: return CleaningResult( original_value=original, cleaned_value=None, is_valid=False, error_message="身份证号长度应为15或18位" ) return CleaningResult( original_value=original, cleaned_value=cleaned, is_valid=True ) def validate_field(self, field_type: str, value: Any) -> Tuple[bool, Optional[str]]: """ 验证字段 Returns: (是否通过, 错误信息) """ rules = self.validation_rules.get(field_type, []) for rule in rules: if not rule.validate(value): return False, rule.error_message return True, None def clean_row(self, row: Dict[str, Any], field_mappings: Dict[str, str]) -> Dict[str, Any]: """ 清洗一行数据 Args: row: 原始行数据 {源字段名: 值} field_mappings: 字段映射 {标准字段名: 源字段名} Returns: 清洗后的数据 {标准字段名: 清洗后的值} """ cleaned = {} errors = {} # 反转映射:标准字段 -> 源字段 standard_to_source = {v: k for k, v in field_mappings.items()} for standard_field, source_field in standard_to_source.items(): raw_value = row.get(source_field) # 根据字段类型选择清洗方法 if standard_field in ["员工姓名", "姓名"]: result = self.clean_employee_name(raw_value) elif standard_field in ["工号", "员工号"]: result = self.clean_employee_id(raw_value) elif standard_field in ["银行账号", "卡号"]: result = self.clean_bank_card(raw_value) elif standard_field in ["身份证号", "证件号"]: result = self.clean_id_card(raw_value) elif standard_field in ["基本工资", "奖金", "补贴", "加班费", "扣款", "应发工资", "实发工资", "社保基数", "养老保险", "医疗保险", "失业保险", "公积金", "个税", "应税收入", "税后收入"]: result = self.clean_amount(raw_value) else: result = CleaningResult( original_value=raw_value, cleaned_value=self.clean_string(raw_value), is_valid=bool(raw_value) ) cleaned[standard_field] = result.cleaned_value if not result.is_valid: errors[standard_field] = result.error_message return cleaned def batch_clean( self, data: List[Dict[str, Any]], field_mappings: Dict[str, str] ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: """ 批量清洗数据 Returns: (有效数据, 无效数据) """ valid_data = [] invalid_data = [] for row in data: cleaned_row = self.clean_row(row, field_mappings) # 检查是否有无效字段 has_invalid = any(v is None for v in cleaned_row.values()) if has_invalid: invalid_data.append({ "original": row, "cleaned": cleaned_row }) else: valid_data.append(cleaned_row) return valid_data, invalid_data # 单例实例 _cleaner_instance: Optional[DataCleaner] = None def get_cleaner() -> DataCleaner: """获取数据清洗器单例""" global _cleaner_instance if _cleaner_instance is None: _cleaner_instance = DataCleaner() return _cleaner_instance async def clean_salary_data( data: List[Dict[str, Any]], field_mappings: Dict[str, str] ) -> Dict[str, Any]: """ 清洗工资数据 Args: data: 原始工资数据 field_mappings: 字段映射 Returns: 清洗结果 """ cleaner = get_cleaner() valid_data, invalid_data = cleaner.batch_clean(data, field_mappings) return { "total_count": len(data), "valid_count": len(valid_data), "invalid_count": len(invalid_data), "valid_data": valid_data, "invalid_data": invalid_data, "cleaning_time": datetime.utcnow().isoformat(), }