#!/usr/bin/env python3 """ 通用月度数据导入脚本 支持: 库存成本、菜品销售明细、菜品成本BOM、营业费用、中央厨房、配送明细、采购货品 用法: python3 import_monthly_data.py --dataset inventory --month 2026-04-01 --file --db bill_query_test """ import argparse import hashlib import os import sys import openpyxl import pandas as pd import psycopg2 from psycopg2.extras import execute_values from datetime import datetime DB_CONFIG = { 'host': 'localhost', 'port': 5432, 'user': 'freedak', 'password': '', } def file_sha256(filepath): h = hashlib.sha256() with open(filepath, 'rb') as f: for chunk in iter(lambda: f.read(8192), b''): h.update(chunk) return h.hexdigest() def to_num(val): if val is None or val == '': return None if isinstance(val, float) and pd.isna(val): return None try: f = float(val) import math if math.isinf(f) or math.isnan(f): return None return f except (ValueError, TypeError): return None def to_text(val): if val is None: return None if isinstance(val, float) and pd.isna(val): return None s = str(val).strip() if s.lower() == 'nan' or s.lower() == 'none': return None return s if s else None def to_date(val): """Convert value to date string YYYY-MM-DD or None""" if val is None: return None if isinstance(val, float) and pd.isna(val): return None if isinstance(val, (datetime,)): return val.strftime('%Y-%m-%d') s = str(val).strip() if not s or s.lower() == 'nan' or s.lower() == 'none': return None # Try pandas Timestamp try: ts = pd.Timestamp(s) if pd.isna(ts): return None return ts.strftime('%Y-%m-%d') except (ValueError, TypeError): return s # Return as-is, let DB handle it def clean_percent(val): """Convert percentage value: string '81.23%' -> 81.23, float 0.8123 -> 81.23""" if val is None: return None if isinstance(val, float) and pd.isna(val): return None if isinstance(val, str): s = val.strip().replace('%', '') if not s or s.lower() == 'nan': return None try: return float(s) except ValueError: return None num = float(val) return num * 100 if abs(num) <= 1 else num def connect(dbname='bill_query'): cfg = DB_CONFIG.copy() cfg['dbname'] = dbname return psycopg2.connect(**cfg) # ============================================================ # 库存成本导入 # ============================================================ def import_inventory_cost(conn, filepath, report_month): """库存倒挤成本导入""" print(f"\n=== 库存成本导入 ===") print(f"文件: {filepath}") print(f"月份: {report_month}") sha = file_sha256(filepath) source_file = os.path.basename(filepath) wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True) ws = wb.active # Row 2: group headers, Row 3: sub-headers, Row 4+: data rows = list(ws.iter_rows(min_row=4, values_only=True)) data_rows = [r for r in rows if r[4] is not None and str(r[4]).strip()] print(f"数据行数: {len(data_rows)}") cur = conn.cursor() # Check if already imported cur.execute( "SELECT 1 FROM public.inventory_cost_import_log WHERE source_file = %s AND report_month = %s", (source_file, report_month) ) if cur.fetchone(): print("已导入过,跳过") wb.close() return # Insert import log cur.execute( """INSERT INTO public.inventory_cost_import_log (source_file, report_month, source_rows, imported_rows, status) VALUES (%s, %s, %s, %s, 'importing')""", (source_file, report_month, len(data_rows), len(data_rows)) ) # Batch insert insert_sql = """INSERT INTO public.inventory_cost_records ( report_month, source_file, source_row, brand, region, cost_unit_name, cost_unit_code, item_name, specification, major_category, minor_category, finance_category, item_code, unit, opening_quantity, opening_unit_cost, opening_amount, opening_tax, opening_tax_inclusive, purchase_quantity, purchase_unit_cost, purchase_amount, purchase_tax, purchase_tax_inclusive, ending_quantity, ending_unit_cost, ending_amount, ending_tax, ending_tax_inclusive, consumption_quantity, consumption_unit_cost, consumption_amount, consumption_tax, consumption_tax_inclusive, conversion_precision_amount, conversion_precision_tax_inclusive, return_loss_amount, return_loss_tax_inclusive ) VALUES %s""" batch = [] batch_size = 500 for idx, r in enumerate(data_rows): source_row = idx + 4 consumption_amount = to_num(r[28]) vals = ( report_month, source_file, source_row, to_text(r[0]), to_text(r[1]), to_text(r[2]), to_text(r[3]), to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]), to_text(r[9]), to_text(r[10]), to_num(r[11]), to_num(r[12]), to_num(r[13]), to_num(r[14]), to_num(r[15]), to_num(r[16]), to_num(r[17]), to_num(r[18]), to_num(r[19]), to_num(r[20]), to_num(r[21]), to_num(r[22]), to_num(r[23]), to_num(r[24]), to_num(r[25]), to_num(r[26]), to_num(r[27]), consumption_amount, to_num(r[29]), to_num(r[30]), to_num(r[31]), to_num(r[32]), to_num(r[33]), to_num(r[34]), ) batch.append(vals) if len(batch) >= batch_size: execute_values(cur, insert_sql, batch, page_size=500) conn.commit() print(f" 已导入 {idx + 1}/{len(data_rows)} 行") batch = [] if batch: execute_values(cur, insert_sql, batch, page_size=500) conn.commit() print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") # Update log status cur.execute( "UPDATE public.inventory_cost_import_log SET status = 'completed' WHERE source_file = %s AND report_month = %s", (source_file, report_month) ) conn.commit() wb.close() print(f"库存成本导入完成: {len(data_rows)} 行") # ============================================================ # 菜品销售明细导入 # ============================================================ def import_dish_sales(conn, filepath, report_month, source_file_label=None): """菜品销售明细导入(支持单文件或目录)""" print(f"\n=== 菜品销售明细导入 ===") files = [] if os.path.isdir(filepath): for f in sorted(os.listdir(filepath)): if f.endswith('.xlsx') and not f.startswith('.'): files.append(os.path.join(filepath, f)) else: files.append(filepath) print(f"文件数: {len(files)}") cur = conn.cursor() total_imported = 0 for file_idx, fpath in enumerate(files): fname = os.path.basename(fpath) print(f"\n[{file_idx+1}/{len(files)}] {fname}") file_size = os.path.getsize(fpath) # Check if already imported cur.execute( "SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s", (fname,) ) if cur.fetchone(): print(f" 已导入过, 跳过") continue # Use pandas to read Excel (handles read_only issues with these files) # Header is on row 3 (0-indexed: header=2) df = pd.read_excel(fpath, header=2, engine='openpyxl') # Drop rows where first column is empty df = df.dropna(subset=[df.columns[0]]) data_rows = df.values.tolist() headers = list(df.columns) print(f" 数据行数: {len(data_rows)}") if not data_rows: continue # Insert import log cur.execute( """INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status) VALUES (%s, %s, %s, 'importing')""", (fname, file_size, len(data_rows)) ) # Map headers to columns by name col_map = {} for i, h in enumerate(headers): if h and str(h).strip(): col_map[str(h).strip()] = i # Build column mapping based on known header names field_mapping = { '门店编码': 'store_code', '门店名称': 'store_name', '门店': 'store_name', '账单号': 'bill_no', '订单号': 'bill_no', '人数': 'guest_count', '开台时间': 'opened_at', '结账时间': 'closed_at', '下单时间': 'ordered_at', '菜品名称': 'dish_name', '品项名称': 'dish_name', '分类': 'category_level1', '菜品分类': 'category_level1', '出品部门': 'production_department', '销量': 'sales_quantity', '金额': 'gross_amount', '实收金额': 'received_amount', '实收': 'received_amount', '单价': 'unit_price', '单位': 'unit', '业务类型': 'business_type', '桌台/取餐号': 'table_or_pickup_no', '台号': 'table_or_pickup_no', '做法': 'preparation_method', } # Determine actual columns cols_to_insert = {} for h_name, db_col in field_mapping.items(): if h_name in col_map: cols_to_insert[db_col] = col_map[h_name] # source_file + source_row + mapped columns all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys()) col_names = ','.join(all_cols) numeric_cols = {'guest_count', 'sales_quantity', 'gross_amount', 'received_amount', 'unit_price'} batch = [] for idx, r in enumerate(data_rows): source_row = idx + 4 # data starts at row 4 (header=3) vals = [fname, source_row] for db_col, excel_idx in cols_to_insert.items(): v = r[excel_idx] if excel_idx < len(r) else None if db_col in numeric_cols: vals.append(to_num(v)) else: vals.append(to_text(v)) batch.append(tuple(vals)) if len(batch) >= 500: execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {idx + 1}/{len(data_rows)} 行") batch = [] if batch: execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") total_imported += len(data_rows) # Update log status cur.execute( "UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s", (fname,) ) conn.commit() print(f"\n菜品销售明细导入完成: 共 {total_imported} 行") # ============================================================ # 菜品成本/BOM导入 # ============================================================ def import_dish_cost(conn, filepath, report_month): """菜品成本分析导入:同时导入summary和detail两张表 Excel结构:Row0=标题, Row1=主表头, Row2=子表头(原料区), Row3+=数据 每个菜品有多行原料明细,菜品列(col0-12)只在首行填充""" print(f"\n=== 菜品成本/BOM导入 ===") print(f"文件: {filepath}") sha = file_sha256(filepath) source_file = os.path.basename(filepath) cur = conn.cursor() cur.execute( "SELECT 1 FROM public.dish_cost_analysis_import_log WHERE source_file = %s AND file_sha256 = %s", (source_file, sha) ) if cur.fetchone(): print("已导入过,跳过") return # Read with pandas raw_df = pd.read_excel(filepath, header=None, engine='openpyxl') print(f" 总行数: {len(raw_df)}") # Headers: Row 1 (main) + Row 2 (sub for material detail cols 13+) main_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[1]] sub_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[2]] # Build combined headers headers = [] for i in range(len(main_headers)): m = main_headers[i] s = sub_headers[i] if m and s and m != s: headers.append(f"{m}_{s}") elif m: headers.append(m) elif s: headers.append(s) else: headers.append(f'col_{i}') data_start = 3 df = raw_df.iloc[data_start:].copy() # Identify summary rows: col 0 originally non-empty (first row of each dish) # Detail rows: col 13 non-empty (material name) summary_mask = df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan') detail_mask = df[13].notna() & (df[13].astype(str).str.strip() != '') & (df[13].astype(str).str.strip() != 'nan') summary_df = df[summary_mask].copy() detail_df = df[detail_mask].copy() # Filter out 合计 rows summary_df = summary_df[~summary_df[0].astype(str).str.strip().isin(['合计', '总计'])] print(f" 菜品汇总行数: {len(summary_df)}") print(f" 原料明细行数: {len(detail_df)}") if len(summary_df) == 0: print("无菜品汇总数据,跳过") return # Calculate totals for log sales_amount_total = to_num(summary_df[7].sum()) if 7 < len(summary_df.columns) else 0 theoretical_cost_total = to_num(summary_df[8].sum()) if 8 < len(summary_df.columns) else 0 actual_cost_total = to_num(summary_df[9].sum()) if 9 < len(summary_df.columns) else 0 cost_variance_total = to_num(summary_df[12].sum()) if 12 < len(summary_df.columns) else 0 # Parse report period from filename or use report_month import re date_match = re.search(r'(\d{4})年(\d{1,2})月', source_file) if date_match: year, month = date_match.groups() period_start = f"{year}-{int(month):02d}-01" period_end = (pd.Timestamp(period_start) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d') else: period_start = report_month period_end = (pd.Timestamp(report_month) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d') # Insert import log cur.execute( """INSERT INTO public.dish_cost_analysis_import_log (source_file, file_sha256, sheet_name, workbook_data_rows, dish_summary_rows, material_detail_rows, sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total, report_period_start, report_period_end, import_status, report_month) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'loading', %s) RETURNING import_id""", (source_file, sha, 'Worksheet', len(df), len(summary_df), len(detail_df), sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total, period_start, period_end, report_month) ) import_id = cur.fetchone()[0] # Import summary rows summary_mapping = { 'dish_name': 0, # 菜品名称 'dish_code': 1, # 菜品编码 'price': 2, # 售价 'category_level1': 3, # 菜品大类 'category_level2': 4, # 菜品小类 'dish_unit': 5, # 菜品单位 'sales_quantity': 6, # 销售数量 'sales_amount': 7, # 销售金额(元) 'theoretical_cost': 8, # 理论成本(元) 'actual_cost': 9, # 实际成本(元) 'theoretical_margin_rate_pct': 10, # 理论毛利率 'actual_margin_rate_pct': 11, # 实际毛利率 'cost_variance_amount': 12, # 成本差异金额(元) } summary_cols = ['import_id', 'source_row', 'dish_sequence'] + list(summary_mapping.keys()) summary_col_names = ','.join(summary_cols) # Fields that need clean_percent vs to_num vs to_text summary_percent_fields = {'theoretical_margin_rate_pct', 'actual_margin_rate_pct'} summary_text_fields = {'dish_name', 'dish_code', 'category_level1', 'category_level2', 'dish_unit'} batch = [] for idx, (row_idx, r) in enumerate(summary_df.iterrows()): source_row = int(row_idx) + data_start + 1 # 1-indexed Excel row vals = [import_id, source_row, idx + 1] # dish_sequence for db_col, excel_idx in summary_mapping.items(): v = r[excel_idx] if excel_idx < len(r) else None if v is None or (isinstance(v, float) and pd.isna(v)): vals.append(None) elif db_col in summary_percent_fields: vals.append(clean_percent(v)) elif db_col in summary_text_fields: vals.append(to_text(v)) else: vals.append(to_num(v)) batch.append(tuple(vals)) if len(batch) >= 500: execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" Summary: 已导入 {idx + 1}/{len(summary_df)} 行") batch = [] if batch: execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" Summary: 已导入 {len(summary_df)}/{len(summary_df)} 行") # Get summary_ids for detail linking cur.execute("SELECT summary_id, source_row FROM public.dish_cost_analysis_summary WHERE import_id = %s ORDER BY source_row", (import_id,)) summary_id_map = {r[1]: r[0] for r in cur.fetchall()} # Import detail rows # Detail columns: col13=原料, col14=单位, col15=规格, col16=属性, # col17=理论数量, col18=理论金额, col19=实际数量, col20=实际金额, # col21=损耗数量, col22=损耗金额, col23=数量量比 detail_mapping = { 'material_name': 13, 'material_unit': 14, 'specification': 15, 'material_type': 16, 'theoretical_quantity': 17, 'theoretical_amount': 18, 'actual_quantity': 19, 'actual_amount': 20, 'loss_quantity': 21, 'loss_amount': 22, 'loss_quantity_rate_pct': 23, } detail_cols = ['import_id', 'summary_id', 'source_row', 'material_sequence'] + list(detail_mapping.keys()) + ['theoretical_quantity_per_dish', 'actual_quantity_per_dish'] detail_col_names = ','.join(detail_cols) # For each detail row, find the parent summary's source_row # The parent dish's source_row is the first non-empty col0 at or before this row # We already forward-filled col0, so we can match by looking up the original row # Actually, we need to find which summary row each detail row belongs to # Since we forward-filled, the dish_name (col0) tells us which dish it belongs to # Build a map: dish_name -> summary source_row dish_to_summary_row = {} for idx, (row_idx, r) in enumerate(summary_df.iterrows()): source_row = int(row_idx) + data_start + 1 dish_name = str(r[0]).strip() if pd.notna(r[0]) else '' if dish_name: dish_to_summary_row[dish_name] = source_row # For detail rows, find parent dish by looking backward in original data for nearest non-empty col0 # Build a sorted list of summary source_rows for binary search summary_source_rows = sorted(summary_id_map.keys()) batch = [] mat_seq = 0 current_dish = None for idx, (row_idx, r) in enumerate(detail_df.iterrows()): source_row = int(row_idx) + data_start + 1 excel_row_idx = int(row_idx) # 0-indexed in raw_df # Find parent dish: look backward from current row for nearest non-empty col0 parent_dish_name = None for back_idx in range(excel_row_idx, data_start - 1, -1): val = raw_df.iloc[back_idx, 0] if pd.notna(val) and str(val).strip() and str(val).strip() not in ('合计', '总计', 'nan'): parent_dish_name = str(val).strip() break if parent_dish_name != current_dish: current_dish = parent_dish_name mat_seq = 0 mat_seq += 1 # Find summary_id parent_source_row = dish_to_summary_row.get(parent_dish_name) summary_id = summary_id_map.get(parent_source_row) if parent_source_row else None vals = [import_id, summary_id, source_row, mat_seq] # Get sales_quantity from parent summary for per_dish calculation parent_sales_qty = None if parent_source_row: for _, srow in summary_df.iterrows(): if int(srow.name) + data_start + 1 == parent_source_row: parent_sales_qty = to_num(srow[6]) if pd.notna(srow[6]) else None break for db_col, excel_idx in detail_mapping.items(): v = r[excel_idx] if excel_idx < len(r) else None if v is None or (isinstance(v, float) and pd.isna(v)): vals.append(None) elif db_col == 'loss_quantity_rate_pct': vals.append(clean_percent(v)) elif db_col in ('material_name', 'material_unit', 'specification', 'material_type'): vals.append(to_text(v)) else: vals.append(to_num(v)) # Calculate per_dish quantities theoretical_qty = vals[-len(detail_mapping) + 6] # theoretical_quantity in vals actual_qty = vals[-len(detail_mapping) + 8] # actual_quantity in vals # Actually let's just get them directly theoretical_qty = to_num(r[17]) if pd.notna(r[17]) else None actual_qty = to_num(r[19]) if pd.notna(r[19]) else None if theoretical_qty is not None and parent_sales_qty not in (None, 0): vals.append(theoretical_qty / parent_sales_qty) else: vals.append(None) if actual_qty is not None and parent_sales_qty not in (None, 0): vals.append(actual_qty / parent_sales_qty) else: vals.append(None) batch.append(tuple(vals)) if len(batch) >= 500: execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" Detail: 已导入 {idx + 1}/{len(detail_df)} 行") batch = [] if batch: execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" Detail: 已导入 {len(detail_df)}/{len(detail_df)} 行") # Update log status cur.execute( "UPDATE public.dish_cost_analysis_import_log SET import_status = 'success' WHERE import_id = %s", (import_id,) ) conn.commit() print(f"菜品成本导入完成: {len(summary_df)} 菜品, {len(detail_df)} 原料明细") # ============================================================ # 营业费用导入 # ============================================================ def import_operating_expense(conn, filepath, report_month, expense_type='operating'): """营业费用导入:宽表转长表,每行=科目×门店""" print(f"\n=== 营业费用导入 ({expense_type}) ===") print(f"文件: {filepath}") sha = file_sha256(filepath) source_file = os.path.basename(filepath) cur = conn.cursor() cur.execute( "SELECT 1 FROM public.operating_expense_import_log WHERE source_file = %s AND file_sha256 = %s", (source_file, sha) ) if cur.fetchone(): print("已导入过,跳过") return # Read .xls file with pandas raw_df = pd.read_excel(filepath, header=None, engine='xlrd') print(f" 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}") # Row 0: headers (科目编码, 科目名称, 统计方式, 方向3, 合计金额, 01 大钟寺金额, ...) headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]] # Data starts from row 1 df = raw_df.iloc[1:].copy() # Filter out summary rows (col 0 = '费用科目' or empty) df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != '费用科目')] print(f" 科目行数: {len(df)}") # Parse store columns: col 5+ are store columns (skip col 0-4: code, name, direction, direction3, total) # Extract store code and name from header store_cols = [] for ci in range(5, len(headers)): h = headers[ci] if not h: continue # Parse "01 大钟寺金额" -> code="01", name="大钟寺" import re m = re.match(r'([A-Z]\d+)\s+(.+?)金额$', h) if m: store_cols.append((ci, m.group(1), m.group(2))) else: # Keep as-is with full header store_cols.append((ci, None, h)) print(f" 门店列数: {len(store_cols)}") # Calculate total amount for log total_amount = 0 for _, r in df.iterrows(): v = to_num(r[4]) if pd.notna(r[4]) else 0 # col 4 = 合计金额 total_amount += v if v else 0 # Insert import log cur.execute( """INSERT INTO public.operating_expense_import_log (report_month, source_file, source_sheet, file_sha256, source_total_amount, imported_record_count) VALUES (%s, %s, %s, %s, %s, 0) RETURNING import_id""", (report_month, source_file, 'Worksheet', sha, total_amount) ) import_id = cur.fetchone()[0] # Insert records: long format (one row per account × store) insert_cols = ['import_id', 'report_month', 'source_file', 'source_sheet', 'source_row', 'source_column', 'account_code', 'account_name', 'accounting_direction', 'is_summary_account', 'parent_account_code', 'cost_unit_source_header', 'cost_unit_source_code', 'cost_unit_source_name', 'amount'] col_names = ','.join(insert_cols) batch = [] total_records = 0 for row_idx, (_, r) in enumerate(df.iterrows()): account_code = to_text(r[0]) account_name = to_text(r[1]) direction = to_text(r[2]) # 统计方式 (借方/贷方) excel_row = int(r.name) + 1 # 1-indexed Excel row # Determine if summary account: 7-digit codes (5032401) are sub-accounts of 5-digit (50324) is_summary = False parent_code = None if account_code and len(account_code) == 7: parent_code = account_code[:5] is_summary = False elif account_code and len(account_code) == 5: is_summary = True for col_idx, store_code, store_name in store_cols: amount = to_num(r[col_idx]) if col_idx < len(r) and pd.notna(r[col_idx]) else None if amount is None: continue # Skip NULL amounts, but keep 0 amounts vals = ( import_id, report_month, source_file, 'Worksheet', excel_row, col_idx + 1, # 1-indexed column account_code, account_name, direction, is_summary, parent_code, headers[col_idx], # full header text store_code, store_name, amount, ) batch.append(vals) total_records += 1 if len(batch) >= 500: execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {total_records} 条记录") batch = [] if batch: execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {total_records} 条记录") # Update log cur.execute( "UPDATE public.operating_expense_import_log SET imported_record_count = %s WHERE import_id = %s", (total_records, import_id) ) conn.commit() print(f"营业费用导入完成: {total_records} 条记录") # ============================================================ # 中央厨房导入 (4表) # ============================================================ def import_central_kitchen(conn, filepath, report_month, ck_type='finished'): """中央厨房导入: finished/recipe/material/processing 使用pandas读取,精确映射列,raw_data存储原始行JSON""" import json type_names = { 'finished': '完工入库', 'recipe': '配方耗用', 'material': '材料日报', 'processing': '加工单价', } table_map = { 'finished': 'central_kitchen_finished_receipt', 'recipe': 'central_kitchen_recipe_consumption', 'material': 'central_kitchen_material_daily', 'processing': 'central_kitchen_processing_cost', } table_name = table_map[ck_type] print(f"\n=== 中央厨房-{type_names.get(ck_type, ck_type)} 导入 ===") print(f"文件: {filepath}") sha = file_sha256(filepath) source_file = os.path.basename(filepath) cur = conn.cursor() cur.execute( "SELECT 1 FROM public.central_kitchen_import_log WHERE source_file = %s AND file_sha256 = %s", (source_file, sha) ) if cur.fetchone(): print("已导入过,跳过") return # Header row varies by type: # finished: row 2 (0-indexed) with sub-header row 3 # recipe: row 0 # material: row 0 with sub-header row 1 # processing: row 2 header_rows = { 'finished': 2, # 0-indexed, merged header rows 2+3 'recipe': 0, 'material': 1, # skip row 0 (group header), use row 1 'processing': 2, } hr = header_rows[ck_type] # Read raw without header to get all rows raw_df = pd.read_excel(filepath, header=None, engine='openpyxl') # Build headers from the appropriate row(s) if ck_type == 'finished': # Row 2 has group headers, row 3 has sub-headers # Don't fill forward - empty cells stay empty (merged cells) h1 = raw_df.iloc[2].fillna('').astype(str).str.strip() h2 = raw_df.iloc[3].fillna('').astype(str).str.strip() headers = [] for i in range(len(h1)): g = h1[i] s = h2[i] if g and s: headers.append(f"{g}_{s}") elif g: headers.append(g) elif s: headers.append(s) else: headers.append(f'col_{i}') # Placeholder for empty headers data_start = 4 elif ck_type == 'material': # Row 0 has group headers, row 1 has sub-headers h1 = raw_df.iloc[0].fillna('').astype(str).str.strip() h2 = raw_df.iloc[1].fillna('').astype(str).str.strip() headers = [] current_group = '' for i in range(len(h1)): g = h1[i] if h1[i] else current_group if g: current_group = g s = h2[i] if h2[i] else '' headers.append(f"{g}_{s}" if s else g) data_start = 2 else: # processing: row 2 (main) + row 3 (sub) — like finished h1 = raw_df.iloc[hr].fillna('').astype(str).str.strip() h2 = raw_df.iloc[hr + 1].fillna('').astype(str).str.strip() headers = [] for i in range(len(h1)): g = h1[i] s = h2[i] if g and s and g != s: headers.append(f"{g}_{s}") elif g and g != 'nan': headers.append(g) elif s and s != 'nan': headers.append(s) else: headers.append(f'col_{i}') data_start = hr + 2 # Extract data rows df = raw_df.iloc[data_start:] df = df.dropna(subset=[df.columns[0]]) # Filter out 合计/total rows by checking all string cells for '合计' mask = pd.Series([True] * len(df), index=df.index) for ci in range(min(15, df.shape[1])): mask = mask & ~df[ci].astype(str).str.strip().isin(['合计', '总计', 'Total', 'total']) # Filter out footer rows (e.g. "制表人:xxx") mask = mask & ~df[0].astype(str).str.strip().str.startswith('制表人') mask = mask & ~df[0].astype(str).str.strip().str.startswith('打印') mask = mask & ~df[0].astype(str).str.strip().str.startswith('第') df = df[mask] data_rows = df.values.tolist() print(f"数据行数: {len(data_rows)}") if not data_rows: return # Insert import log cur.execute( """INSERT INTO public.central_kitchen_import_log (report_month, dataset_type, source_file, source_sheet, file_sha256, workbook_rows, imported_rows, excluded_rows, status) VALUES (%s, %s, %s, %s, %s, %s, %s, 0, 'loading') RETURNING import_id""", (report_month, ck_type, source_file, 'Worksheet', sha, len(data_rows), len(data_rows)) ) import_id = cur.fetchone()[0] # Get DB columns (exclude record_id, import_id, source_row, raw_data) cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{table_name}' ORDER BY ordinal_position") db_cols = [r[0] for r in cur.fetchall()] insertable_cols = [c for c in db_cols if c not in ('record_id', 'import_id', 'source_row', 'raw_data')] # Build header name -> excel index map (keep first occurrence) col_map = {} for i, h in enumerate(headers): if h and str(h).strip(): key = str(h).strip() if key not in col_map: col_map[key] = i # Type-specific column mappings (db_col -> list of possible header names) mappings = { 'finished': { 'receipt_date': ['入库日期'], 'workshop': ['col_1'], 'processing_warehouse': ['加工仓库'], 'recipe_name': ['配方名称'], 'product_name': ['货品名称'], 'specification': ['规格'], 'unit': ['库存单位'], 'return_quantity': ['退库数量'], 'return_amount': ['退库金额'], 'inbound_avg_unit_price': ['平均单价'], 'inbound_quantity': ['入库数量'], 'inbound_amount': ['入库金额'], 'raw_material_cost': ['加工利润分析_原料成本', '原料成本'], 'source_fee_cost': ['费用成本'], 'source_processing_profit': ['增值金额(加工利润)'], 'source_margin_rate': ['毛利率'], 'source_cost_rate': ['成本率'], 'theoretical_inbound_quantity': ['加工达成率分析_理论入库数量', '理论入库数量'], 'actual_inbound_quantity': ['实际入库数量'], 'inbound_difference_quantity': ['差异数量'], 'achievement_rate': ['加工达成率'], 'standard_expected_quantity': ['加工应产量分析_标准应产量', '标准应产量'], 'average_expected_quantity': ['平均应产量'], 'expected_quantity_variance_rate': ['应产量差异量比'], 'water_cost': ['水费'], 'electricity_cost': ['电费'], 'gas_cost': ['燃气费'], 'labor_cost': ['人工费'], 'other_cost': ['其他费用'], 'detailed_fee_total': ['费用合计'], }, 'recipe': { 'business_date': ['日期'], 'processing_warehouse': ['加工仓库'], 'finance_category_major': ['财务大类'], 'finance_category_minor': ['财务小类'], 'category_major': ['所属大类'], 'category_minor': ['所属小类'], 'item_id': ['货品ID'], 'item_code': ['编码'], 'item_name': ['货品名称'], 'specification': ['规格'], 'unit': ['库存单位'], 'unit_price_excl_tax': ['未税单价'], 'unit_price': ['单价'], 'recipe_name': ['配方名称'], 'standard_usage': ['标准用量'], 'actual_average_usage': ['实际平均用量'], 'planned_output_quantity': ['计划加工数量'], 'actual_output_quantity': ['实际加工数量'], 'output_unit': ['加工数量库存单位'], 'theoretical_quantity': ['理论用量'], 'net_quantity': ['净料用量'], 'theoretical_amount': ['理论金额'], 'issue_quantity': ['领用数量'], 'issue_amount': ['领用金额'], 'source_inventory_cost_quantity': ['成本数量(进销存)'], 'source_inventory_cost_amount': ['成本金额(进销存)'], 'actual_vs_theory_quantity': ['实际与理论差异数量'], 'actual_vs_theory_amount': ['实际与理论差异金额'], 'actual_vs_theory_rate': ['实际差异量比'], 'issue_vs_theory_quantity': ['领用与理论差异数量'], 'issue_vs_theory_amount': ['领用与理论差异金额'], 'issue_vs_theory_rate': ['领用差异量比'], 'actual_yield': ['实际出成率'], 'recipe_yield': ['配方出成率'], 'yield_difference': ['出成率差异'], }, 'material': { 'business_date': ['日期'], 'finance_category_major': ['财务大类'], 'finance_category_minor': ['财务小类'], 'category_major': ['所属大类'], 'category_minor': ['所属小类'], 'item_id': ['货品ID'], 'item_code': ['编码'], 'item_name': ['货品名称'], 'specification': ['规格'], 'unit': ['库存单位'], }, 'processing': { 'report_month': ['report_month'], # Will be set from parameter 'item_id': ['货品ID'], 'recipe_name': ['配方名称'], 'product_code': ['货品编码'], 'product_name': ['货品名称'], 'category_major': ['所属大类'], 'category_minor': ['所属小类'], 'finance_category_major': ['财务大类'], 'finance_category_minor': ['财务小类'], 'specification': ['规格'], 'unit': ['单位'], 'inbound_quantity': ['入库数量'], 'inbound_avg_unit_price': ['入库均价'], 'theoretical_unit_cost': ['理论单价'], 'standard_unit_cost': ['标准单价'], 'actual_unit_cost': ['实际单价'], 'theoretical_cost': ['理论成本'], 'standard_cost': ['标准成本'], 'actual_cost': ['实际成本'], }, } field_mapping = mappings[ck_type] cols_to_insert = {} fixed_values = {} # Values not from Excel (e.g. report_month) for db_col, h_names in field_mapping.items(): if db_col not in insertable_cols: continue found = False for h_name in h_names: if h_name in col_map: cols_to_insert[db_col] = col_map[h_name] found = True break if not found and db_col == 'report_month': fixed_values['report_month'] = report_month # Determine numeric columns from DB schema cur.execute(f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name='{table_name}'") col_types = {r[0]: r[1] for r in cur.fetchall()} numeric_db_cols = {c for c, t in col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')} date_db_cols = {c for c, t in col_types.items() if t == 'date'} all_cols = ['import_id', 'source_row'] + list(cols_to_insert.keys()) + list(fixed_values.keys()) + ['raw_data'] col_names = ','.join(all_cols) batch = [] for idx, r in enumerate(data_rows): source_row = idx + data_start + 1 # 1-indexed Excel row vals = [import_id, source_row] for db_col, excel_idx in cols_to_insert.items(): v = r[excel_idx] if excel_idx < len(r) else None if db_col in numeric_db_cols: vals.append(to_num(v) if v is not None else None) elif db_col in date_db_cols: vals.append(to_date(v)) else: vals.append(to_text(v)) # Fixed values (not from Excel) for fv_col in fixed_values: if fv_col in date_db_cols: vals.append(to_date(fixed_values[fv_col])) else: vals.append(fixed_values[fv_col]) # raw_data: store entire row as JSON raw_dict = {} for hi, hv in enumerate(headers): if hv and hi < len(r): rv = r[hi] if pd.notna(rv): raw_dict[str(hv)] = str(rv) if not isinstance(rv, (int, float)) else rv vals.append(json.dumps(raw_dict, ensure_ascii=False)) batch.append(tuple(vals)) if len(batch) >= 500: execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {idx + 1}/{len(data_rows)} 行") batch = [] if batch: execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") # Update log status cur.execute( "UPDATE public.central_kitchen_import_log SET status = 'success', completed_at = now() WHERE import_id = %s", (import_id,) ) conn.commit() print(f"中央厨房{type_names[ck_type]}导入完成: {len(data_rows)} 行") # ============================================================ # 配送明细导入 # ============================================================ def import_distribution(conn, filepath, report_month): """配送明细导入(支持单文件或目录)""" import json print(f"\n=== 配送明细导入 ===") files = [] if os.path.isdir(filepath): for f in sorted(os.listdir(filepath)): if f.endswith('.xlsx') and not f.startswith('.'): files.append(os.path.join(filepath, f)) else: files.append(filepath) print(f"文件数: {len(files)}") cur = conn.cursor() total_imported = 0 for file_idx, fpath in enumerate(files): fname = os.path.basename(fpath) print(f"\n[{file_idx+1}/{len(files)}] {fname}") sha = file_sha256(fpath) cur.execute( "SELECT 1 FROM public.distribution_import_log WHERE source_file = %s AND file_sha256 = %s", (fname, sha) ) if cur.fetchone(): print(" 已导入过, 跳过") continue # Read with pandas, header on row 0 raw_df = pd.read_excel(fpath, header=None, engine='openpyxl') headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]] df = raw_df.iloc[1:] df = df.dropna(subset=[df.columns[0]]) data_rows = df.values.tolist() print(f" 数据行数: {len(data_rows)}") if not data_rows: continue # Parse period from filename or use report_month import re date_match = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日.*?(\d{1,2})日', fname) if date_match: year, month, start_day, end_day = date_match.groups() period_start = f"{year}-{int(month):02d}-{int(start_day):02d}" period_end = f"{year}-{int(month):02d}-{int(end_day):02d}" else: # Use full month period_start = report_month # Last day of month rm_date = pd.Timestamp(report_month) period_end = (rm_date + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d') cur.execute( """INSERT INTO public.distribution_import_log (source_file, source_sheet, file_sha256, workbook_rows, summary_rows_excluded, imported_rows, report_month, period_start, period_end, status) VALUES (%s, %s, %s, %s, 0, %s, %s, %s, %s, 'loading') RETURNING import_id""", (fname, 'Worksheet', sha, len(data_rows), len(data_rows), report_month, period_start, period_end) ) import_id = cur.fetchone()[0] # Get DB columns cur.execute("SELECT column_name, data_type FROM information_schema.columns WHERE table_name='distribution_detail_records' ORDER BY ordinal_position") db_col_types = {r[0]: r[1] for r in cur.fetchall()} insertable_cols = [c for c in db_col_types if c not in ('record_id', 'import_id', 'source_row', 'raw_data')] # Build header map col_map = {} for i, h in enumerate(headers): if h and str(h).strip(): col_map[str(h).strip()] = i # Known mappings for distribution based on actual Excel headers field_mapping = { 'business_date': ['日期', '业务日期'], 'distribution_center_code': ['配送中心编码'], 'distribution_center_name': ['配送中心'], 'distribution_center_company': ['配送中心所属公司'], 'store_company': ['门店所属公司'], 'store_type': ['门店类型'], 'store_brand': ['门店品牌'], 'region_level1': ['一级区域'], 'terminal_region': ['末级区域'], 'store_code': ['门店编码'], 'store_name': ['门店'], 'store_tax_entity': ['门店财税主体'], 'original_batch_unit_price': ['原始批次单价'], 'cost_avg_unit_price': ['成本平均单价'], 'cost_total_amount': ['成本总金额'], 'team': ['班组'], 'cost_excl_tax_amount': ['成本未含税金额'], 'cost_tax_amount': ['成本税额'], 'cost_excl_tax_unit_price': ['成本未含税单价'], 'outbound_tax_amount': ['配出税额'], 'distribution_sales_tax_rate': ['配送销售税点'], 'route': ['路线'], 'profit_amount': ['利润'], 'profit_excl_tax_amount': ['未含税利润', '利润未含税'], 'distribution_type': ['配送类型'], 'logistics_attribute': ['物流属性'], 'item_id': ['货品ID'], 'item_name': ['货品名称', '货品'], 'item_alias': ['货品别名'], 'specification': ['规格'], 'item_purpose': ['货品用途', '用途'], 'unit': ['单位'], 'normal_quantity': ['正常品数量', '正常数量'], 'gift_quantity': ['赠品数量'], 'total_quantity': ['总数量', '合计数量'], 'outbound_avg_unit_price': ['配出平均单价', '配出均价'], 'outbound_total_amount': ['配出总金额', '配出金额'], 'outbound_margin_rate': ['配出毛利率'], 'outbound_excl_tax_unit_price': ['配出未含税单价'], 'outbound_excl_tax_amount': ['配出未含税金额'], 'document_no': ['单号'], 'requisition_no': ['要货单号', '申请单号'], 'payment_method': ['支付方式', '结算方式'], 'value_added_coefficient': ['增值系数'], 'audit_time': ['审核时间'], 'inbound_batch': ['入库批次'], 'inbound_batch_tax_rate': ['入库批次税点', '入库批次税率'], 'custom_batch_code': ['自定义批次码', '自定义批次编码'], 'invoice_type': ['发票种类', '发票类型'], 'production_time': ['生产时间'], 'shelf_life': ['保质期'], 'supplier_batch': ['供应商批次'], 'supplier_code': ['供应商编码'], 'supplier_category': ['供应商类别', '供应商分类'], 'supplier_name': ['供应商'], 'supplier_contact': ['供应商联系人'], 'supplier_phone': ['供应商联系电话', '供应商电话'], 'supplier_address': ['供应商地址'], 'from_warehouse': ['调出仓库', '发货仓库'], 'to_warehouse': ['调入仓库', '收货仓库'], 'item_code': ['货品编码'], 'item_barcode': ['货品条形码', '货品条码'], 'minor_category': ['所属小类', '小类'], 'major_category': ['所属大类', '大类'], 'internal_brand': ['内部品牌'], 'finance_category': ['财务分类'], 'created_time': ['制单时间', '创建时间'], 'creator': ['制单人', '创建人'], 'auditor': ['审核人'], 'arrival_time': ['到货时间'], 'document_remark': ['单据备注'], 'document_item_remark': ['单据项备注'], 'reason_remark': ['原因备注'], 'business_date': ['业务日期'], 'contact_address': ['联系地址'], 'contact_phone': ['联系电话'], 'contact_person': ['联系人'], } cols_to_insert = {} for db_col, h_names in field_mapping.items(): if db_col not in insertable_cols: continue for h_name in h_names: if h_name in col_map: cols_to_insert[db_col] = col_map[h_name] break numeric_db_cols = {c for c, t in db_col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')} date_db_cols = {c for c, t in db_col_types.items() if t == 'date'} all_cols = ['import_id', 'source_row', 'report_month', 'source_file', 'source_sheet'] + list(cols_to_insert.keys()) col_names = ','.join(all_cols) batch = [] for idx, r in enumerate(data_rows): source_row = idx + 2 # 1-indexed, data starts at row 2 vals = [import_id, source_row, report_month, fname, 'Worksheet'] for db_col, excel_idx in cols_to_insert.items(): v = r[excel_idx] if excel_idx < len(r) else None if db_col in numeric_db_cols: vals.append(to_num(v) if v is not None else None) elif db_col in date_db_cols: vals.append(to_date(v)) else: vals.append(to_text(v)) batch.append(tuple(vals)) if len(batch) >= 500: execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {idx + 1}/{len(data_rows)} 行") batch = [] if batch: execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500) conn.commit() print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") total_imported += len(data_rows) # Update log status cur.execute( "UPDATE public.distribution_import_log SET status = 'success', completed_at = now() WHERE import_id = %s", (import_id,) ) conn.commit() print(f"\n配送明细导入完成: 共 {total_imported} 行") # ============================================================ # Main # ============================================================ def main(): parser = argparse.ArgumentParser(description='月度数据导入工具') parser.add_argument('--dataset', required=True, choices=['inventory', 'dish_sales', 'dish_cost', 'operating_expense', 'central_kitchen', 'distribution'], help='数据域') parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)') parser.add_argument('--file', required=True, help='文件路径或目录') parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)') parser.add_argument('--ck-type', default='finished', choices=['finished', 'recipe', 'material', 'processing'], help='中央厨房子类型') parser.add_argument('--expense-type', default='operating', choices=['operating', 'management'], help='费用类型') args = parser.parse_args() conn = connect(args.db) conn.autocommit = False try: if args.dataset == 'inventory': import_inventory_cost(conn, args.file, args.month) elif args.dataset == 'dish_sales': import_dish_sales(conn, args.file, args.month) elif args.dataset == 'dish_cost': import_dish_cost(conn, args.file, args.month) elif args.dataset == 'operating_expense': import_operating_expense(conn, args.file, args.month, args.expense_type) elif args.dataset == 'central_kitchen': import_central_kitchen(conn, args.file, args.month, args.ck_type) elif args.dataset == 'distribution': import_distribution(conn, args.file, args.month) print("\n=== 导入完成 ===") except Exception as e: conn.rollback() print(f"错误: {e}", file=sys.stderr) raise finally: conn.close() if __name__ == '__main__': main()