Files
SBrainCO/db/pipeline/import_monthly_data.py
T
freedakgmail 2fc1b96f44 feat: 新增db/pipeline数据导入管线,适配5月数据格式
- 整合所有导入脚本到db/pipeline/目录
- config.py: 按月份动态配置数据源路径,支持4月/5月不同目录结构
- import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details)
- import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径
- import_may_data.sh: 5月一键导入脚本(12步全流程)
- run_all.py: 一键全量导入+物化视图刷新编排
- refresh_materialized_views.py: 按依赖顺序刷新38个物化视图
- verify.py/verify_import.py: 数据一致性验证
- README.md: 管线文档
2026-08-18 22:17:26 +08:00

1302 lines
54 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""
通用月度数据导入脚本
支持: 库存成本、菜品销售明细、菜品成本BOM、营业费用、中央厨房、配送明细、采购货品
用法: python3 import_monthly_data.py --dataset inventory --month 2026-04-01 --file <path> --db bill_query_test
"""
import argparse
import hashlib
import os
import sys
import openpyxl
import pandas as pd
import psycopg2
from psycopg2.extras import execute_values
from datetime import datetime
DB_CONFIG = {
'host': 'localhost',
'port': 5432,
'user': 'freedak',
'password': '',
}
def file_sha256(filepath):
h = hashlib.sha256()
with open(filepath, 'rb') as f:
for chunk in iter(lambda: f.read(8192), b''):
h.update(chunk)
return h.hexdigest()
def to_num(val):
if val is None or val == '':
return None
if isinstance(val, float) and pd.isna(val):
return None
try:
f = float(val)
import math
if math.isinf(f) or math.isnan(f):
return None
return f
except (ValueError, TypeError):
return None
def to_text(val):
if val is None:
return None
if isinstance(val, float) and pd.isna(val):
return None
s = str(val).strip()
if s.lower() == 'nan' or s.lower() == 'none':
return None
return s if s else None
def to_date(val):
"""Convert value to date string YYYY-MM-DD or None"""
if val is None:
return None
if isinstance(val, float) and pd.isna(val):
return None
if isinstance(val, (datetime,)):
return val.strftime('%Y-%m-%d')
s = str(val).strip()
if not s or s.lower() == 'nan' or s.lower() == 'none':
return None
# Try pandas Timestamp
try:
ts = pd.Timestamp(s)
if pd.isna(ts):
return None
return ts.strftime('%Y-%m-%d')
except (ValueError, TypeError):
return s # Return as-is, let DB handle it
def clean_percent(val):
"""Convert percentage value: string '81.23%' -> 81.23, float 0.8123 -> 81.23"""
if val is None:
return None
if isinstance(val, float) and pd.isna(val):
return None
if isinstance(val, str):
s = val.strip().replace('%', '')
if not s or s.lower() == 'nan':
return None
try:
return float(s)
except ValueError:
return None
num = float(val)
return num * 100 if abs(num) <= 1 else num
def connect(dbname='bill_query'):
cfg = DB_CONFIG.copy()
cfg['dbname'] = dbname
return psycopg2.connect(**cfg)
# ============================================================
# 库存成本导入
# ============================================================
def import_inventory_cost(conn, filepath, report_month):
"""库存倒挤成本导入"""
print(f"\n=== 库存成本导入 ===")
print(f"文件: {filepath}")
print(f"月份: {report_month}")
sha = file_sha256(filepath)
source_file = os.path.basename(filepath)
wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
ws = wb.active
# Row 2: group headers, Row 3: sub-headers, Row 4+: data
rows = list(ws.iter_rows(min_row=4, values_only=True))
data_rows = [r for r in rows if r[4] is not None and str(r[4]).strip()]
print(f"数据行数: {len(data_rows)}")
cur = conn.cursor()
# Check if already imported
cur.execute(
"SELECT 1 FROM public.inventory_cost_import_log WHERE source_file = %s AND report_month = %s",
(source_file, report_month)
)
if cur.fetchone():
print("已导入过,跳过")
wb.close()
return
# Insert import log
cur.execute(
"""INSERT INTO public.inventory_cost_import_log (source_file, report_month, source_rows, imported_rows, status)
VALUES (%s, %s, %s, %s, 'importing')""",
(source_file, report_month, len(data_rows), len(data_rows))
)
# Batch insert
insert_sql = """INSERT INTO public.inventory_cost_records (
report_month, source_file, source_row,
brand, region, cost_unit_name, cost_unit_code, item_name, specification,
major_category, minor_category, finance_category, item_code, unit,
opening_quantity, opening_unit_cost, opening_amount, opening_tax, opening_tax_inclusive,
purchase_quantity, purchase_unit_cost, purchase_amount, purchase_tax, purchase_tax_inclusive,
ending_quantity, ending_unit_cost, ending_amount, ending_tax, ending_tax_inclusive,
consumption_quantity, consumption_unit_cost, consumption_amount, consumption_tax, consumption_tax_inclusive,
conversion_precision_amount, conversion_precision_tax_inclusive,
return_loss_amount, return_loss_tax_inclusive
) VALUES %s"""
batch = []
batch_size = 500
for idx, r in enumerate(data_rows):
source_row = idx + 4
consumption_amount = to_num(r[28])
vals = (
report_month, source_file, source_row,
to_text(r[0]), to_text(r[1]), to_text(r[2]), to_text(r[3]),
to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]),
to_text(r[9]), to_text(r[10]),
to_num(r[11]), to_num(r[12]), to_num(r[13]), to_num(r[14]), to_num(r[15]),
to_num(r[16]), to_num(r[17]), to_num(r[18]), to_num(r[19]), to_num(r[20]),
to_num(r[21]), to_num(r[22]), to_num(r[23]), to_num(r[24]), to_num(r[25]),
to_num(r[26]), to_num(r[27]), consumption_amount, to_num(r[29]), to_num(r[30]),
to_num(r[31]), to_num(r[32]),
to_num(r[33]), to_num(r[34]),
)
batch.append(vals)
if len(batch) >= batch_size:
execute_values(cur, insert_sql, batch, page_size=500)
conn.commit()
print(f" 已导入 {idx + 1}/{len(data_rows)}")
batch = []
if batch:
execute_values(cur, insert_sql, batch, page_size=500)
conn.commit()
print(f" 已导入 {len(data_rows)}/{len(data_rows)}")
# Update log status
cur.execute(
"UPDATE public.inventory_cost_import_log SET status = 'completed' WHERE source_file = %s AND report_month = %s",
(source_file, report_month)
)
conn.commit()
wb.close()
print(f"库存成本导入完成: {len(data_rows)}")
# ============================================================
# 菜品销售明细导入
# ============================================================
def import_dish_sales(conn, filepath, report_month, source_file_label=None):
"""菜品销售明细导入(支持单文件或目录)"""
print(f"\n=== 菜品销售明细导入 ===")
files = []
if os.path.isdir(filepath):
for f in sorted(os.listdir(filepath)):
if f.endswith('.xlsx') and not f.startswith('.'):
files.append(os.path.join(filepath, f))
else:
files.append(filepath)
print(f"文件数: {len(files)}")
cur = conn.cursor()
total_imported = 0
for file_idx, fpath in enumerate(files):
fname = os.path.basename(fpath)
print(f"\n[{file_idx+1}/{len(files)}] {fname}")
file_size = os.path.getsize(fpath)
# Check if already imported
cur.execute(
"SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s",
(fname,)
)
if cur.fetchone():
print(f" 已导入过, 跳过")
continue
# Use pandas to read Excel (handles read_only issues with these files)
# Header is on row 3 (0-indexed: header=2)
df = pd.read_excel(fpath, header=2, engine='openpyxl')
# Drop rows where first column is empty
df = df.dropna(subset=[df.columns[0]])
data_rows = df.values.tolist()
headers = list(df.columns)
print(f" 数据行数: {len(data_rows)}")
if not data_rows:
continue
# Insert import log
cur.execute(
"""INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status)
VALUES (%s, %s, %s, 'importing')""",
(fname, file_size, len(data_rows))
)
# Map headers to columns by name
col_map = {}
for i, h in enumerate(headers):
if h and str(h).strip():
col_map[str(h).strip()] = i
# Build column mapping based on known header names
field_mapping = {
'门店编码': 'store_code',
'门店名称': 'store_name',
'门店': 'store_name',
'账单号': 'bill_no',
'订单号': 'bill_no',
'人数': 'guest_count',
'开台时间': 'opened_at',
'结账时间': 'closed_at',
'下单时间': 'ordered_at',
'菜品名称': 'dish_name',
'品项名称': 'dish_name',
'分类': 'category_level1',
'菜品分类': 'category_level1',
'出品部门': 'production_department',
'销量': 'sales_quantity',
'金额': 'gross_amount',
'实收金额': 'received_amount',
'实收': 'received_amount',
'单价': 'unit_price',
'单位': 'unit',
'业务类型': 'business_type',
'桌台/取餐号': 'table_or_pickup_no',
'台号': 'table_or_pickup_no',
'做法': 'preparation_method',
}
# Determine actual columns
cols_to_insert = {}
for h_name, db_col in field_mapping.items():
if h_name in col_map:
cols_to_insert[db_col] = col_map[h_name]
# source_file + source_row + mapped columns
all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys())
col_names = ','.join(all_cols)
numeric_cols = {'guest_count', 'sales_quantity', 'gross_amount', 'received_amount', 'unit_price'}
batch = []
for idx, r in enumerate(data_rows):
source_row = idx + 4 # data starts at row 4 (header=3)
vals = [fname, source_row]
for db_col, excel_idx in cols_to_insert.items():
v = r[excel_idx] if excel_idx < len(r) else None
if db_col in numeric_cols:
vals.append(to_num(v))
else:
vals.append(to_text(v))
batch.append(tuple(vals))
if len(batch) >= 500:
execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {idx + 1}/{len(data_rows)}")
batch = []
if batch:
execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {len(data_rows)}/{len(data_rows)}")
total_imported += len(data_rows)
# Update log status
cur.execute(
"UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s",
(fname,)
)
conn.commit()
print(f"\n菜品销售明细导入完成: 共 {total_imported}")
# ============================================================
# 菜品成本/BOM导入
# ============================================================
def import_dish_cost(conn, filepath, report_month):
"""菜品成本分析导入:同时导入summary和detail两张表
Excel结构:Row0=标题, Row1=主表头, Row2=子表头(原料区), Row3+=数据
每个菜品有多行原料明细,菜品列(col0-12)只在首行填充"""
print(f"\n=== 菜品成本/BOM导入 ===")
print(f"文件: {filepath}")
sha = file_sha256(filepath)
source_file = os.path.basename(filepath)
cur = conn.cursor()
cur.execute(
"SELECT 1 FROM public.dish_cost_analysis_import_log WHERE source_file = %s AND file_sha256 = %s",
(source_file, sha)
)
if cur.fetchone():
print("已导入过,跳过")
return
# Read with pandas
raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
print(f" 总行数: {len(raw_df)}")
# Headers: Row 1 (main) + Row 2 (sub for material detail cols 13+)
main_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[1]]
sub_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[2]]
# Build combined headers
headers = []
for i in range(len(main_headers)):
m = main_headers[i]
s = sub_headers[i]
if m and s and m != s:
headers.append(f"{m}_{s}")
elif m:
headers.append(m)
elif s:
headers.append(s)
else:
headers.append(f'col_{i}')
data_start = 3
df = raw_df.iloc[data_start:].copy()
# Identify summary rows: col 0 originally non-empty (first row of each dish)
# Detail rows: col 13 non-empty (material name)
summary_mask = df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan')
detail_mask = df[13].notna() & (df[13].astype(str).str.strip() != '') & (df[13].astype(str).str.strip() != 'nan')
summary_df = df[summary_mask].copy()
detail_df = df[detail_mask].copy()
# Filter out 合计 rows
summary_df = summary_df[~summary_df[0].astype(str).str.strip().isin(['合计', '总计'])]
print(f" 菜品汇总行数: {len(summary_df)}")
print(f" 原料明细行数: {len(detail_df)}")
if len(summary_df) == 0:
print("无菜品汇总数据,跳过")
return
# Calculate totals for log
sales_amount_total = to_num(summary_df[7].sum()) if 7 < len(summary_df.columns) else 0
theoretical_cost_total = to_num(summary_df[8].sum()) if 8 < len(summary_df.columns) else 0
actual_cost_total = to_num(summary_df[9].sum()) if 9 < len(summary_df.columns) else 0
cost_variance_total = to_num(summary_df[12].sum()) if 12 < len(summary_df.columns) else 0
# Parse report period from filename or use report_month
import re
date_match = re.search(r'(\d{4})年(\d{1,2})月', source_file)
if date_match:
year, month = date_match.groups()
period_start = f"{year}-{int(month):02d}-01"
period_end = (pd.Timestamp(period_start) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d')
else:
period_start = report_month
period_end = (pd.Timestamp(report_month) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d')
# Insert import log
cur.execute(
"""INSERT INTO public.dish_cost_analysis_import_log
(source_file, file_sha256, sheet_name, workbook_data_rows,
dish_summary_rows, material_detail_rows,
sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total,
report_period_start, report_period_end, import_status, report_month)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'loading', %s) RETURNING import_id""",
(source_file, sha, 'Worksheet', len(df), len(summary_df), len(detail_df),
sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total,
period_start, period_end, report_month)
)
import_id = cur.fetchone()[0]
# Import summary rows
summary_mapping = {
'dish_name': 0, # 菜品名称
'dish_code': 1, # 菜品编码
'price': 2, # 售价
'category_level1': 3, # 菜品大类
'category_level2': 4, # 菜品小类
'dish_unit': 5, # 菜品单位
'sales_quantity': 6, # 销售数量
'sales_amount': 7, # 销售金额(元)
'theoretical_cost': 8, # 理论成本(元)
'actual_cost': 9, # 实际成本(元)
'theoretical_margin_rate_pct': 10, # 理论毛利率
'actual_margin_rate_pct': 11, # 实际毛利率
'cost_variance_amount': 12, # 成本差异金额(元)
}
summary_cols = ['import_id', 'source_row', 'dish_sequence'] + list(summary_mapping.keys())
summary_col_names = ','.join(summary_cols)
# Fields that need clean_percent vs to_num vs to_text
summary_percent_fields = {'theoretical_margin_rate_pct', 'actual_margin_rate_pct'}
summary_text_fields = {'dish_name', 'dish_code', 'category_level1', 'category_level2', 'dish_unit'}
batch = []
for idx, (row_idx, r) in enumerate(summary_df.iterrows()):
source_row = int(row_idx) + data_start + 1 # 1-indexed Excel row
vals = [import_id, source_row, idx + 1] # dish_sequence
for db_col, excel_idx in summary_mapping.items():
v = r[excel_idx] if excel_idx < len(r) else None
if v is None or (isinstance(v, float) and pd.isna(v)):
vals.append(None)
elif db_col in summary_percent_fields:
vals.append(clean_percent(v))
elif db_col in summary_text_fields:
vals.append(to_text(v))
else:
vals.append(to_num(v))
batch.append(tuple(vals))
if len(batch) >= 500:
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" Summary: 已导入 {idx + 1}/{len(summary_df)}")
batch = []
if batch:
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" Summary: 已导入 {len(summary_df)}/{len(summary_df)}")
# Get summary_ids for detail linking
cur.execute("SELECT summary_id, source_row FROM public.dish_cost_analysis_summary WHERE import_id = %s ORDER BY source_row", (import_id,))
summary_id_map = {r[1]: r[0] for r in cur.fetchall()}
# Import detail rows
# Detail columns: col13=原料, col14=单位, col15=规格, col16=属性,
# col17=理论数量, col18=理论金额, col19=实际数量, col20=实际金额,
# col21=损耗数量, col22=损耗金额, col23=数量量比
detail_mapping = {
'material_name': 13,
'material_unit': 14,
'specification': 15,
'material_type': 16,
'theoretical_quantity': 17,
'theoretical_amount': 18,
'actual_quantity': 19,
'actual_amount': 20,
'loss_quantity': 21,
'loss_amount': 22,
'loss_quantity_rate_pct': 23,
}
detail_cols = ['import_id', 'summary_id', 'source_row', 'material_sequence'] + list(detail_mapping.keys()) + ['theoretical_quantity_per_dish', 'actual_quantity_per_dish']
detail_col_names = ','.join(detail_cols)
# For each detail row, find the parent summary's source_row
# The parent dish's source_row is the first non-empty col0 at or before this row
# We already forward-filled col0, so we can match by looking up the original row
# Actually, we need to find which summary row each detail row belongs to
# Since we forward-filled, the dish_name (col0) tells us which dish it belongs to
# Build a map: dish_name -> summary source_row
dish_to_summary_row = {}
for idx, (row_idx, r) in enumerate(summary_df.iterrows()):
source_row = int(row_idx) + data_start + 1
dish_name = str(r[0]).strip() if pd.notna(r[0]) else ''
if dish_name:
dish_to_summary_row[dish_name] = source_row
# For detail rows, find parent dish by looking backward in original data for nearest non-empty col0
# Build a sorted list of summary source_rows for binary search
summary_source_rows = sorted(summary_id_map.keys())
batch = []
mat_seq = 0
current_dish = None
for idx, (row_idx, r) in enumerate(detail_df.iterrows()):
source_row = int(row_idx) + data_start + 1
excel_row_idx = int(row_idx) # 0-indexed in raw_df
# Find parent dish: look backward from current row for nearest non-empty col0
parent_dish_name = None
for back_idx in range(excel_row_idx, data_start - 1, -1):
val = raw_df.iloc[back_idx, 0]
if pd.notna(val) and str(val).strip() and str(val).strip() not in ('合计', '总计', 'nan'):
parent_dish_name = str(val).strip()
break
if parent_dish_name != current_dish:
current_dish = parent_dish_name
mat_seq = 0
mat_seq += 1
# Find summary_id
parent_source_row = dish_to_summary_row.get(parent_dish_name)
summary_id = summary_id_map.get(parent_source_row) if parent_source_row else None
vals = [import_id, summary_id, source_row, mat_seq]
# Get sales_quantity from parent summary for per_dish calculation
parent_sales_qty = None
if parent_source_row:
for _, srow in summary_df.iterrows():
if int(srow.name) + data_start + 1 == parent_source_row:
parent_sales_qty = to_num(srow[6]) if pd.notna(srow[6]) else None
break
for db_col, excel_idx in detail_mapping.items():
v = r[excel_idx] if excel_idx < len(r) else None
if v is None or (isinstance(v, float) and pd.isna(v)):
vals.append(None)
elif db_col == 'loss_quantity_rate_pct':
vals.append(clean_percent(v))
elif db_col in ('material_name', 'material_unit', 'specification', 'material_type'):
vals.append(to_text(v))
else:
vals.append(to_num(v))
# Calculate per_dish quantities
theoretical_qty = vals[-len(detail_mapping) + 6] # theoretical_quantity in vals
actual_qty = vals[-len(detail_mapping) + 8] # actual_quantity in vals
# Actually let's just get them directly
theoretical_qty = to_num(r[17]) if pd.notna(r[17]) else None
actual_qty = to_num(r[19]) if pd.notna(r[19]) else None
if theoretical_qty is not None and parent_sales_qty not in (None, 0):
vals.append(theoretical_qty / parent_sales_qty)
else:
vals.append(None)
if actual_qty is not None and parent_sales_qty not in (None, 0):
vals.append(actual_qty / parent_sales_qty)
else:
vals.append(None)
batch.append(tuple(vals))
if len(batch) >= 500:
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" Detail: 已导入 {idx + 1}/{len(detail_df)}")
batch = []
if batch:
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" Detail: 已导入 {len(detail_df)}/{len(detail_df)}")
# Update log status
cur.execute(
"UPDATE public.dish_cost_analysis_import_log SET import_status = 'success' WHERE import_id = %s",
(import_id,)
)
conn.commit()
print(f"菜品成本导入完成: {len(summary_df)} 菜品, {len(detail_df)} 原料明细")
# ============================================================
# 营业费用导入
# ============================================================
def import_operating_expense(conn, filepath, report_month, expense_type='operating'):
"""营业费用导入:宽表转长表,每行=科目×门店"""
print(f"\n=== 营业费用导入 ({expense_type}) ===")
print(f"文件: {filepath}")
sha = file_sha256(filepath)
source_file = os.path.basename(filepath)
cur = conn.cursor()
cur.execute(
"SELECT 1 FROM public.operating_expense_import_log WHERE source_file = %s AND file_sha256 = %s",
(source_file, sha)
)
if cur.fetchone():
print("已导入过,跳过")
return
# Read .xls file with pandas
raw_df = pd.read_excel(filepath, header=None, engine='xlrd')
print(f" 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}")
# Row 0: headers (科目编码, 科目名称, 统计方式, 方向3, 合计金额, 01 大钟寺金额, ...)
headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]]
# Data starts from row 1
df = raw_df.iloc[1:].copy()
# Filter out summary rows (col 0 = '费用科目' or empty)
df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != '费用科目')]
print(f" 科目行数: {len(df)}")
# Parse store columns: col 5+ are store columns (skip col 0-4: code, name, direction, direction3, total)
# Extract store code and name from header
store_cols = []
for ci in range(5, len(headers)):
h = headers[ci]
if not h:
continue
# Parse "01 大钟寺金额" -> code="01", name="大钟寺"
import re
m = re.match(r'([A-Z]\d+)\s+(.+?)金额$', h)
if m:
store_cols.append((ci, m.group(1), m.group(2)))
else:
# Keep as-is with full header
store_cols.append((ci, None, h))
print(f" 门店列数: {len(store_cols)}")
# Calculate total amount for log
total_amount = 0
for _, r in df.iterrows():
v = to_num(r[4]) if pd.notna(r[4]) else 0 # col 4 = 合计金额
total_amount += v if v else 0
# Insert import log
cur.execute(
"""INSERT INTO public.operating_expense_import_log
(report_month, source_file, source_sheet, file_sha256, source_total_amount, imported_record_count)
VALUES (%s, %s, %s, %s, %s, 0) RETURNING import_id""",
(report_month, source_file, 'Worksheet', sha, total_amount)
)
import_id = cur.fetchone()[0]
# Insert records: long format (one row per account × store)
insert_cols = ['import_id', 'report_month', 'source_file', 'source_sheet',
'source_row', 'source_column', 'account_code', 'account_name',
'accounting_direction', 'is_summary_account', 'parent_account_code',
'cost_unit_source_header', 'cost_unit_source_code',
'cost_unit_source_name', 'amount']
col_names = ','.join(insert_cols)
batch = []
total_records = 0
for row_idx, (_, r) in enumerate(df.iterrows()):
account_code = to_text(r[0])
account_name = to_text(r[1])
direction = to_text(r[2]) # 统计方式 (借方/贷方)
excel_row = int(r.name) + 1 # 1-indexed Excel row
# Determine if summary account: 7-digit codes (5032401) are sub-accounts of 5-digit (50324)
is_summary = False
parent_code = None
if account_code and len(account_code) == 7:
parent_code = account_code[:5]
is_summary = False
elif account_code and len(account_code) == 5:
is_summary = True
for col_idx, store_code, store_name in store_cols:
amount = to_num(r[col_idx]) if col_idx < len(r) and pd.notna(r[col_idx]) else None
if amount is None:
continue # Skip NULL amounts, but keep 0 amounts
vals = (
import_id,
report_month,
source_file,
'Worksheet',
excel_row,
col_idx + 1, # 1-indexed column
account_code,
account_name,
direction,
is_summary,
parent_code,
headers[col_idx], # full header text
store_code,
store_name,
amount,
)
batch.append(vals)
total_records += 1
if len(batch) >= 500:
execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {total_records} 条记录")
batch = []
if batch:
execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {total_records} 条记录")
# Update log
cur.execute(
"UPDATE public.operating_expense_import_log SET imported_record_count = %s WHERE import_id = %s",
(total_records, import_id)
)
conn.commit()
print(f"营业费用导入完成: {total_records} 条记录")
# ============================================================
# 中央厨房导入 (4表)
# ============================================================
def import_central_kitchen(conn, filepath, report_month, ck_type='finished'):
"""中央厨房导入: finished/recipe/material/processing
使用pandas读取,精确映射列,raw_data存储原始行JSON"""
import json
type_names = {
'finished': '完工入库',
'recipe': '配方耗用',
'material': '材料日报',
'processing': '加工单价',
}
table_map = {
'finished': 'central_kitchen_finished_receipt',
'recipe': 'central_kitchen_recipe_consumption',
'material': 'central_kitchen_material_daily',
'processing': 'central_kitchen_processing_cost',
}
table_name = table_map[ck_type]
print(f"\n=== 中央厨房-{type_names.get(ck_type, ck_type)} 导入 ===")
print(f"文件: {filepath}")
sha = file_sha256(filepath)
source_file = os.path.basename(filepath)
cur = conn.cursor()
cur.execute(
"SELECT 1 FROM public.central_kitchen_import_log WHERE source_file = %s AND file_sha256 = %s",
(source_file, sha)
)
if cur.fetchone():
print("已导入过,跳过")
return
# Header row varies by type:
# finished: row 2 (0-indexed) with sub-header row 3
# recipe: row 0
# material: row 0 with sub-header row 1
# processing: row 2
header_rows = {
'finished': 2, # 0-indexed, merged header rows 2+3
'recipe': 0,
'material': 1, # skip row 0 (group header), use row 1
'processing': 2,
}
hr = header_rows[ck_type]
# Read raw without header to get all rows
raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
# Build headers from the appropriate row(s)
if ck_type == 'finished':
# Row 2 has group headers, row 3 has sub-headers
# Don't fill forward - empty cells stay empty (merged cells)
h1 = raw_df.iloc[2].fillna('').astype(str).str.strip()
h2 = raw_df.iloc[3].fillna('').astype(str).str.strip()
headers = []
for i in range(len(h1)):
g = h1[i]
s = h2[i]
if g and s:
headers.append(f"{g}_{s}")
elif g:
headers.append(g)
elif s:
headers.append(s)
else:
headers.append(f'col_{i}') # Placeholder for empty headers
data_start = 4
elif ck_type == 'material':
# Row 0 has group headers, row 1 has sub-headers
h1 = raw_df.iloc[0].fillna('').astype(str).str.strip()
h2 = raw_df.iloc[1].fillna('').astype(str).str.strip()
headers = []
current_group = ''
for i in range(len(h1)):
g = h1[i] if h1[i] else current_group
if g:
current_group = g
s = h2[i] if h2[i] else ''
headers.append(f"{g}_{s}" if s else g)
data_start = 2
else:
# processing: row 2 (main) + row 3 (sub) — like finished
h1 = raw_df.iloc[hr].fillna('').astype(str).str.strip()
h2 = raw_df.iloc[hr + 1].fillna('').astype(str).str.strip()
headers = []
for i in range(len(h1)):
g = h1[i]
s = h2[i]
if g and s and g != s:
headers.append(f"{g}_{s}")
elif g and g != 'nan':
headers.append(g)
elif s and s != 'nan':
headers.append(s)
else:
headers.append(f'col_{i}')
data_start = hr + 2
# Extract data rows
df = raw_df.iloc[data_start:]
df = df.dropna(subset=[df.columns[0]])
# Filter out 合计/total rows by checking all string cells for '合计'
mask = pd.Series([True] * len(df), index=df.index)
for ci in range(min(15, df.shape[1])):
mask = mask & ~df[ci].astype(str).str.strip().isin(['合计', '总计', 'Total', 'total'])
# Filter out footer rows (e.g. "制表人:xxx")
mask = mask & ~df[0].astype(str).str.strip().str.startswith('制表人')
mask = mask & ~df[0].astype(str).str.strip().str.startswith('打印')
mask = mask & ~df[0].astype(str).str.strip().str.startswith('')
df = df[mask]
data_rows = df.values.tolist()
print(f"数据行数: {len(data_rows)}")
if not data_rows:
return
# Insert import log
cur.execute(
"""INSERT INTO public.central_kitchen_import_log
(report_month, dataset_type, source_file, source_sheet, file_sha256,
workbook_rows, imported_rows, excluded_rows, status)
VALUES (%s, %s, %s, %s, %s, %s, %s, 0, 'loading') RETURNING import_id""",
(report_month, ck_type, source_file, 'Worksheet', sha, len(data_rows), len(data_rows))
)
import_id = cur.fetchone()[0]
# Get DB columns (exclude record_id, import_id, source_row, raw_data)
cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{table_name}' ORDER BY ordinal_position")
db_cols = [r[0] for r in cur.fetchall()]
insertable_cols = [c for c in db_cols if c not in ('record_id', 'import_id', 'source_row', 'raw_data')]
# Build header name -> excel index map (keep first occurrence)
col_map = {}
for i, h in enumerate(headers):
if h and str(h).strip():
key = str(h).strip()
if key not in col_map:
col_map[key] = i
# Type-specific column mappings (db_col -> list of possible header names)
mappings = {
'finished': {
'receipt_date': ['入库日期'],
'workshop': ['col_1'],
'processing_warehouse': ['加工仓库'],
'recipe_name': ['配方名称'],
'product_name': ['货品名称'],
'specification': ['规格'],
'unit': ['库存单位'],
'return_quantity': ['退库数量'],
'return_amount': ['退库金额'],
'inbound_avg_unit_price': ['平均单价'],
'inbound_quantity': ['入库数量'],
'inbound_amount': ['入库金额'],
'raw_material_cost': ['加工利润分析_原料成本', '原料成本'],
'source_fee_cost': ['费用成本'],
'source_processing_profit': ['增值金额(加工利润)'],
'source_margin_rate': ['毛利率'],
'source_cost_rate': ['成本率'],
'theoretical_inbound_quantity': ['加工达成率分析_理论入库数量', '理论入库数量'],
'actual_inbound_quantity': ['实际入库数量'],
'inbound_difference_quantity': ['差异数量'],
'achievement_rate': ['加工达成率'],
'standard_expected_quantity': ['加工应产量分析_标准应产量', '标准应产量'],
'average_expected_quantity': ['平均应产量'],
'expected_quantity_variance_rate': ['应产量差异量比'],
'water_cost': ['水费'],
'electricity_cost': ['电费'],
'gas_cost': ['燃气费'],
'labor_cost': ['人工费'],
'other_cost': ['其他费用'],
'detailed_fee_total': ['费用合计'],
},
'recipe': {
'business_date': ['日期'],
'processing_warehouse': ['加工仓库'],
'finance_category_major': ['财务大类'],
'finance_category_minor': ['财务小类'],
'category_major': ['所属大类'],
'category_minor': ['所属小类'],
'item_id': ['货品ID'],
'item_code': ['编码'],
'item_name': ['货品名称'],
'specification': ['规格'],
'unit': ['库存单位'],
'unit_price_excl_tax': ['未税单价'],
'unit_price': ['单价'],
'recipe_name': ['配方名称'],
'standard_usage': ['标准用量'],
'actual_average_usage': ['实际平均用量'],
'planned_output_quantity': ['计划加工数量'],
'actual_output_quantity': ['实际加工数量'],
'output_unit': ['加工数量库存单位'],
'theoretical_quantity': ['理论用量'],
'net_quantity': ['净料用量'],
'theoretical_amount': ['理论金额'],
'issue_quantity': ['领用数量'],
'issue_amount': ['领用金额'],
'source_inventory_cost_quantity': ['成本数量(进销存)'],
'source_inventory_cost_amount': ['成本金额(进销存)'],
'actual_vs_theory_quantity': ['实际与理论差异数量'],
'actual_vs_theory_amount': ['实际与理论差异金额'],
'actual_vs_theory_rate': ['实际差异量比'],
'issue_vs_theory_quantity': ['领用与理论差异数量'],
'issue_vs_theory_amount': ['领用与理论差异金额'],
'issue_vs_theory_rate': ['领用差异量比'],
'actual_yield': ['实际出成率'],
'recipe_yield': ['配方出成率'],
'yield_difference': ['出成率差异'],
},
'material': {
'business_date': ['日期'],
'finance_category_major': ['财务大类'],
'finance_category_minor': ['财务小类'],
'category_major': ['所属大类'],
'category_minor': ['所属小类'],
'item_id': ['货品ID'],
'item_code': ['编码'],
'item_name': ['货品名称'],
'specification': ['规格'],
'unit': ['库存单位'],
},
'processing': {
'report_month': ['report_month'], # Will be set from parameter
'item_id': ['货品ID'],
'recipe_name': ['配方名称'],
'product_code': ['货品编码'],
'product_name': ['货品名称'],
'category_major': ['所属大类'],
'category_minor': ['所属小类'],
'finance_category_major': ['财务大类'],
'finance_category_minor': ['财务小类'],
'specification': ['规格'],
'unit': ['单位'],
'inbound_quantity': ['入库数量'],
'inbound_avg_unit_price': ['入库均价'],
'theoretical_unit_cost': ['理论单价'],
'standard_unit_cost': ['标准单价'],
'actual_unit_cost': ['实际单价'],
'theoretical_cost': ['理论成本'],
'standard_cost': ['标准成本'],
'actual_cost': ['实际成本'],
},
}
field_mapping = mappings[ck_type]
cols_to_insert = {}
fixed_values = {} # Values not from Excel (e.g. report_month)
for db_col, h_names in field_mapping.items():
if db_col not in insertable_cols:
continue
found = False
for h_name in h_names:
if h_name in col_map:
cols_to_insert[db_col] = col_map[h_name]
found = True
break
if not found and db_col == 'report_month':
fixed_values['report_month'] = report_month
# Determine numeric columns from DB schema
cur.execute(f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name='{table_name}'")
col_types = {r[0]: r[1] for r in cur.fetchall()}
numeric_db_cols = {c for c, t in col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')}
date_db_cols = {c for c, t in col_types.items() if t == 'date'}
all_cols = ['import_id', 'source_row'] + list(cols_to_insert.keys()) + list(fixed_values.keys()) + ['raw_data']
col_names = ','.join(all_cols)
batch = []
for idx, r in enumerate(data_rows):
source_row = idx + data_start + 1 # 1-indexed Excel row
vals = [import_id, source_row]
for db_col, excel_idx in cols_to_insert.items():
v = r[excel_idx] if excel_idx < len(r) else None
if db_col in numeric_db_cols:
vals.append(to_num(v) if v is not None else None)
elif db_col in date_db_cols:
vals.append(to_date(v))
else:
vals.append(to_text(v))
# Fixed values (not from Excel)
for fv_col in fixed_values:
if fv_col in date_db_cols:
vals.append(to_date(fixed_values[fv_col]))
else:
vals.append(fixed_values[fv_col])
# raw_data: store entire row as JSON
raw_dict = {}
for hi, hv in enumerate(headers):
if hv and hi < len(r):
rv = r[hi]
if pd.notna(rv):
raw_dict[str(hv)] = str(rv) if not isinstance(rv, (int, float)) else rv
vals.append(json.dumps(raw_dict, ensure_ascii=False))
batch.append(tuple(vals))
if len(batch) >= 500:
execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {idx + 1}/{len(data_rows)}")
batch = []
if batch:
execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {len(data_rows)}/{len(data_rows)}")
# Update log status
cur.execute(
"UPDATE public.central_kitchen_import_log SET status = 'success', completed_at = now() WHERE import_id = %s",
(import_id,)
)
conn.commit()
print(f"中央厨房{type_names[ck_type]}导入完成: {len(data_rows)}")
# ============================================================
# 配送明细导入
# ============================================================
def import_distribution(conn, filepath, report_month):
"""配送明细导入(支持单文件或目录)"""
import json
print(f"\n=== 配送明细导入 ===")
files = []
if os.path.isdir(filepath):
for f in sorted(os.listdir(filepath)):
if f.endswith('.xlsx') and not f.startswith('.'):
files.append(os.path.join(filepath, f))
else:
files.append(filepath)
print(f"文件数: {len(files)}")
cur = conn.cursor()
total_imported = 0
for file_idx, fpath in enumerate(files):
fname = os.path.basename(fpath)
print(f"\n[{file_idx+1}/{len(files)}] {fname}")
sha = file_sha256(fpath)
cur.execute(
"SELECT 1 FROM public.distribution_import_log WHERE source_file = %s AND file_sha256 = %s",
(fname, sha)
)
if cur.fetchone():
print(" 已导入过, 跳过")
continue
# Read with pandas, header on row 0
raw_df = pd.read_excel(fpath, header=None, engine='openpyxl')
headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]]
df = raw_df.iloc[1:]
df = df.dropna(subset=[df.columns[0]])
data_rows = df.values.tolist()
print(f" 数据行数: {len(data_rows)}")
if not data_rows:
continue
# Parse period from filename or use report_month
import re
date_match = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日.*?(\d{1,2})日', fname)
if date_match:
year, month, start_day, end_day = date_match.groups()
period_start = f"{year}-{int(month):02d}-{int(start_day):02d}"
period_end = f"{year}-{int(month):02d}-{int(end_day):02d}"
else:
# Use full month
period_start = report_month
# Last day of month
rm_date = pd.Timestamp(report_month)
period_end = (rm_date + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d')
cur.execute(
"""INSERT INTO public.distribution_import_log
(source_file, source_sheet, file_sha256, workbook_rows, summary_rows_excluded,
imported_rows, report_month, period_start, period_end, status)
VALUES (%s, %s, %s, %s, 0, %s, %s, %s, %s, 'loading') RETURNING import_id""",
(fname, 'Worksheet', sha, len(data_rows), len(data_rows), report_month, period_start, period_end)
)
import_id = cur.fetchone()[0]
# Get DB columns
cur.execute("SELECT column_name, data_type FROM information_schema.columns WHERE table_name='distribution_detail_records' ORDER BY ordinal_position")
db_col_types = {r[0]: r[1] for r in cur.fetchall()}
insertable_cols = [c for c in db_col_types if c not in ('record_id', 'import_id', 'source_row', 'raw_data')]
# Build header map
col_map = {}
for i, h in enumerate(headers):
if h and str(h).strip():
col_map[str(h).strip()] = i
# Known mappings for distribution based on actual Excel headers
field_mapping = {
'business_date': ['日期', '业务日期'],
'distribution_center_code': ['配送中心编码'],
'distribution_center_name': ['配送中心'],
'distribution_center_company': ['配送中心所属公司'],
'store_company': ['门店所属公司'],
'store_type': ['门店类型'],
'store_brand': ['门店品牌'],
'region_level1': ['一级区域'],
'terminal_region': ['末级区域'],
'store_code': ['门店编码'],
'store_name': ['门店'],
'store_tax_entity': ['门店财税主体'],
'original_batch_unit_price': ['原始批次单价'],
'cost_avg_unit_price': ['成本平均单价'],
'cost_total_amount': ['成本总金额'],
'team': ['班组'],
'cost_excl_tax_amount': ['成本未含税金额'],
'cost_tax_amount': ['成本税额'],
'cost_excl_tax_unit_price': ['成本未含税单价'],
'outbound_tax_amount': ['配出税额'],
'distribution_sales_tax_rate': ['配送销售税点'],
'route': ['路线'],
'profit_amount': ['利润'],
'profit_excl_tax_amount': ['未含税利润', '利润未含税'],
'distribution_type': ['配送类型'],
'logistics_attribute': ['物流属性'],
'item_id': ['货品ID'],
'item_name': ['货品名称', '货品'],
'item_alias': ['货品别名'],
'specification': ['规格'],
'item_purpose': ['货品用途', '用途'],
'unit': ['单位'],
'normal_quantity': ['正常品数量', '正常数量'],
'gift_quantity': ['赠品数量'],
'total_quantity': ['总数量', '合计数量'],
'outbound_avg_unit_price': ['配出平均单价', '配出均价'],
'outbound_total_amount': ['配出总金额', '配出金额'],
'outbound_margin_rate': ['配出毛利率'],
'outbound_excl_tax_unit_price': ['配出未含税单价'],
'outbound_excl_tax_amount': ['配出未含税金额'],
'document_no': ['单号'],
'requisition_no': ['要货单号', '申请单号'],
'payment_method': ['支付方式', '结算方式'],
'value_added_coefficient': ['增值系数'],
'audit_time': ['审核时间'],
'inbound_batch': ['入库批次'],
'inbound_batch_tax_rate': ['入库批次税点', '入库批次税率'],
'custom_batch_code': ['自定义批次码', '自定义批次编码'],
'invoice_type': ['发票种类', '发票类型'],
'production_time': ['生产时间'],
'shelf_life': ['保质期'],
'supplier_batch': ['供应商批次'],
'supplier_code': ['供应商编码'],
'supplier_category': ['供应商类别', '供应商分类'],
'supplier_name': ['供应商'],
'supplier_contact': ['供应商联系人'],
'supplier_phone': ['供应商联系电话', '供应商电话'],
'supplier_address': ['供应商地址'],
'from_warehouse': ['调出仓库', '发货仓库'],
'to_warehouse': ['调入仓库', '收货仓库'],
'item_code': ['货品编码'],
'item_barcode': ['货品条形码', '货品条码'],
'minor_category': ['所属小类', '小类'],
'major_category': ['所属大类', '大类'],
'internal_brand': ['内部品牌'],
'finance_category': ['财务分类'],
'created_time': ['制单时间', '创建时间'],
'creator': ['制单人', '创建人'],
'auditor': ['审核人'],
'arrival_time': ['到货时间'],
'document_remark': ['单据备注'],
'document_item_remark': ['单据项备注'],
'reason_remark': ['原因备注'],
'business_date': ['业务日期'],
'contact_address': ['联系地址'],
'contact_phone': ['联系电话'],
'contact_person': ['联系人'],
}
cols_to_insert = {}
for db_col, h_names in field_mapping.items():
if db_col not in insertable_cols:
continue
for h_name in h_names:
if h_name in col_map:
cols_to_insert[db_col] = col_map[h_name]
break
numeric_db_cols = {c for c, t in db_col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')}
date_db_cols = {c for c, t in db_col_types.items() if t == 'date'}
all_cols = ['import_id', 'source_row', 'report_month', 'source_file', 'source_sheet'] + list(cols_to_insert.keys())
col_names = ','.join(all_cols)
batch = []
for idx, r in enumerate(data_rows):
source_row = idx + 2 # 1-indexed, data starts at row 2
vals = [import_id, source_row, report_month, fname, 'Worksheet']
for db_col, excel_idx in cols_to_insert.items():
v = r[excel_idx] if excel_idx < len(r) else None
if db_col in numeric_db_cols:
vals.append(to_num(v) if v is not None else None)
elif db_col in date_db_cols:
vals.append(to_date(v))
else:
vals.append(to_text(v))
batch.append(tuple(vals))
if len(batch) >= 500:
execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {idx + 1}/{len(data_rows)}")
batch = []
if batch:
execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500)
conn.commit()
print(f" 已导入 {len(data_rows)}/{len(data_rows)}")
total_imported += len(data_rows)
# Update log status
cur.execute(
"UPDATE public.distribution_import_log SET status = 'success', completed_at = now() WHERE import_id = %s",
(import_id,)
)
conn.commit()
print(f"\n配送明细导入完成: 共 {total_imported}")
# ============================================================
# Main
# ============================================================
def main():
parser = argparse.ArgumentParser(description='月度数据导入工具')
parser.add_argument('--dataset', required=True,
choices=['inventory', 'dish_sales', 'dish_cost', 'operating_expense',
'central_kitchen', 'distribution'],
help='数据域')
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
parser.add_argument('--file', required=True, help='文件路径或目录')
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
parser.add_argument('--ck-type', default='finished',
choices=['finished', 'recipe', 'material', 'processing'],
help='中央厨房子类型')
parser.add_argument('--expense-type', default='operating',
choices=['operating', 'management'],
help='费用类型')
args = parser.parse_args()
conn = connect(args.db)
conn.autocommit = False
try:
if args.dataset == 'inventory':
import_inventory_cost(conn, args.file, args.month)
elif args.dataset == 'dish_sales':
import_dish_sales(conn, args.file, args.month)
elif args.dataset == 'dish_cost':
import_dish_cost(conn, args.file, args.month)
elif args.dataset == 'operating_expense':
import_operating_expense(conn, args.file, args.month, args.expense_type)
elif args.dataset == 'central_kitchen':
import_central_kitchen(conn, args.file, args.month, args.ck_type)
elif args.dataset == 'distribution':
import_distribution(conn, args.file, args.month)
print("\n=== 导入完成 ===")
except Exception as e:
conn.rollback()
print(f"错误: {e}", file=sys.stderr)
raise
finally:
conn.close()
if __name__ == '__main__':
main()