2fc1b96f44
- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径,支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
1302 lines
54 KiB
Python
1302 lines
54 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
通用月度数据导入脚本
|
||
支持: 库存成本、菜品销售明细、菜品成本BOM、营业费用、中央厨房、配送明细、采购货品
|
||
用法: python3 import_monthly_data.py --dataset inventory --month 2026-04-01 --file <path> --db bill_query_test
|
||
"""
|
||
import argparse
|
||
import hashlib
|
||
import os
|
||
import sys
|
||
import openpyxl
|
||
import pandas as pd
|
||
import psycopg2
|
||
from psycopg2.extras import execute_values
|
||
from datetime import datetime
|
||
|
||
DB_CONFIG = {
|
||
'host': 'localhost',
|
||
'port': 5432,
|
||
'user': 'freedak',
|
||
'password': '',
|
||
}
|
||
|
||
def file_sha256(filepath):
|
||
h = hashlib.sha256()
|
||
with open(filepath, 'rb') as f:
|
||
for chunk in iter(lambda: f.read(8192), b''):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
def to_num(val):
|
||
if val is None or val == '':
|
||
return None
|
||
if isinstance(val, float) and pd.isna(val):
|
||
return None
|
||
try:
|
||
f = float(val)
|
||
import math
|
||
if math.isinf(f) or math.isnan(f):
|
||
return None
|
||
return f
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
def to_text(val):
|
||
if val is None:
|
||
return None
|
||
if isinstance(val, float) and pd.isna(val):
|
||
return None
|
||
s = str(val).strip()
|
||
if s.lower() == 'nan' or s.lower() == 'none':
|
||
return None
|
||
return s if s else None
|
||
|
||
def to_date(val):
|
||
"""Convert value to date string YYYY-MM-DD or None"""
|
||
if val is None:
|
||
return None
|
||
if isinstance(val, float) and pd.isna(val):
|
||
return None
|
||
if isinstance(val, (datetime,)):
|
||
return val.strftime('%Y-%m-%d')
|
||
s = str(val).strip()
|
||
if not s or s.lower() == 'nan' or s.lower() == 'none':
|
||
return None
|
||
# Try pandas Timestamp
|
||
try:
|
||
ts = pd.Timestamp(s)
|
||
if pd.isna(ts):
|
||
return None
|
||
return ts.strftime('%Y-%m-%d')
|
||
except (ValueError, TypeError):
|
||
return s # Return as-is, let DB handle it
|
||
|
||
def clean_percent(val):
|
||
"""Convert percentage value: string '81.23%' -> 81.23, float 0.8123 -> 81.23"""
|
||
if val is None:
|
||
return None
|
||
if isinstance(val, float) and pd.isna(val):
|
||
return None
|
||
if isinstance(val, str):
|
||
s = val.strip().replace('%', '')
|
||
if not s or s.lower() == 'nan':
|
||
return None
|
||
try:
|
||
return float(s)
|
||
except ValueError:
|
||
return None
|
||
num = float(val)
|
||
return num * 100 if abs(num) <= 1 else num
|
||
|
||
def connect(dbname='bill_query'):
|
||
cfg = DB_CONFIG.copy()
|
||
cfg['dbname'] = dbname
|
||
return psycopg2.connect(**cfg)
|
||
|
||
# ============================================================
|
||
# 库存成本导入
|
||
# ============================================================
|
||
def import_inventory_cost(conn, filepath, report_month):
|
||
"""库存倒挤成本导入"""
|
||
print(f"\n=== 库存成本导入 ===")
|
||
print(f"文件: {filepath}")
|
||
print(f"月份: {report_month}")
|
||
|
||
sha = file_sha256(filepath)
|
||
source_file = os.path.basename(filepath)
|
||
|
||
wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
|
||
ws = wb.active
|
||
|
||
# Row 2: group headers, Row 3: sub-headers, Row 4+: data
|
||
rows = list(ws.iter_rows(min_row=4, values_only=True))
|
||
data_rows = [r for r in rows if r[4] is not None and str(r[4]).strip()]
|
||
print(f"数据行数: {len(data_rows)}")
|
||
|
||
cur = conn.cursor()
|
||
# Check if already imported
|
||
cur.execute(
|
||
"SELECT 1 FROM public.inventory_cost_import_log WHERE source_file = %s AND report_month = %s",
|
||
(source_file, report_month)
|
||
)
|
||
if cur.fetchone():
|
||
print("已导入过,跳过")
|
||
wb.close()
|
||
return
|
||
|
||
# Insert import log
|
||
cur.execute(
|
||
"""INSERT INTO public.inventory_cost_import_log (source_file, report_month, source_rows, imported_rows, status)
|
||
VALUES (%s, %s, %s, %s, 'importing')""",
|
||
(source_file, report_month, len(data_rows), len(data_rows))
|
||
)
|
||
|
||
# Batch insert
|
||
insert_sql = """INSERT INTO public.inventory_cost_records (
|
||
report_month, source_file, source_row,
|
||
brand, region, cost_unit_name, cost_unit_code, item_name, specification,
|
||
major_category, minor_category, finance_category, item_code, unit,
|
||
opening_quantity, opening_unit_cost, opening_amount, opening_tax, opening_tax_inclusive,
|
||
purchase_quantity, purchase_unit_cost, purchase_amount, purchase_tax, purchase_tax_inclusive,
|
||
ending_quantity, ending_unit_cost, ending_amount, ending_tax, ending_tax_inclusive,
|
||
consumption_quantity, consumption_unit_cost, consumption_amount, consumption_tax, consumption_tax_inclusive,
|
||
conversion_precision_amount, conversion_precision_tax_inclusive,
|
||
return_loss_amount, return_loss_tax_inclusive
|
||
) VALUES %s"""
|
||
|
||
batch = []
|
||
batch_size = 500
|
||
for idx, r in enumerate(data_rows):
|
||
source_row = idx + 4
|
||
consumption_amount = to_num(r[28])
|
||
vals = (
|
||
report_month, source_file, source_row,
|
||
to_text(r[0]), to_text(r[1]), to_text(r[2]), to_text(r[3]),
|
||
to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]),
|
||
to_text(r[9]), to_text(r[10]),
|
||
to_num(r[11]), to_num(r[12]), to_num(r[13]), to_num(r[14]), to_num(r[15]),
|
||
to_num(r[16]), to_num(r[17]), to_num(r[18]), to_num(r[19]), to_num(r[20]),
|
||
to_num(r[21]), to_num(r[22]), to_num(r[23]), to_num(r[24]), to_num(r[25]),
|
||
to_num(r[26]), to_num(r[27]), consumption_amount, to_num(r[29]), to_num(r[30]),
|
||
to_num(r[31]), to_num(r[32]),
|
||
to_num(r[33]), to_num(r[34]),
|
||
)
|
||
batch.append(vals)
|
||
if len(batch) >= batch_size:
|
||
execute_values(cur, insert_sql, batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, insert_sql, batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||
|
||
# Update log status
|
||
cur.execute(
|
||
"UPDATE public.inventory_cost_import_log SET status = 'completed' WHERE source_file = %s AND report_month = %s",
|
||
(source_file, report_month)
|
||
)
|
||
conn.commit()
|
||
wb.close()
|
||
print(f"库存成本导入完成: {len(data_rows)} 行")
|
||
|
||
# ============================================================
|
||
# 菜品销售明细导入
|
||
# ============================================================
|
||
def import_dish_sales(conn, filepath, report_month, source_file_label=None):
|
||
"""菜品销售明细导入(支持单文件或目录)"""
|
||
print(f"\n=== 菜品销售明细导入 ===")
|
||
|
||
files = []
|
||
if os.path.isdir(filepath):
|
||
for f in sorted(os.listdir(filepath)):
|
||
if f.endswith('.xlsx') and not f.startswith('.'):
|
||
files.append(os.path.join(filepath, f))
|
||
else:
|
||
files.append(filepath)
|
||
|
||
print(f"文件数: {len(files)}")
|
||
|
||
cur = conn.cursor()
|
||
total_imported = 0
|
||
|
||
for file_idx, fpath in enumerate(files):
|
||
fname = os.path.basename(fpath)
|
||
print(f"\n[{file_idx+1}/{len(files)}] {fname}")
|
||
|
||
file_size = os.path.getsize(fpath)
|
||
|
||
# Check if already imported
|
||
cur.execute(
|
||
"SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s",
|
||
(fname,)
|
||
)
|
||
if cur.fetchone():
|
||
print(f" 已导入过, 跳过")
|
||
continue
|
||
|
||
# Use pandas to read Excel (handles read_only issues with these files)
|
||
# Header is on row 3 (0-indexed: header=2)
|
||
df = pd.read_excel(fpath, header=2, engine='openpyxl')
|
||
# Drop rows where first column is empty
|
||
df = df.dropna(subset=[df.columns[0]])
|
||
data_rows = df.values.tolist()
|
||
headers = list(df.columns)
|
||
print(f" 数据行数: {len(data_rows)}")
|
||
|
||
if not data_rows:
|
||
continue
|
||
|
||
# Insert import log
|
||
cur.execute(
|
||
"""INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status)
|
||
VALUES (%s, %s, %s, 'importing')""",
|
||
(fname, file_size, len(data_rows))
|
||
)
|
||
|
||
# Map headers to columns by name
|
||
col_map = {}
|
||
for i, h in enumerate(headers):
|
||
if h and str(h).strip():
|
||
col_map[str(h).strip()] = i
|
||
|
||
# Build column mapping based on known header names
|
||
field_mapping = {
|
||
'门店编码': 'store_code',
|
||
'门店名称': 'store_name',
|
||
'门店': 'store_name',
|
||
'账单号': 'bill_no',
|
||
'订单号': 'bill_no',
|
||
'人数': 'guest_count',
|
||
'开台时间': 'opened_at',
|
||
'结账时间': 'closed_at',
|
||
'下单时间': 'ordered_at',
|
||
'菜品名称': 'dish_name',
|
||
'品项名称': 'dish_name',
|
||
'分类': 'category_level1',
|
||
'菜品分类': 'category_level1',
|
||
'出品部门': 'production_department',
|
||
'销量': 'sales_quantity',
|
||
'金额': 'gross_amount',
|
||
'实收金额': 'received_amount',
|
||
'实收': 'received_amount',
|
||
'单价': 'unit_price',
|
||
'单位': 'unit',
|
||
'业务类型': 'business_type',
|
||
'桌台/取餐号': 'table_or_pickup_no',
|
||
'台号': 'table_or_pickup_no',
|
||
'做法': 'preparation_method',
|
||
}
|
||
|
||
# Determine actual columns
|
||
cols_to_insert = {}
|
||
for h_name, db_col in field_mapping.items():
|
||
if h_name in col_map:
|
||
cols_to_insert[db_col] = col_map[h_name]
|
||
|
||
# source_file + source_row + mapped columns
|
||
all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys())
|
||
col_names = ','.join(all_cols)
|
||
|
||
numeric_cols = {'guest_count', 'sales_quantity', 'gross_amount', 'received_amount', 'unit_price'}
|
||
|
||
batch = []
|
||
for idx, r in enumerate(data_rows):
|
||
source_row = idx + 4 # data starts at row 4 (header=3)
|
||
vals = [fname, source_row]
|
||
for db_col, excel_idx in cols_to_insert.items():
|
||
v = r[excel_idx] if excel_idx < len(r) else None
|
||
if db_col in numeric_cols:
|
||
vals.append(to_num(v))
|
||
else:
|
||
vals.append(to_text(v))
|
||
batch.append(tuple(vals))
|
||
if len(batch) >= 500:
|
||
execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||
|
||
total_imported += len(data_rows)
|
||
|
||
# Update log status
|
||
cur.execute(
|
||
"UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s",
|
||
(fname,)
|
||
)
|
||
conn.commit()
|
||
|
||
print(f"\n菜品销售明细导入完成: 共 {total_imported} 行")
|
||
|
||
# ============================================================
|
||
# 菜品成本/BOM导入
|
||
# ============================================================
|
||
def import_dish_cost(conn, filepath, report_month):
|
||
"""菜品成本分析导入:同时导入summary和detail两张表
|
||
Excel结构:Row0=标题, Row1=主表头, Row2=子表头(原料区), Row3+=数据
|
||
每个菜品有多行原料明细,菜品列(col0-12)只在首行填充"""
|
||
print(f"\n=== 菜品成本/BOM导入 ===")
|
||
print(f"文件: {filepath}")
|
||
|
||
sha = file_sha256(filepath)
|
||
source_file = os.path.basename(filepath)
|
||
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"SELECT 1 FROM public.dish_cost_analysis_import_log WHERE source_file = %s AND file_sha256 = %s",
|
||
(source_file, sha)
|
||
)
|
||
if cur.fetchone():
|
||
print("已导入过,跳过")
|
||
return
|
||
|
||
# Read with pandas
|
||
raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
|
||
print(f" 总行数: {len(raw_df)}")
|
||
|
||
# Headers: Row 1 (main) + Row 2 (sub for material detail cols 13+)
|
||
main_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[1]]
|
||
sub_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[2]]
|
||
|
||
# Build combined headers
|
||
headers = []
|
||
for i in range(len(main_headers)):
|
||
m = main_headers[i]
|
||
s = sub_headers[i]
|
||
if m and s and m != s:
|
||
headers.append(f"{m}_{s}")
|
||
elif m:
|
||
headers.append(m)
|
||
elif s:
|
||
headers.append(s)
|
||
else:
|
||
headers.append(f'col_{i}')
|
||
|
||
data_start = 3
|
||
df = raw_df.iloc[data_start:].copy()
|
||
|
||
# Identify summary rows: col 0 originally non-empty (first row of each dish)
|
||
# Detail rows: col 13 non-empty (material name)
|
||
summary_mask = df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan')
|
||
detail_mask = df[13].notna() & (df[13].astype(str).str.strip() != '') & (df[13].astype(str).str.strip() != 'nan')
|
||
|
||
summary_df = df[summary_mask].copy()
|
||
detail_df = df[detail_mask].copy()
|
||
|
||
# Filter out 合计 rows
|
||
summary_df = summary_df[~summary_df[0].astype(str).str.strip().isin(['合计', '总计'])]
|
||
|
||
print(f" 菜品汇总行数: {len(summary_df)}")
|
||
print(f" 原料明细行数: {len(detail_df)}")
|
||
|
||
if len(summary_df) == 0:
|
||
print("无菜品汇总数据,跳过")
|
||
return
|
||
|
||
# Calculate totals for log
|
||
sales_amount_total = to_num(summary_df[7].sum()) if 7 < len(summary_df.columns) else 0
|
||
theoretical_cost_total = to_num(summary_df[8].sum()) if 8 < len(summary_df.columns) else 0
|
||
actual_cost_total = to_num(summary_df[9].sum()) if 9 < len(summary_df.columns) else 0
|
||
cost_variance_total = to_num(summary_df[12].sum()) if 12 < len(summary_df.columns) else 0
|
||
|
||
# Parse report period from filename or use report_month
|
||
import re
|
||
date_match = re.search(r'(\d{4})年(\d{1,2})月', source_file)
|
||
if date_match:
|
||
year, month = date_match.groups()
|
||
period_start = f"{year}-{int(month):02d}-01"
|
||
period_end = (pd.Timestamp(period_start) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d')
|
||
else:
|
||
period_start = report_month
|
||
period_end = (pd.Timestamp(report_month) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d')
|
||
|
||
# Insert import log
|
||
cur.execute(
|
||
"""INSERT INTO public.dish_cost_analysis_import_log
|
||
(source_file, file_sha256, sheet_name, workbook_data_rows,
|
||
dish_summary_rows, material_detail_rows,
|
||
sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total,
|
||
report_period_start, report_period_end, import_status, report_month)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'loading', %s) RETURNING import_id""",
|
||
(source_file, sha, 'Worksheet', len(df), len(summary_df), len(detail_df),
|
||
sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total,
|
||
period_start, period_end, report_month)
|
||
)
|
||
import_id = cur.fetchone()[0]
|
||
|
||
# Import summary rows
|
||
summary_mapping = {
|
||
'dish_name': 0, # 菜品名称
|
||
'dish_code': 1, # 菜品编码
|
||
'price': 2, # 售价
|
||
'category_level1': 3, # 菜品大类
|
||
'category_level2': 4, # 菜品小类
|
||
'dish_unit': 5, # 菜品单位
|
||
'sales_quantity': 6, # 销售数量
|
||
'sales_amount': 7, # 销售金额(元)
|
||
'theoretical_cost': 8, # 理论成本(元)
|
||
'actual_cost': 9, # 实际成本(元)
|
||
'theoretical_margin_rate_pct': 10, # 理论毛利率
|
||
'actual_margin_rate_pct': 11, # 实际毛利率
|
||
'cost_variance_amount': 12, # 成本差异金额(元)
|
||
}
|
||
|
||
summary_cols = ['import_id', 'source_row', 'dish_sequence'] + list(summary_mapping.keys())
|
||
summary_col_names = ','.join(summary_cols)
|
||
|
||
# Fields that need clean_percent vs to_num vs to_text
|
||
summary_percent_fields = {'theoretical_margin_rate_pct', 'actual_margin_rate_pct'}
|
||
summary_text_fields = {'dish_name', 'dish_code', 'category_level1', 'category_level2', 'dish_unit'}
|
||
|
||
batch = []
|
||
for idx, (row_idx, r) in enumerate(summary_df.iterrows()):
|
||
source_row = int(row_idx) + data_start + 1 # 1-indexed Excel row
|
||
vals = [import_id, source_row, idx + 1] # dish_sequence
|
||
for db_col, excel_idx in summary_mapping.items():
|
||
v = r[excel_idx] if excel_idx < len(r) else None
|
||
if v is None or (isinstance(v, float) and pd.isna(v)):
|
||
vals.append(None)
|
||
elif db_col in summary_percent_fields:
|
||
vals.append(clean_percent(v))
|
||
elif db_col in summary_text_fields:
|
||
vals.append(to_text(v))
|
||
else:
|
||
vals.append(to_num(v))
|
||
batch.append(tuple(vals))
|
||
if len(batch) >= 500:
|
||
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" Summary: 已导入 {idx + 1}/{len(summary_df)} 行")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" Summary: 已导入 {len(summary_df)}/{len(summary_df)} 行")
|
||
|
||
# Get summary_ids for detail linking
|
||
cur.execute("SELECT summary_id, source_row FROM public.dish_cost_analysis_summary WHERE import_id = %s ORDER BY source_row", (import_id,))
|
||
summary_id_map = {r[1]: r[0] for r in cur.fetchall()}
|
||
|
||
# Import detail rows
|
||
# Detail columns: col13=原料, col14=单位, col15=规格, col16=属性,
|
||
# col17=理论数量, col18=理论金额, col19=实际数量, col20=实际金额,
|
||
# col21=损耗数量, col22=损耗金额, col23=数量量比
|
||
detail_mapping = {
|
||
'material_name': 13,
|
||
'material_unit': 14,
|
||
'specification': 15,
|
||
'material_type': 16,
|
||
'theoretical_quantity': 17,
|
||
'theoretical_amount': 18,
|
||
'actual_quantity': 19,
|
||
'actual_amount': 20,
|
||
'loss_quantity': 21,
|
||
'loss_amount': 22,
|
||
'loss_quantity_rate_pct': 23,
|
||
}
|
||
|
||
detail_cols = ['import_id', 'summary_id', 'source_row', 'material_sequence'] + list(detail_mapping.keys()) + ['theoretical_quantity_per_dish', 'actual_quantity_per_dish']
|
||
detail_col_names = ','.join(detail_cols)
|
||
|
||
# For each detail row, find the parent summary's source_row
|
||
# The parent dish's source_row is the first non-empty col0 at or before this row
|
||
# We already forward-filled col0, so we can match by looking up the original row
|
||
# Actually, we need to find which summary row each detail row belongs to
|
||
# Since we forward-filled, the dish_name (col0) tells us which dish it belongs to
|
||
|
||
# Build a map: dish_name -> summary source_row
|
||
dish_to_summary_row = {}
|
||
for idx, (row_idx, r) in enumerate(summary_df.iterrows()):
|
||
source_row = int(row_idx) + data_start + 1
|
||
dish_name = str(r[0]).strip() if pd.notna(r[0]) else ''
|
||
if dish_name:
|
||
dish_to_summary_row[dish_name] = source_row
|
||
|
||
# For detail rows, find parent dish by looking backward in original data for nearest non-empty col0
|
||
# Build a sorted list of summary source_rows for binary search
|
||
summary_source_rows = sorted(summary_id_map.keys())
|
||
|
||
batch = []
|
||
mat_seq = 0
|
||
current_dish = None
|
||
for idx, (row_idx, r) in enumerate(detail_df.iterrows()):
|
||
source_row = int(row_idx) + data_start + 1
|
||
excel_row_idx = int(row_idx) # 0-indexed in raw_df
|
||
|
||
# Find parent dish: look backward from current row for nearest non-empty col0
|
||
parent_dish_name = None
|
||
for back_idx in range(excel_row_idx, data_start - 1, -1):
|
||
val = raw_df.iloc[back_idx, 0]
|
||
if pd.notna(val) and str(val).strip() and str(val).strip() not in ('合计', '总计', 'nan'):
|
||
parent_dish_name = str(val).strip()
|
||
break
|
||
|
||
if parent_dish_name != current_dish:
|
||
current_dish = parent_dish_name
|
||
mat_seq = 0
|
||
mat_seq += 1
|
||
|
||
# Find summary_id
|
||
parent_source_row = dish_to_summary_row.get(parent_dish_name)
|
||
summary_id = summary_id_map.get(parent_source_row) if parent_source_row else None
|
||
|
||
vals = [import_id, summary_id, source_row, mat_seq]
|
||
# Get sales_quantity from parent summary for per_dish calculation
|
||
parent_sales_qty = None
|
||
if parent_source_row:
|
||
for _, srow in summary_df.iterrows():
|
||
if int(srow.name) + data_start + 1 == parent_source_row:
|
||
parent_sales_qty = to_num(srow[6]) if pd.notna(srow[6]) else None
|
||
break
|
||
|
||
for db_col, excel_idx in detail_mapping.items():
|
||
v = r[excel_idx] if excel_idx < len(r) else None
|
||
if v is None or (isinstance(v, float) and pd.isna(v)):
|
||
vals.append(None)
|
||
elif db_col == 'loss_quantity_rate_pct':
|
||
vals.append(clean_percent(v))
|
||
elif db_col in ('material_name', 'material_unit', 'specification', 'material_type'):
|
||
vals.append(to_text(v))
|
||
else:
|
||
vals.append(to_num(v))
|
||
|
||
# Calculate per_dish quantities
|
||
theoretical_qty = vals[-len(detail_mapping) + 6] # theoretical_quantity in vals
|
||
actual_qty = vals[-len(detail_mapping) + 8] # actual_quantity in vals
|
||
# Actually let's just get them directly
|
||
theoretical_qty = to_num(r[17]) if pd.notna(r[17]) else None
|
||
actual_qty = to_num(r[19]) if pd.notna(r[19]) else None
|
||
if theoretical_qty is not None and parent_sales_qty not in (None, 0):
|
||
vals.append(theoretical_qty / parent_sales_qty)
|
||
else:
|
||
vals.append(None)
|
||
if actual_qty is not None and parent_sales_qty not in (None, 0):
|
||
vals.append(actual_qty / parent_sales_qty)
|
||
else:
|
||
vals.append(None)
|
||
batch.append(tuple(vals))
|
||
if len(batch) >= 500:
|
||
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" Detail: 已导入 {idx + 1}/{len(detail_df)} 行")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" Detail: 已导入 {len(detail_df)}/{len(detail_df)} 行")
|
||
|
||
# Update log status
|
||
cur.execute(
|
||
"UPDATE public.dish_cost_analysis_import_log SET import_status = 'success' WHERE import_id = %s",
|
||
(import_id,)
|
||
)
|
||
conn.commit()
|
||
print(f"菜品成本导入完成: {len(summary_df)} 菜品, {len(detail_df)} 原料明细")
|
||
|
||
# ============================================================
|
||
# 营业费用导入
|
||
# ============================================================
|
||
def import_operating_expense(conn, filepath, report_month, expense_type='operating'):
|
||
"""营业费用导入:宽表转长表,每行=科目×门店"""
|
||
print(f"\n=== 营业费用导入 ({expense_type}) ===")
|
||
print(f"文件: {filepath}")
|
||
|
||
sha = file_sha256(filepath)
|
||
source_file = os.path.basename(filepath)
|
||
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"SELECT 1 FROM public.operating_expense_import_log WHERE source_file = %s AND file_sha256 = %s",
|
||
(source_file, sha)
|
||
)
|
||
if cur.fetchone():
|
||
print("已导入过,跳过")
|
||
return
|
||
|
||
# Read .xls file with pandas
|
||
raw_df = pd.read_excel(filepath, header=None, engine='xlrd')
|
||
print(f" 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}")
|
||
|
||
# Row 0: headers (科目编码, 科目名称, 统计方式, 方向3, 合计金额, 01 大钟寺金额, ...)
|
||
headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]]
|
||
|
||
# Data starts from row 1
|
||
df = raw_df.iloc[1:].copy()
|
||
|
||
# Filter out summary rows (col 0 = '费用科目' or empty)
|
||
df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != '费用科目')]
|
||
|
||
print(f" 科目行数: {len(df)}")
|
||
|
||
# Parse store columns: col 5+ are store columns (skip col 0-4: code, name, direction, direction3, total)
|
||
# Extract store code and name from header
|
||
store_cols = []
|
||
for ci in range(5, len(headers)):
|
||
h = headers[ci]
|
||
if not h:
|
||
continue
|
||
# Parse "01 大钟寺金额" -> code="01", name="大钟寺"
|
||
import re
|
||
m = re.match(r'([A-Z]\d+)\s+(.+?)金额$', h)
|
||
if m:
|
||
store_cols.append((ci, m.group(1), m.group(2)))
|
||
else:
|
||
# Keep as-is with full header
|
||
store_cols.append((ci, None, h))
|
||
|
||
print(f" 门店列数: {len(store_cols)}")
|
||
|
||
# Calculate total amount for log
|
||
total_amount = 0
|
||
for _, r in df.iterrows():
|
||
v = to_num(r[4]) if pd.notna(r[4]) else 0 # col 4 = 合计金额
|
||
total_amount += v if v else 0
|
||
|
||
# Insert import log
|
||
cur.execute(
|
||
"""INSERT INTO public.operating_expense_import_log
|
||
(report_month, source_file, source_sheet, file_sha256, source_total_amount, imported_record_count)
|
||
VALUES (%s, %s, %s, %s, %s, 0) RETURNING import_id""",
|
||
(report_month, source_file, 'Worksheet', sha, total_amount)
|
||
)
|
||
import_id = cur.fetchone()[0]
|
||
|
||
# Insert records: long format (one row per account × store)
|
||
insert_cols = ['import_id', 'report_month', 'source_file', 'source_sheet',
|
||
'source_row', 'source_column', 'account_code', 'account_name',
|
||
'accounting_direction', 'is_summary_account', 'parent_account_code',
|
||
'cost_unit_source_header', 'cost_unit_source_code',
|
||
'cost_unit_source_name', 'amount']
|
||
col_names = ','.join(insert_cols)
|
||
|
||
batch = []
|
||
total_records = 0
|
||
|
||
for row_idx, (_, r) in enumerate(df.iterrows()):
|
||
account_code = to_text(r[0])
|
||
account_name = to_text(r[1])
|
||
direction = to_text(r[2]) # 统计方式 (借方/贷方)
|
||
excel_row = int(r.name) + 1 # 1-indexed Excel row
|
||
|
||
# Determine if summary account: 7-digit codes (5032401) are sub-accounts of 5-digit (50324)
|
||
is_summary = False
|
||
parent_code = None
|
||
if account_code and len(account_code) == 7:
|
||
parent_code = account_code[:5]
|
||
is_summary = False
|
||
elif account_code and len(account_code) == 5:
|
||
is_summary = True
|
||
|
||
for col_idx, store_code, store_name in store_cols:
|
||
amount = to_num(r[col_idx]) if col_idx < len(r) and pd.notna(r[col_idx]) else None
|
||
if amount is None:
|
||
continue # Skip NULL amounts, but keep 0 amounts
|
||
|
||
vals = (
|
||
import_id,
|
||
report_month,
|
||
source_file,
|
||
'Worksheet',
|
||
excel_row,
|
||
col_idx + 1, # 1-indexed column
|
||
account_code,
|
||
account_name,
|
||
direction,
|
||
is_summary,
|
||
parent_code,
|
||
headers[col_idx], # full header text
|
||
store_code,
|
||
store_name,
|
||
amount,
|
||
)
|
||
batch.append(vals)
|
||
total_records += 1
|
||
|
||
if len(batch) >= 500:
|
||
execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {total_records} 条记录")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {total_records} 条记录")
|
||
|
||
# Update log
|
||
cur.execute(
|
||
"UPDATE public.operating_expense_import_log SET imported_record_count = %s WHERE import_id = %s",
|
||
(total_records, import_id)
|
||
)
|
||
conn.commit()
|
||
print(f"营业费用导入完成: {total_records} 条记录")
|
||
|
||
# ============================================================
|
||
# 中央厨房导入 (4表)
|
||
# ============================================================
|
||
def import_central_kitchen(conn, filepath, report_month, ck_type='finished'):
|
||
"""中央厨房导入: finished/recipe/material/processing
|
||
使用pandas读取,精确映射列,raw_data存储原始行JSON"""
|
||
import json
|
||
type_names = {
|
||
'finished': '完工入库',
|
||
'recipe': '配方耗用',
|
||
'material': '材料日报',
|
||
'processing': '加工单价',
|
||
}
|
||
table_map = {
|
||
'finished': 'central_kitchen_finished_receipt',
|
||
'recipe': 'central_kitchen_recipe_consumption',
|
||
'material': 'central_kitchen_material_daily',
|
||
'processing': 'central_kitchen_processing_cost',
|
||
}
|
||
table_name = table_map[ck_type]
|
||
|
||
print(f"\n=== 中央厨房-{type_names.get(ck_type, ck_type)} 导入 ===")
|
||
print(f"文件: {filepath}")
|
||
|
||
sha = file_sha256(filepath)
|
||
source_file = os.path.basename(filepath)
|
||
|
||
cur = conn.cursor()
|
||
cur.execute(
|
||
"SELECT 1 FROM public.central_kitchen_import_log WHERE source_file = %s AND file_sha256 = %s",
|
||
(source_file, sha)
|
||
)
|
||
if cur.fetchone():
|
||
print("已导入过,跳过")
|
||
return
|
||
|
||
# Header row varies by type:
|
||
# finished: row 2 (0-indexed) with sub-header row 3
|
||
# recipe: row 0
|
||
# material: row 0 with sub-header row 1
|
||
# processing: row 2
|
||
header_rows = {
|
||
'finished': 2, # 0-indexed, merged header rows 2+3
|
||
'recipe': 0,
|
||
'material': 1, # skip row 0 (group header), use row 1
|
||
'processing': 2,
|
||
}
|
||
hr = header_rows[ck_type]
|
||
|
||
# Read raw without header to get all rows
|
||
raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
|
||
|
||
# Build headers from the appropriate row(s)
|
||
if ck_type == 'finished':
|
||
# Row 2 has group headers, row 3 has sub-headers
|
||
# Don't fill forward - empty cells stay empty (merged cells)
|
||
h1 = raw_df.iloc[2].fillna('').astype(str).str.strip()
|
||
h2 = raw_df.iloc[3].fillna('').astype(str).str.strip()
|
||
headers = []
|
||
for i in range(len(h1)):
|
||
g = h1[i]
|
||
s = h2[i]
|
||
if g and s:
|
||
headers.append(f"{g}_{s}")
|
||
elif g:
|
||
headers.append(g)
|
||
elif s:
|
||
headers.append(s)
|
||
else:
|
||
headers.append(f'col_{i}') # Placeholder for empty headers
|
||
data_start = 4
|
||
elif ck_type == 'material':
|
||
# Row 0 has group headers, row 1 has sub-headers
|
||
h1 = raw_df.iloc[0].fillna('').astype(str).str.strip()
|
||
h2 = raw_df.iloc[1].fillna('').astype(str).str.strip()
|
||
headers = []
|
||
current_group = ''
|
||
for i in range(len(h1)):
|
||
g = h1[i] if h1[i] else current_group
|
||
if g:
|
||
current_group = g
|
||
s = h2[i] if h2[i] else ''
|
||
headers.append(f"{g}_{s}" if s else g)
|
||
data_start = 2
|
||
else:
|
||
# processing: row 2 (main) + row 3 (sub) — like finished
|
||
h1 = raw_df.iloc[hr].fillna('').astype(str).str.strip()
|
||
h2 = raw_df.iloc[hr + 1].fillna('').astype(str).str.strip()
|
||
headers = []
|
||
for i in range(len(h1)):
|
||
g = h1[i]
|
||
s = h2[i]
|
||
if g and s and g != s:
|
||
headers.append(f"{g}_{s}")
|
||
elif g and g != 'nan':
|
||
headers.append(g)
|
||
elif s and s != 'nan':
|
||
headers.append(s)
|
||
else:
|
||
headers.append(f'col_{i}')
|
||
data_start = hr + 2
|
||
|
||
# Extract data rows
|
||
df = raw_df.iloc[data_start:]
|
||
df = df.dropna(subset=[df.columns[0]])
|
||
# Filter out 合计/total rows by checking all string cells for '合计'
|
||
mask = pd.Series([True] * len(df), index=df.index)
|
||
for ci in range(min(15, df.shape[1])):
|
||
mask = mask & ~df[ci].astype(str).str.strip().isin(['合计', '总计', 'Total', 'total'])
|
||
# Filter out footer rows (e.g. "制表人:xxx")
|
||
mask = mask & ~df[0].astype(str).str.strip().str.startswith('制表人')
|
||
mask = mask & ~df[0].astype(str).str.strip().str.startswith('打印')
|
||
mask = mask & ~df[0].astype(str).str.strip().str.startswith('第')
|
||
df = df[mask]
|
||
data_rows = df.values.tolist()
|
||
print(f"数据行数: {len(data_rows)}")
|
||
|
||
if not data_rows:
|
||
return
|
||
|
||
# Insert import log
|
||
cur.execute(
|
||
"""INSERT INTO public.central_kitchen_import_log
|
||
(report_month, dataset_type, source_file, source_sheet, file_sha256,
|
||
workbook_rows, imported_rows, excluded_rows, status)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, 0, 'loading') RETURNING import_id""",
|
||
(report_month, ck_type, source_file, 'Worksheet', sha, len(data_rows), len(data_rows))
|
||
)
|
||
import_id = cur.fetchone()[0]
|
||
|
||
# Get DB columns (exclude record_id, import_id, source_row, raw_data)
|
||
cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{table_name}' ORDER BY ordinal_position")
|
||
db_cols = [r[0] for r in cur.fetchall()]
|
||
insertable_cols = [c for c in db_cols if c not in ('record_id', 'import_id', 'source_row', 'raw_data')]
|
||
|
||
# Build header name -> excel index map (keep first occurrence)
|
||
col_map = {}
|
||
for i, h in enumerate(headers):
|
||
if h and str(h).strip():
|
||
key = str(h).strip()
|
||
if key not in col_map:
|
||
col_map[key] = i
|
||
|
||
# Type-specific column mappings (db_col -> list of possible header names)
|
||
mappings = {
|
||
'finished': {
|
||
'receipt_date': ['入库日期'],
|
||
'workshop': ['col_1'],
|
||
'processing_warehouse': ['加工仓库'],
|
||
'recipe_name': ['配方名称'],
|
||
'product_name': ['货品名称'],
|
||
'specification': ['规格'],
|
||
'unit': ['库存单位'],
|
||
'return_quantity': ['退库数量'],
|
||
'return_amount': ['退库金额'],
|
||
'inbound_avg_unit_price': ['平均单价'],
|
||
'inbound_quantity': ['入库数量'],
|
||
'inbound_amount': ['入库金额'],
|
||
'raw_material_cost': ['加工利润分析_原料成本', '原料成本'],
|
||
'source_fee_cost': ['费用成本'],
|
||
'source_processing_profit': ['增值金额(加工利润)'],
|
||
'source_margin_rate': ['毛利率'],
|
||
'source_cost_rate': ['成本率'],
|
||
'theoretical_inbound_quantity': ['加工达成率分析_理论入库数量', '理论入库数量'],
|
||
'actual_inbound_quantity': ['实际入库数量'],
|
||
'inbound_difference_quantity': ['差异数量'],
|
||
'achievement_rate': ['加工达成率'],
|
||
'standard_expected_quantity': ['加工应产量分析_标准应产量', '标准应产量'],
|
||
'average_expected_quantity': ['平均应产量'],
|
||
'expected_quantity_variance_rate': ['应产量差异量比'],
|
||
'water_cost': ['水费'],
|
||
'electricity_cost': ['电费'],
|
||
'gas_cost': ['燃气费'],
|
||
'labor_cost': ['人工费'],
|
||
'other_cost': ['其他费用'],
|
||
'detailed_fee_total': ['费用合计'],
|
||
},
|
||
'recipe': {
|
||
'business_date': ['日期'],
|
||
'processing_warehouse': ['加工仓库'],
|
||
'finance_category_major': ['财务大类'],
|
||
'finance_category_minor': ['财务小类'],
|
||
'category_major': ['所属大类'],
|
||
'category_minor': ['所属小类'],
|
||
'item_id': ['货品ID'],
|
||
'item_code': ['编码'],
|
||
'item_name': ['货品名称'],
|
||
'specification': ['规格'],
|
||
'unit': ['库存单位'],
|
||
'unit_price_excl_tax': ['未税单价'],
|
||
'unit_price': ['单价'],
|
||
'recipe_name': ['配方名称'],
|
||
'standard_usage': ['标准用量'],
|
||
'actual_average_usage': ['实际平均用量'],
|
||
'planned_output_quantity': ['计划加工数量'],
|
||
'actual_output_quantity': ['实际加工数量'],
|
||
'output_unit': ['加工数量库存单位'],
|
||
'theoretical_quantity': ['理论用量'],
|
||
'net_quantity': ['净料用量'],
|
||
'theoretical_amount': ['理论金额'],
|
||
'issue_quantity': ['领用数量'],
|
||
'issue_amount': ['领用金额'],
|
||
'source_inventory_cost_quantity': ['成本数量(进销存)'],
|
||
'source_inventory_cost_amount': ['成本金额(进销存)'],
|
||
'actual_vs_theory_quantity': ['实际与理论差异数量'],
|
||
'actual_vs_theory_amount': ['实际与理论差异金额'],
|
||
'actual_vs_theory_rate': ['实际差异量比'],
|
||
'issue_vs_theory_quantity': ['领用与理论差异数量'],
|
||
'issue_vs_theory_amount': ['领用与理论差异金额'],
|
||
'issue_vs_theory_rate': ['领用差异量比'],
|
||
'actual_yield': ['实际出成率'],
|
||
'recipe_yield': ['配方出成率'],
|
||
'yield_difference': ['出成率差异'],
|
||
},
|
||
'material': {
|
||
'business_date': ['日期'],
|
||
'finance_category_major': ['财务大类'],
|
||
'finance_category_minor': ['财务小类'],
|
||
'category_major': ['所属大类'],
|
||
'category_minor': ['所属小类'],
|
||
'item_id': ['货品ID'],
|
||
'item_code': ['编码'],
|
||
'item_name': ['货品名称'],
|
||
'specification': ['规格'],
|
||
'unit': ['库存单位'],
|
||
},
|
||
'processing': {
|
||
'report_month': ['report_month'], # Will be set from parameter
|
||
'item_id': ['货品ID'],
|
||
'recipe_name': ['配方名称'],
|
||
'product_code': ['货品编码'],
|
||
'product_name': ['货品名称'],
|
||
'category_major': ['所属大类'],
|
||
'category_minor': ['所属小类'],
|
||
'finance_category_major': ['财务大类'],
|
||
'finance_category_minor': ['财务小类'],
|
||
'specification': ['规格'],
|
||
'unit': ['单位'],
|
||
'inbound_quantity': ['入库数量'],
|
||
'inbound_avg_unit_price': ['入库均价'],
|
||
'theoretical_unit_cost': ['理论单价'],
|
||
'standard_unit_cost': ['标准单价'],
|
||
'actual_unit_cost': ['实际单价'],
|
||
'theoretical_cost': ['理论成本'],
|
||
'standard_cost': ['标准成本'],
|
||
'actual_cost': ['实际成本'],
|
||
},
|
||
}
|
||
|
||
field_mapping = mappings[ck_type]
|
||
cols_to_insert = {}
|
||
fixed_values = {} # Values not from Excel (e.g. report_month)
|
||
for db_col, h_names in field_mapping.items():
|
||
if db_col not in insertable_cols:
|
||
continue
|
||
found = False
|
||
for h_name in h_names:
|
||
if h_name in col_map:
|
||
cols_to_insert[db_col] = col_map[h_name]
|
||
found = True
|
||
break
|
||
if not found and db_col == 'report_month':
|
||
fixed_values['report_month'] = report_month
|
||
|
||
# Determine numeric columns from DB schema
|
||
cur.execute(f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name='{table_name}'")
|
||
col_types = {r[0]: r[1] for r in cur.fetchall()}
|
||
numeric_db_cols = {c for c, t in col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')}
|
||
date_db_cols = {c for c, t in col_types.items() if t == 'date'}
|
||
|
||
all_cols = ['import_id', 'source_row'] + list(cols_to_insert.keys()) + list(fixed_values.keys()) + ['raw_data']
|
||
col_names = ','.join(all_cols)
|
||
|
||
batch = []
|
||
for idx, r in enumerate(data_rows):
|
||
source_row = idx + data_start + 1 # 1-indexed Excel row
|
||
vals = [import_id, source_row]
|
||
for db_col, excel_idx in cols_to_insert.items():
|
||
v = r[excel_idx] if excel_idx < len(r) else None
|
||
if db_col in numeric_db_cols:
|
||
vals.append(to_num(v) if v is not None else None)
|
||
elif db_col in date_db_cols:
|
||
vals.append(to_date(v))
|
||
else:
|
||
vals.append(to_text(v))
|
||
# Fixed values (not from Excel)
|
||
for fv_col in fixed_values:
|
||
if fv_col in date_db_cols:
|
||
vals.append(to_date(fixed_values[fv_col]))
|
||
else:
|
||
vals.append(fixed_values[fv_col])
|
||
# raw_data: store entire row as JSON
|
||
raw_dict = {}
|
||
for hi, hv in enumerate(headers):
|
||
if hv and hi < len(r):
|
||
rv = r[hi]
|
||
if pd.notna(rv):
|
||
raw_dict[str(hv)] = str(rv) if not isinstance(rv, (int, float)) else rv
|
||
vals.append(json.dumps(raw_dict, ensure_ascii=False))
|
||
batch.append(tuple(vals))
|
||
if len(batch) >= 500:
|
||
execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||
|
||
# Update log status
|
||
cur.execute(
|
||
"UPDATE public.central_kitchen_import_log SET status = 'success', completed_at = now() WHERE import_id = %s",
|
||
(import_id,)
|
||
)
|
||
conn.commit()
|
||
print(f"中央厨房{type_names[ck_type]}导入完成: {len(data_rows)} 行")
|
||
|
||
# ============================================================
|
||
# 配送明细导入
|
||
# ============================================================
|
||
def import_distribution(conn, filepath, report_month):
|
||
"""配送明细导入(支持单文件或目录)"""
|
||
import json
|
||
print(f"\n=== 配送明细导入 ===")
|
||
|
||
files = []
|
||
if os.path.isdir(filepath):
|
||
for f in sorted(os.listdir(filepath)):
|
||
if f.endswith('.xlsx') and not f.startswith('.'):
|
||
files.append(os.path.join(filepath, f))
|
||
else:
|
||
files.append(filepath)
|
||
|
||
print(f"文件数: {len(files)}")
|
||
|
||
cur = conn.cursor()
|
||
total_imported = 0
|
||
|
||
for file_idx, fpath in enumerate(files):
|
||
fname = os.path.basename(fpath)
|
||
print(f"\n[{file_idx+1}/{len(files)}] {fname}")
|
||
|
||
sha = file_sha256(fpath)
|
||
|
||
cur.execute(
|
||
"SELECT 1 FROM public.distribution_import_log WHERE source_file = %s AND file_sha256 = %s",
|
||
(fname, sha)
|
||
)
|
||
if cur.fetchone():
|
||
print(" 已导入过, 跳过")
|
||
continue
|
||
|
||
# Read with pandas, header on row 0
|
||
raw_df = pd.read_excel(fpath, header=None, engine='openpyxl')
|
||
headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]]
|
||
df = raw_df.iloc[1:]
|
||
df = df.dropna(subset=[df.columns[0]])
|
||
data_rows = df.values.tolist()
|
||
print(f" 数据行数: {len(data_rows)}")
|
||
|
||
if not data_rows:
|
||
continue
|
||
|
||
# Parse period from filename or use report_month
|
||
import re
|
||
date_match = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日.*?(\d{1,2})日', fname)
|
||
if date_match:
|
||
year, month, start_day, end_day = date_match.groups()
|
||
period_start = f"{year}-{int(month):02d}-{int(start_day):02d}"
|
||
period_end = f"{year}-{int(month):02d}-{int(end_day):02d}"
|
||
else:
|
||
# Use full month
|
||
period_start = report_month
|
||
# Last day of month
|
||
rm_date = pd.Timestamp(report_month)
|
||
period_end = (rm_date + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d')
|
||
|
||
cur.execute(
|
||
"""INSERT INTO public.distribution_import_log
|
||
(source_file, source_sheet, file_sha256, workbook_rows, summary_rows_excluded,
|
||
imported_rows, report_month, period_start, period_end, status)
|
||
VALUES (%s, %s, %s, %s, 0, %s, %s, %s, %s, 'loading') RETURNING import_id""",
|
||
(fname, 'Worksheet', sha, len(data_rows), len(data_rows), report_month, period_start, period_end)
|
||
)
|
||
import_id = cur.fetchone()[0]
|
||
|
||
# Get DB columns
|
||
cur.execute("SELECT column_name, data_type FROM information_schema.columns WHERE table_name='distribution_detail_records' ORDER BY ordinal_position")
|
||
db_col_types = {r[0]: r[1] for r in cur.fetchall()}
|
||
insertable_cols = [c for c in db_col_types if c not in ('record_id', 'import_id', 'source_row', 'raw_data')]
|
||
|
||
# Build header map
|
||
col_map = {}
|
||
for i, h in enumerate(headers):
|
||
if h and str(h).strip():
|
||
col_map[str(h).strip()] = i
|
||
|
||
# Known mappings for distribution based on actual Excel headers
|
||
field_mapping = {
|
||
'business_date': ['日期', '业务日期'],
|
||
'distribution_center_code': ['配送中心编码'],
|
||
'distribution_center_name': ['配送中心'],
|
||
'distribution_center_company': ['配送中心所属公司'],
|
||
'store_company': ['门店所属公司'],
|
||
'store_type': ['门店类型'],
|
||
'store_brand': ['门店品牌'],
|
||
'region_level1': ['一级区域'],
|
||
'terminal_region': ['末级区域'],
|
||
'store_code': ['门店编码'],
|
||
'store_name': ['门店'],
|
||
'store_tax_entity': ['门店财税主体'],
|
||
'original_batch_unit_price': ['原始批次单价'],
|
||
'cost_avg_unit_price': ['成本平均单价'],
|
||
'cost_total_amount': ['成本总金额'],
|
||
'team': ['班组'],
|
||
'cost_excl_tax_amount': ['成本未含税金额'],
|
||
'cost_tax_amount': ['成本税额'],
|
||
'cost_excl_tax_unit_price': ['成本未含税单价'],
|
||
'outbound_tax_amount': ['配出税额'],
|
||
'distribution_sales_tax_rate': ['配送销售税点'],
|
||
'route': ['路线'],
|
||
'profit_amount': ['利润'],
|
||
'profit_excl_tax_amount': ['未含税利润', '利润未含税'],
|
||
'distribution_type': ['配送类型'],
|
||
'logistics_attribute': ['物流属性'],
|
||
'item_id': ['货品ID'],
|
||
'item_name': ['货品名称', '货品'],
|
||
'item_alias': ['货品别名'],
|
||
'specification': ['规格'],
|
||
'item_purpose': ['货品用途', '用途'],
|
||
'unit': ['单位'],
|
||
'normal_quantity': ['正常品数量', '正常数量'],
|
||
'gift_quantity': ['赠品数量'],
|
||
'total_quantity': ['总数量', '合计数量'],
|
||
'outbound_avg_unit_price': ['配出平均单价', '配出均价'],
|
||
'outbound_total_amount': ['配出总金额', '配出金额'],
|
||
'outbound_margin_rate': ['配出毛利率'],
|
||
'outbound_excl_tax_unit_price': ['配出未含税单价'],
|
||
'outbound_excl_tax_amount': ['配出未含税金额'],
|
||
'document_no': ['单号'],
|
||
'requisition_no': ['要货单号', '申请单号'],
|
||
'payment_method': ['支付方式', '结算方式'],
|
||
'value_added_coefficient': ['增值系数'],
|
||
'audit_time': ['审核时间'],
|
||
'inbound_batch': ['入库批次'],
|
||
'inbound_batch_tax_rate': ['入库批次税点', '入库批次税率'],
|
||
'custom_batch_code': ['自定义批次码', '自定义批次编码'],
|
||
'invoice_type': ['发票种类', '发票类型'],
|
||
'production_time': ['生产时间'],
|
||
'shelf_life': ['保质期'],
|
||
'supplier_batch': ['供应商批次'],
|
||
'supplier_code': ['供应商编码'],
|
||
'supplier_category': ['供应商类别', '供应商分类'],
|
||
'supplier_name': ['供应商'],
|
||
'supplier_contact': ['供应商联系人'],
|
||
'supplier_phone': ['供应商联系电话', '供应商电话'],
|
||
'supplier_address': ['供应商地址'],
|
||
'from_warehouse': ['调出仓库', '发货仓库'],
|
||
'to_warehouse': ['调入仓库', '收货仓库'],
|
||
'item_code': ['货品编码'],
|
||
'item_barcode': ['货品条形码', '货品条码'],
|
||
'minor_category': ['所属小类', '小类'],
|
||
'major_category': ['所属大类', '大类'],
|
||
'internal_brand': ['内部品牌'],
|
||
'finance_category': ['财务分类'],
|
||
'created_time': ['制单时间', '创建时间'],
|
||
'creator': ['制单人', '创建人'],
|
||
'auditor': ['审核人'],
|
||
'arrival_time': ['到货时间'],
|
||
'document_remark': ['单据备注'],
|
||
'document_item_remark': ['单据项备注'],
|
||
'reason_remark': ['原因备注'],
|
||
'business_date': ['业务日期'],
|
||
'contact_address': ['联系地址'],
|
||
'contact_phone': ['联系电话'],
|
||
'contact_person': ['联系人'],
|
||
}
|
||
|
||
cols_to_insert = {}
|
||
for db_col, h_names in field_mapping.items():
|
||
if db_col not in insertable_cols:
|
||
continue
|
||
for h_name in h_names:
|
||
if h_name in col_map:
|
||
cols_to_insert[db_col] = col_map[h_name]
|
||
break
|
||
|
||
numeric_db_cols = {c for c, t in db_col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')}
|
||
date_db_cols = {c for c, t in db_col_types.items() if t == 'date'}
|
||
|
||
all_cols = ['import_id', 'source_row', 'report_month', 'source_file', 'source_sheet'] + list(cols_to_insert.keys())
|
||
col_names = ','.join(all_cols)
|
||
|
||
batch = []
|
||
for idx, r in enumerate(data_rows):
|
||
source_row = idx + 2 # 1-indexed, data starts at row 2
|
||
vals = [import_id, source_row, report_month, fname, 'Worksheet']
|
||
for db_col, excel_idx in cols_to_insert.items():
|
||
v = r[excel_idx] if excel_idx < len(r) else None
|
||
if db_col in numeric_db_cols:
|
||
vals.append(to_num(v) if v is not None else None)
|
||
elif db_col in date_db_cols:
|
||
vals.append(to_date(v))
|
||
else:
|
||
vals.append(to_text(v))
|
||
batch.append(tuple(vals))
|
||
if len(batch) >= 500:
|
||
execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||
batch = []
|
||
|
||
if batch:
|
||
execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500)
|
||
conn.commit()
|
||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||
|
||
total_imported += len(data_rows)
|
||
|
||
# Update log status
|
||
cur.execute(
|
||
"UPDATE public.distribution_import_log SET status = 'success', completed_at = now() WHERE import_id = %s",
|
||
(import_id,)
|
||
)
|
||
conn.commit()
|
||
|
||
print(f"\n配送明细导入完成: 共 {total_imported} 行")
|
||
|
||
# ============================================================
|
||
# Main
|
||
# ============================================================
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='月度数据导入工具')
|
||
parser.add_argument('--dataset', required=True,
|
||
choices=['inventory', 'dish_sales', 'dish_cost', 'operating_expense',
|
||
'central_kitchen', 'distribution'],
|
||
help='数据域')
|
||
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
||
parser.add_argument('--file', required=True, help='文件路径或目录')
|
||
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
||
parser.add_argument('--ck-type', default='finished',
|
||
choices=['finished', 'recipe', 'material', 'processing'],
|
||
help='中央厨房子类型')
|
||
parser.add_argument('--expense-type', default='operating',
|
||
choices=['operating', 'management'],
|
||
help='费用类型')
|
||
|
||
args = parser.parse_args()
|
||
|
||
conn = connect(args.db)
|
||
conn.autocommit = False
|
||
|
||
try:
|
||
if args.dataset == 'inventory':
|
||
import_inventory_cost(conn, args.file, args.month)
|
||
elif args.dataset == 'dish_sales':
|
||
import_dish_sales(conn, args.file, args.month)
|
||
elif args.dataset == 'dish_cost':
|
||
import_dish_cost(conn, args.file, args.month)
|
||
elif args.dataset == 'operating_expense':
|
||
import_operating_expense(conn, args.file, args.month, args.expense_type)
|
||
elif args.dataset == 'central_kitchen':
|
||
import_central_kitchen(conn, args.file, args.month, args.ck_type)
|
||
elif args.dataset == 'distribution':
|
||
import_distribution(conn, args.file, args.month)
|
||
|
||
print("\n=== 导入完成 ===")
|
||
except Exception as e:
|
||
conn.rollback()
|
||
print(f"错误: {e}", file=sys.stderr)
|
||
raise
|
||
finally:
|
||
conn.close()
|
||
|
||
if __name__ == '__main__':
|
||
main()
|