From 2fc1b96f44f077e20687f2e0169513086c39fa1a Mon Sep 17 00:00:00 2001 From: freedakgmail Date: Tue, 18 Aug 2026 22:17:26 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9Edb/pipeline=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=AF=BC=E5=85=A5=E7=AE=A1=E7=BA=BF=EF=BC=8C=E9=80=82?= =?UTF-8?q?=E9=85=8D5=E6=9C=88=E6=95=B0=E6=8D=AE=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档 --- .gitignore | 1 + db/pipeline/README.md | 67 + db/pipeline/config.py | 264 ++++ db/pipeline/import_bill_records.py | 198 +++ db/pipeline/import_bill_records_may.py | 549 +++++++ db/pipeline/import_derived_data.py | 294 ++++ db/pipeline/import_may_data.sh | 170 +++ db/pipeline/import_monthly_data.py | 1301 +++++++++++++++++ db/{ => pipeline}/import_salary_attendance.py | 143 +- db/pipeline/import_store_location.py | 619 ++++++++ db/pipeline/refresh_materialized_views.py | 104 ++ db/pipeline/run_all.py | 195 +++ db/pipeline/verify.py | 111 ++ db/pipeline/verify_import.py | 145 ++ 14 files changed, 4096 insertions(+), 65 deletions(-) create mode 100644 db/pipeline/README.md create mode 100644 db/pipeline/config.py create mode 100644 db/pipeline/import_bill_records.py create mode 100644 db/pipeline/import_bill_records_may.py create mode 100644 db/pipeline/import_derived_data.py create mode 100755 db/pipeline/import_may_data.sh create mode 100644 db/pipeline/import_monthly_data.py rename db/{ => pipeline}/import_salary_attendance.py (63%) create mode 100644 db/pipeline/import_store_location.py create mode 100644 db/pipeline/refresh_materialized_views.py create mode 100644 db/pipeline/run_all.py create mode 100644 db/pipeline/verify.py create mode 100644 db/pipeline/verify_import.py diff --git a/.gitignore b/.gitignore index c0b54ba..e9515c4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ backups/ .playwright-cli/ .tmp/ 数据/ +__pycache__/ diff --git a/db/pipeline/README.md b/db/pipeline/README.md new file mode 100644 index 0000000..ce5e46a --- /dev/null +++ b/db/pipeline/README.md @@ -0,0 +1,67 @@ +# 数据导入与物化刷新流水线 + +## 目录结构 + +``` +db/pipeline/ +├── README.md ← 本文件 +├── config.py ← 数据源路径与数据库配置 +├── run_all.py ← 一键全量导入+物化刷新 +├── refresh_materialized_views.py ← 物化视图按依赖顺序刷新 +├── verify.py ← 测试库与正式库行数对比验证 +├── verify_import.py ← 详细数据对比验证 +├── import_bill_records.py ← 账单查询原始数据导入（4月格式，196列宽表） +├── import_bill_records_may.py ← 5月专用账单导入（51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details） +├── import_monthly_data.py ← 月度业务数据导入（库存/销售/成本/费用/中央厨房/配送） +├── import_salary_attendance.py ← 薪资考勤数据导入 +├── import_store_location.py ← 门店位置与映射表导入 +└── import_derived_data.py ← 派生分析数据生成 +``` + +## 导入脚本 + +| 脚本 | 数据表 | 数据源 | +|------|--------|--------| +| `import_bill_records.py` | bill_records, bill_columns | 账单查询 Excel 目录（17个文件） | +| `import_monthly_data.py` | inventory_cost_records, dish_sales_details, dish_cost_analysis_*, operating_expense_records, central_kitchen_*, distribution_detail_records | 各业务 Excel 文件 | +| `import_salary_attendance.py` | salary_detail_records, attendance_records | 薪资/考勤 Excel | +| `import_store_location.py` | store_location_master, store_location_source_rows, sales_store_location_mapping, store_name_mapping, inventory_store_mapping, operating_expense_store_mapping | 各店信息 Excel | +| `import_derived_data.py` | dish_diagnosis_snapshot, central_kitchen_manufacturing_cost_pool | 从已有表派生 | + +## 物化视图刷新顺序（按依赖分层） + +| 层级 | 物化视图 | 依赖 | +|------|----------|------| +| L0 | bill_fact | 基础表 | +| L1 | mv_daily_revenue, mv_dish_basket_monthly, mv_dish_pair_summary_monthly, mv_dish_sku_summary_monthly, mv_dish_store_summary_monthly, mv_district_site_benchmark_monthly, mv_inventory_cost_classified_monthly, mv_site_segment_benchmark_monthly, mv_store_*_monthly (12个), mv_store_scorecard | bill_fact | +| L1 | dish_sales_april | bill_fact | +| L2 | dish_basket_april, dish_category_summary_april, dish_member_sku_april, dish_sku_summary_april, dish_store_sku_april | dish_sales_april | +| L2 | mv_dish_sku_abc_monthly | mv_dish_sku_summary_monthly | +| L2 | v_store_benchmark, v_store_category_mix, v_store_platform_economics | bill_fact | +| L3 | dish_store_summary_april | dish_basket_april | +| L3 | v_store_action_list | v_store_benchmark, v_store_category_mix, v_store_platform_economics | +| L3 | mv_store_action_priority_deep_april, mv_store_theoretical_actual_cost_april | bill_fact / dish_store_summary_april | +| L4 | v_store_execution_priority | v_store_action_list | + +## 使用方法 + +### 一键全量导入 +```bash +python3 db/pipeline/run_all.py --month 2026-04-01 --db bill_query_test +``` + +### 仅刷新物化视图 +```bash +python3 db/pipeline/refresh_materialized_views.py --db bill_query_test +``` + +### 验证数据一致性 +```bash +python3 db/pipeline/verify.py --db bill_query_test +``` + +### 单独导入某类数据 +参见各脚本 `--help`，例如： +```bash +python3 db/pipeline/import_monthly_data.py --dataset inventory --month 2026-04-01 --file --db bill_query_test +``` diff --git a/db/pipeline/config.py b/db/pipeline/config.py new file mode 100644 index 0000000..2e92857 --- /dev/null +++ b/db/pipeline/config.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +""" +数据导入流水线配置 +统一管理数据源路径、数据库连接参数、导入脚本路径 +""" +import os +import glob +import sys + +# ============================================================ +# 路径常量 +# ============================================================ +PIPELINE_DIR = os.path.dirname(os.path.abspath(__file__)) +DB_DIR = os.path.dirname(PIPELINE_DIR) +BASE_DIR = os.path.dirname(DB_DIR) + +# 数据源根目录（西部马华数据分析 下有完整的业务数据） +DATA_ROOT = os.path.join(os.path.dirname(BASE_DIR), '西部马华数据分析') +# SBrainCO3 数据目录（部分文件只在这里有） +DATA_SBRAIN = os.path.join(BASE_DIR, '数据') + +# ============================================================ +# 数据库配置 +# ============================================================ +DB_CONFIG = { + 'host': 'localhost', + 'port': 5432, + 'user': 'freedak', + 'password': '', +} + +# ============================================================ +# 月份配置 +# ============================================================ +MONTH_CONFIG = { + '2026-04-01': { + 'month_dir': '4月', + 'date_range': '2026-04-01--2026-04-30', + 'dist_half1': '2026年4月1-15日', + 'dist_half2': '2026年4月16-30日', + 'inventory_file': '4.13盘点倒挤成本报表2026年4月明细.xlsx', + 'dish_cost_file': '菜品成本分析报表.xlsx', + 'expense_file': '2026年4月营业费用分析.xls', + 'ck_subdir': '中央厨房4月', + 'salary_file': '薪资拆分明细表_脱敏.xlsx', + 'attendance_file': '考勤数据表-脱敏.xlsx', + 'store_info_file': '各店信息新（20260508）.xls', + 'bill_records_subdir': '账单查询', + 'dish_sales_subdir': '账单查询-菜品销售明细表', + 'data_subdirs': {}, # 4月文件平铺在月份目录下 + }, + '2026-05-01': { + 'month_dir': '5月', + 'date_range': '2026-05-01--2026-05-31', + 'dist_half1': '2026年5月1-15日', + 'dist_half2': '2026年5月16-31日', + 'inventory_file': '4.13盘点倒挤成本报表2026年5月.xlsx', + 'dish_cost_file': '2026年5月菜品成本分析报表.xlsx', + 'expense_file': '2026年5月营业费用分析.xls', + 'ck_subdir': '中央厨房', + 'salary_file': '2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx', + 'attendance_file': '2026年5月考勤数据表.xlsx', + 'store_info_file': None, # 5月无新各店信息文件，复用4月 + 'bill_records_subdir': None, # 5月无独立账单查询目录 + 'dish_sales_subdir': '账单查询-菜品销售明细表', + 'data_subdirs': { # 5月文件在子目录中 + 'inventory': '供应链数据', + 'dish_cost': '营业数据', + 'operating_expense': '', + 'distribution': '供应链数据', + 'salary': '营业数据', + 'attendance': '营业数据', + 'central_kitchen': '中央厨房', + 'dish_sales': '营业数据', + }, + }, +} + + +def _find_file(base_dirs, pattern): + """在多个候选目录中查找文件""" + for d in base_dirs: + if not d: + continue + path = os.path.join(d, pattern) + if os.path.isfile(path): + return path + matches = glob.glob(os.path.join(d, pattern)) + if matches: + return matches[0] + return None + + +def get_data_sources(month): + """ + 根据月份返回数据源路径字典 + month: 'YYYY-MM-01' 格式 + """ + cfg = MONTH_CONFIG.get(month) + if not cfg: + print(f"警告: 未知月份 {month}，请在 config.py MONTH_CONFIG 中添加配置") + return {} + + month_dir = cfg['month_dir'] + data_dir = os.path.join(DATA_ROOT, month_dir) + sbrain_data = os.path.join(DATA_SBRAIN, month_dir) + subdirs = cfg.get('data_subdirs', {}) + + sources = {} + + def resolve(subdir_key, filename): + """在 data_dir 和 sbrain_data 中按子目录结构查找文件""" + sub = subdirs.get(subdir_key, '') + candidates = [] + if sub: + candidates.append(os.path.join(sbrain_data, sub, filename)) + candidates.append(os.path.join(data_dir, sub, filename)) + candidates.append(os.path.join(sbrain_data, filename)) + candidates.append(os.path.join(data_dir, filename)) + for c in candidates: + if os.path.isfile(c): + return c + return candidates[0] + + def resolve_dir(subdir_key, dirname): + """在 data_dir 和 sbrain_data 中按子目录结构查找目录""" + sub = subdirs.get(subdir_key, '') + candidates = [] + if sub: + candidates.append(os.path.join(sbrain_data, sub, dirname)) + candidates.append(os.path.join(data_dir, sub, dirname)) + candidates.append(os.path.join(sbrain_data, dirname)) + candidates.append(os.path.join(data_dir, dirname)) + for c in candidates: + if os.path.isdir(c): + return c + return candidates[0] + + # 1. 账单查询（bill_records）- 目录 + bill_subdir = cfg.get('bill_records_subdir') + if bill_subdir: + sources['bill_records'] = resolve_dir('bill_records', bill_subdir) + else: + sources['bill_records'] = None + + # 2. 菜品销售明细 - 目录 + sources['dish_sales'] = resolve_dir('dish_sales', cfg['dish_sales_subdir']) + + # 3. 库存成本 + sources['inventory'] = resolve('inventory', cfg['inventory_file']) + + # 4. 菜品成本BOM + sources['dish_cost'] = resolve('dish_cost', cfg['dish_cost_file']) + + # 5. 营业费用 + sources['operating_expense'] = resolve('operating_expense', cfg['expense_file']) + + # 6. 中央厨房（4个子文件） + ck_subdir = cfg['ck_subdir'] + ck_dir_sbrain = os.path.join(sbrain_data, ck_subdir) + ck_dir_data = os.path.join(data_dir, ck_subdir) + ck_dir = ck_dir_sbrain if os.path.isdir(ck_dir_sbrain) else ck_dir_data + date_range = cfg['date_range'] + sources['central_kitchen_finished'] = _find_file( + [ck_dir], f'{date_range} 23 59 59完工入库统计分析报表.xlsx' + ) or os.path.join(ck_dir, f'{date_range} 23 59 59完工入库统计分析报表.xlsx') + sources['central_kitchen_recipe'] = _find_file( + [ck_dir], f'按配方导出-货品实际与理论耗用({date_range}).xlsx' + ) or os.path.join(ck_dir, f'按配方导出-货品实际与理论耗用({date_range}).xlsx') + sources['central_kitchen_material'] = _find_file( + [ck_dir], f'按货品导出-货品实际与理论耗用({date_range}).xlsx' + ) or os.path.join(ck_dir, f'按货品导出-货品实际与理论耗用({date_range}).xlsx') + sources['central_kitchen_processing'] = _find_file( + [ck_dir], '加工单价分析报表.xlsx' + ) or os.path.join(ck_dir, '加工单价分析报表.xlsx') + + # 7. 配送明细（2个文件） + sources['distribution'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half1"]}.xlsx') + sources['distribution_2'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half2"]}.xlsx') + + # 8. 薪资+考勤 + sources['salary'] = resolve('salary', cfg['salary_file']) + sources['attendance'] = resolve('attendance', cfg['attendance_file']) + + # 9. 门店位置 + store_info_file = cfg.get('store_info_file') + if store_info_file: + sources['store_location'] = resolve('', store_info_file) + else: + # 5月无新文件，复用4月 + for p in [ + os.path.join(DATA_SBRAIN, '4月', '各店信息新（20260508）.xls'), + os.path.join(DATA_ROOT, '4月', '各店信息新（20260508）.xls'), + ]: + if os.path.isfile(p): + sources['store_location'] = p + break + else: + sources['store_location'] = None + + return sources + + +# ============================================================ +# 物化视图刷新顺序（按依赖分层，同层可并行） +# ============================================================ +MATVIEW_LAYERS = [ + # L0: 基础物化视图，依赖基础表 + ['bill_fact'], + + # L1: 依赖 bill_fact 或基础表 + [ + 'mv_daily_revenue', + 'mv_dish_basket_monthly', + 'mv_dish_pair_summary_monthly', + 'mv_dish_sku_summary_monthly', + 'mv_dish_store_summary_monthly', + 'mv_district_site_benchmark_monthly', + 'mv_inventory_cost_classified_monthly', + 'mv_site_segment_benchmark_monthly', + 'mv_store_action_priority_deep_monthly', + 'mv_store_area_efficiency_monthly', + 'mv_store_benchmark_composite_monthly', + 'mv_store_category_mix_monthly', + 'mv_store_deep_diagnosis_monthly', + 'mv_store_member_opportunity_monthly', + 'mv_store_overlap_risk_monthly', + 'mv_store_platform_economics_monthly', + 'mv_store_repeat_summary_monthly', + 'mv_store_risk_rating_monthly', + 'mv_store_scorecard', + 'mv_store_site_profile_monthly', + 'mv_store_site_replication_monthly', + 'mv_store_theoretical_actual_cost_monthly', + 'dish_sales_april', + 'v_store_benchmark', + 'v_store_category_mix', + 'v_store_platform_economics', + ], + + # L2: 依赖 L1 物化视图 + [ + 'dish_basket_april', + 'dish_category_summary_april', + 'dish_member_sku_april', + 'dish_sku_summary_april', + 'dish_store_sku_april', + 'mv_dish_sku_abc_monthly', + 'v_store_action_list', + ], + + # L3: 依赖 L2 + [ + 'dish_store_summary_april', + 'mv_store_action_priority_deep_april', + 'mv_store_theoretical_actual_cost_april', + 'v_store_execution_priority', + ], +] + + +def get_dbname(): + """获取默认数据库名""" + return 'bill_query_test' diff --git a/db/pipeline/import_bill_records.py b/db/pipeline/import_bill_records.py new file mode 100644 index 0000000..5ce9560 --- /dev/null +++ b/db/pipeline/import_bill_records.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +""" +账单查询原始数据导入脚本 +从Excel文件导入bill_records和bill_columns表。 + +用法: + python3 import_bill_records.py --month 2026-04-01 --file <文件或目录> --db bill_query_test + python3 import_bill_records.py --month 2026-04-01 --file <目录> --db bill_query_test --skip-columns +""" +import argparse +import hashlib +import os +import sys + +import pandas as pd +import psycopg2 +from psycopg2.extras import execute_values + + +def file_sha256(filepath): + h = hashlib.sha256() + with open(filepath, 'rb') as f: + for chunk in iter(lambda: f.read(8192), b''): + h.update(chunk) + return h.hexdigest() + + +def to_text(val): + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + s = str(val).strip() + if s.lower() in ('nan', 'none', ''): + return None + return s + + +def import_bill_columns(conn, filepaths): + """从第一个Excel文件提取列定义，导入bill_columns表""" + cur = conn.cursor() + cur.execute("SELECT count(*) FROM public.bill_columns") + existing = cur.fetchone()[0] + if existing > 0: + print(f"bill_columns已有{existing}行，跳过列定义导入") + return + + if not filepaths: + print("无文件可用，跳过列定义导入") + return + + fpath = filepaths[0] + df = pd.read_excel(fpath, header=None, nrows=2, engine='openpyxl') + main_headers = df.iloc[0].tolist() + sub_headers = df.iloc[1].tolist() + + columns = [] + for i, (mh, sh) in enumerate(zip(main_headers, sub_headers)): + col_name = f"c{i+1:03d}" + group_h = to_text(mh) or '' + sub_h = to_text(sh) or '' + # If group and sub are the same, only keep group + if group_h and sub_h and group_h == sub_h: + sub_h = '' + columns.append((col_name, i + 1, group_h, sub_h)) + + execute_values( + cur, + "INSERT INTO public.bill_columns (column_name, excel_position, group_header, sub_header) VALUES %s", + columns, + page_size=500 + ) + conn.commit() + print(f"bill_columns导入完成: {len(columns)} 列") + cur.close() + + +def import_bill_records(conn, filepath, report_month): + """导入单个Excel文件的账单数据到bill_records""" + fname = os.path.basename(filepath) + sha = file_sha256(filepath) + + cur = conn.cursor() + + # 幂等检查 + cur.execute( + "SELECT 1 FROM public.bill_records WHERE source_file = %s LIMIT 1", + (fname,) + ) + if cur.fetchone(): + print(f" {fname} 已导入过，跳过") + cur.close() + return 0 + + # 读取Excel：row0=主表头, row1=子表头, 数据从row2开始 + df = pd.read_excel(filepath, header=None, engine='openpyxl') + # 跳过前2行（表头） + data_df = df.iloc[2:] + # 过滤空行（第一列为空） + data_df = data_df.dropna(subset=[data_df.columns[0]]) + data_rows = data_df.values.tolist() + + print(f" 数据行数: {len(data_rows)}, 列数: {data_df.shape[1]}") + + if not data_rows: + cur.close() + return 0 + + # 获取DB列名 (c001 ~ c196) + num_cols = data_df.shape[1] + col_names = [f"c{i+1:03d}" for i in range(num_cols)] + all_cols = ['source_file', 'source_row'] + col_names + col_str = ','.join(all_cols) + + batch = [] + for idx, row in enumerate(data_rows): + source_row = idx + 3 # 1-indexed, data starts at Excel row 3 + vals = [fname, source_row] + for i in range(num_cols): + v = row[i] if i < len(row) else None + vals.append(to_text(v)) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values( + cur, + f"INSERT INTO public.bill_records ({col_str}) VALUES %s", + batch, + page_size=500 + ) + conn.commit() + if (idx + 1) % 10000 == 0: + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values( + cur, + f"INSERT INTO public.bill_records ({col_str}) VALUES %s", + batch, + page_size=500 + ) + conn.commit() + + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + cur.close() + return len(data_rows) + + +def main(): + parser = argparse.ArgumentParser(description='账单查询原始数据导入') + parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)') + parser.add_argument('--file', required=True, help='Excel文件路径或目录') + parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)') + parser.add_argument('--skip-columns', action='store_true', help='跳过bill_columns导入') + + args = parser.parse_args() + + conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak') + conn.autocommit = False + + # 收集文件 + files = [] + if os.path.isdir(args.file): + for f in sorted(os.listdir(args.file)): + if f.endswith('.xlsx') and not f.startswith('.'): + files.append(os.path.join(args.file, f)) + else: + files.append(args.file) + + print(f"\n=== 账单查询原始数据导入 ===") + print(f"文件数: {len(files)}") + print(f"数据库: {args.db}") + + try: + # 导入列定义 + if not args.skip_columns: + import_bill_columns(conn, files) + + # 导入数据 + total = 0 + for idx, fpath in enumerate(files): + fname = os.path.basename(fpath) + print(f"\n[{idx+1}/{len(files)}] {fname}") + total += import_bill_records(conn, fpath, args.month) + + print(f"\n账单数据导入完成: 共 {total} 行") + print("=== 导入完成 ===") + + except Exception as e: + conn.rollback() + print(f"错误: {e}", file=sys.stderr) + raise + finally: + conn.close() + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/import_bill_records_may.py b/db/pipeline/import_bill_records_may.py new file mode 100644 index 0000000..52bbeb0 --- /dev/null +++ b/db/pipeline/import_bill_records_may.py @@ -0,0 +1,549 @@ +#!/usr/bin/env python3 +""" +5月账单数据导入脚本 +将5月格式的"正餐事业部_全渠道订单明细"(51列)导入bill_records(196列结构)， +将"正餐事业部_品项销售明细"(50列)导入dish_sales_details。 + +5月数据格式与4月不同： +- 4月: 账单查询196列宽表 + 菜品销售明细表(多文件) +- 5月: 全渠道订单明细51列 + 品项销售明细50列 (各1个文件) + +用法: + python3 import_bill_records_may.py --month 2026-05-01 \ + --orders <全渠道订单明细.xlsx> \ + --items <品项销售明细.xlsx> \ + --db bill_query_test +""" +import argparse +import hashlib +import os +import re +import sys + +import pandas as pd +import psycopg2 +from psycopg2.extras import execute_values + + +DB_CONFIG = { + 'host': 'localhost', + 'port': 5432, + 'user': 'freedak', + 'password': '', +} + + +def file_sha256(filepath): + h = hashlib.sha256() + with open(filepath, 'rb') as f: + for chunk in iter(lambda: f.read(8192), b''): + h.update(chunk) + return h.hexdigest() + + +def to_text(val): + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + s = str(val).strip() + if s.lower() in ('nan', 'none', '', '--'): + return None + return s + + +def to_num(val): + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + s = str(val).strip().replace(',', '').replace('¥', '').replace('￥', '') + if s.lower() in ('nan', 'none', '', '--'): + return None + try: + return str(float(s)) + except (ValueError, TypeError): + return None + + +def to_timestamp_str(val): + """将日期/时间值转为字符串格式 YYYY/MM/DD HH:MM:SS""" + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + if isinstance(val, pd.Timestamp): + return val.strftime('%Y/%m/%d %H:%M:%S') + s = str(val).strip() + if s.lower() in ('nan', 'none', '', '--'): + return None + # 尝试解析各种日期格式 + for fmt in ['%Y/%m/%d %H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y/%m/%d', '%Y-%m-%d']: + try: + ts = pd.to_datetime(s, format=fmt) + return ts.strftime('%Y/%m/%d %H:%M:%S') + except (ValueError, TypeError): + continue + # pandas通用解析 + try: + ts = pd.to_datetime(s) + return ts.strftime('%Y/%m/%d %H:%M:%S') + except (ValueError, TypeError): + return s + + +# ============================================================ +# 5月全渠道订单明细 → bill_records 列映射 +# 5月col索引 → bill_records cXXX +# ============================================================ +ORDERS_COL_MAP = { + # 5月col: (bill_records col_name, converter) + 0: None, # 省份 → 无对应 + 1: None, # 城市 → 无对应 + 2: 'c003', # 门店名称 → c003 store_name + 3: None, # 营业日期 → 无直接对应(bill_fact用closed_at) + 4: 'c004', # 餐段 → c004 meal_period + 5: 'c005', # 订单号 → c005 bill_no + 6: None, # 原单号 + 7: None, # 外卖订单号 + 8: None, # 全渠道流水号 + 9: 'c008', # 取餐号 → c008 table_no + 10: None, # 桌牌号 + 11: 'c006', # 桌台区域 → c006 business_area + 12: 'c009', # 订单金额 → c009 consumption + 13: None, # 顾客实付 + 14: 'c114', # 订单收入 → c114 received_total + 15: 'c068', # 订单优惠 → c068 discount_total + 16: None, # 订单来源 + 17: None, # 订单子来源 + 18: None, # 经营模式 + 19: None, # 用餐方式 + 20: None, # 宴会类型 + 21: 'c194', # 订单状态 → c194 bill_status + 22: None, # 退单标识 + 23: 'c190', # 下单员 → c190 waiter + 24: 'c191', # 收银员 → c191 cashier + 25: None, # 桌台提成人 + 26: None, # 销售员 → c192 (不在bill_fact中，但可填) + 27: 'c175', # 创建时间 → c175 opened_at + 28: 'c176', # 完成时间 → c176 closed_at + 29: None, # 接单时间 + 30: None, # 下单制作时间 + 31: None, # 预定用餐时间 + 32: None, # 订单备注 + 33: None, # 桌台备注 + 34: 'c178', # 用餐人数 → c178 guest_count + 35: 'c179', # 席数 → c179 table_count + 36: None, # 是否会员 + 37: None, # 会员姓名 + 38: 'c185', # 会员卡号 → c185 member_id + 39: None, # 会员手机号 + 40: None, # 下单人手机号 + 41: None, # 发票 + 42: None, # 菜品金额 + 43: None, # 菜品收入 + 44: None, # 退单时间 + 45: None, # 取消时间 + 46: None, # 外卖订单已退金额 + 47: None, # 敏感操作 + 48: None, # 结账方式 → 需要解析支付方式 + 49: None, # 企业版订单 + 50: None, # 企业版订单是否开发票 +} + +# 结账方式解析 → bill_records支付方式列 +PAYMENT_PATTERNS = { + 'c140': [r'现金'], # 现金 + 'c143': [r'支付宝'], # 支付宝支付实收 + 'c144': [r'微信'], # 微信支付实收 + 'c145': [r'美团.*实收', r'大众点评.*实收', r'新美大实收', r'一键买单'], # 新美大实收 + 'c146': [r'云闪付'], # 云闪付实收 + 'c147': [r'抖音'], # 抖音券实收 + 'c151': [r'美团外卖'], # 美团外卖实收 + 'c152': [r'淘宝闪购', r'闪购'], # 淘宝闪购实收 + 'c150': [r'京东外卖'], # 京东外卖实收 + 'c149': [r'挂账'], # 挂账消费 +} + +# 折扣/佣金解析 +DISCOUNT_PATTERNS = { + 'c087': [r'会员积分'], # 会员积分抵现 + 'c088': [r'会员代金券'], # 会员代金券折扣 + 'c089': [r'会员预存'], # 会员预存折扣 + 'c095': [r'抖音.*折扣', r'抖音券折扣'], # 抖音券折扣 + 'c097': [r'美团.*佣金'], # 美团外卖佣金 + 'c098': [r'淘宝.*佣金', r'闪购.*佣金'], # 淘宝闪购佣金 + 'c100': [r'京东.*佣金'], # 京东外卖佣金 +} + + +def parse_payment(payment_str): + """解析结账方式字段，返回 {cXXX: amount_str}""" + result = {} + if not payment_str or payment_str == '--': + return result + + # 格式: "方式1 金额1,方式2 金额2" 或 "方式1 金额1" + parts = payment_str.split(',') + for part in parts: + part = part.strip() + # 提取金额（最后一个数字） + amount_match = re.search(r'([\d.]+)\s*$', part) + amount = amount_match.group(1) if amount_match else None + + if not amount: + continue + + # 匹配支付方式 + for col, patterns in PAYMENT_PATTERNS.items(): + if col in result: + continue + for pat in patterns: + if re.search(pat, part): + if col in result: + result[col] = str(float(result[col]) + float(amount)) + else: + result[col] = amount + break + + return result + + +def parse_discounts(discount_str): + """解析折扣字段""" + return parse_payment(discount_str) # 同样的解析逻辑 + + +def build_store_code_map(items_filepath): + """从品项销售明细中构建 门店名称→机构编码 映射""" + df = pd.read_excel(items_filepath, header=2, engine='openpyxl') + df = df.dropna(subset=[df.columns[0]]) + + mapping = {} + if '门店名称' in df.columns and '机构编码' in df.columns: + for _, row in df[['门店名称', '机构编码']].drop_duplicates().iterrows(): + name = str(row['门店名称']).strip() + code = str(row['机构编码']).strip() + if name and code and name != 'nan' and code != 'nan' and code != '--': + mapping[name] = code + + print(f" 门店编码映射: {len(mapping)} 个门店") + for name, code in sorted(mapping.items()): + print(f" {code} → {name}") + + return mapping + + +def import_bill_records_may(conn, orders_filepath, store_code_map, report_month): + """将5月全渠道订单明细导入bill_records""" + fname = os.path.basename(orders_filepath) + sha = file_sha256(orders_filepath) + + cur = conn.cursor() + + # 幂等检查 + cur.execute( + "SELECT 1 FROM public.bill_records WHERE source_file = %s LIMIT 1", + (fname,) + ) + if cur.fetchone(): + print(f" {fname} 已导入过，跳过") + cur.close() + return 0 + + # 读取Excel：row0=标题, row1=日期范围, row2=列头, row3+=数据 + df = pd.read_excel(orders_filepath, header=2, engine='openpyxl') + df = df.dropna(subset=[df.columns[0]]) + data_rows = df.values.tolist() + + print(f" 数据行数: {len(data_rows)}, 列数: {df.shape[1]}") + if not data_rows: + cur.close() + return 0 + + # 构建196列的列名 + all_col_names = [f"c{i+1:03d}" for i in range(196)] + insert_cols = ['source_file', 'source_row'] + all_col_names + col_str = ','.join(insert_cols) + + # 反转映射: 5月col_index → cXXX + col_to_c = {} + for may_col, c_name in ORDERS_COL_MAP.items(): + if c_name: + col_to_c[may_col] = c_name + + batch = [] + for idx, row in enumerate(data_rows): + source_row = idx + 4 # 数据从Excel第4行开始(1-indexed) + # 初始化196列全为None + c_vals = {f"c{i+1:03d}": None for i in range(196)} + + # 映射已知字段 + for may_idx, c_name in col_to_c.items(): + if may_idx < len(row): + val = row[may_idx] + # 时间字段特殊处理 + if c_name in ('c175', 'c176'): + c_vals[c_name] = to_timestamp_str(val) + elif c_name in ('c009', 'c068', 'c114', 'c178', 'c179'): + c_vals[c_name] = to_num(val) + else: + c_vals[c_name] = to_text(val) + + # 从门店名称查找门店编码 → c002 + store_name = to_text(row[2]) if len(row) > 2 else None + if store_name and store_name in store_code_map: + c_vals['c002'] = store_code_map[store_name] + + # 序号 → c001 + c_vals['c001'] = str(idx + 1) + + # 解析结账方式 → 支付方式列 + payment_str = to_text(row[48]) if len(row) > 48 else None + if payment_str: + payments = parse_payment(payment_str) + for c_name, amount in payments.items(): + c_vals[c_name] = amount + + # 计算时长(分) → c177 + opened = c_vals.get('c175') + closed = c_vals.get('c176') + if opened and closed: + try: + t1 = pd.to_datetime(opened, format='%Y/%m/%d %H:%M:%S') + t2 = pd.to_datetime(closed, format='%Y/%m/%d %H:%M:%S') + duration = (t2 - t1).total_seconds() / 60 + if duration >= 0: + c_vals['c177'] = str(int(duration)) + except (ValueError, TypeError): + pass + + # 构建行数据 + vals = [fname, source_row] + [c_vals[f"c{i+1:03d}"] for i in range(196)] + batch.append(tuple(vals)) + + if len(batch) >= 500: + execute_values( + cur, + f"INSERT INTO public.bill_records ({col_str}) VALUES %s", + batch, + page_size=500 + ) + conn.commit() + if (idx + 1) % 5000 == 0: + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values( + cur, + f"INSERT INTO public.bill_records ({col_str}) VALUES %s", + batch, + page_size=500 + ) + conn.commit() + + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + cur.close() + return len(data_rows) + + +# ============================================================ +# 5月品项销售明细 → dish_sales_details 字段映射 +# ============================================================ +ITEMS_FIELD_MAP = { + '城市': None, # col0 + '机构编码': 'store_code', # col1 + '门店名称': 'store_name', # col2 + '营业日期': None, # col3 + '下单时间所属餐段': 'business_type', # col4 + '出品部门': 'production_department', # col5 + '菜品大类': 'category_level1', # col6 + '菜品小类': 'category_level2', # col7 + '菜品编码': None, # col8 + '品项名称': 'dish_name', # col9 + '关联菜品名称': None, # col10 + '商品别名': None, # col11 + '品项类型': 'item_type', # col12 + '菜品类型': None, # col13 + '菜品标签': None, # col14 + '规格': None, # col15 + '单位': 'unit', # col16 + '关联做法': 'preparation_method', # col17 + '关联加料': None, # col18 + '关联餐盒': None, # col19 + '销售方式': None, # col20 + '订单号': 'bill_no', # col21 + '销售数量': 'sales_quantity', # col22 + '赠送数量': None, # col23 + '销售金额(元)': 'gross_amount', # col24 + '赠送金额(元)': None, # col25 + '优惠金额(元)': None, # col26 + '品项收入(元)': 'received_amount', # col27 + '点菜时间': None, # col28 + '下单时间': 'ordered_at', # col29 + '接单/结账/退菜时间': None, # col30 + '收银员': None, # col31 + '点菜员': None, # col32 + '下单人': None, # col33 + '订单分类': None, # col34 + '订单来源': None, # col35 + '新订单来源': None, # col36 + '订单子来源': None, # col37 + '桌台区域': None, # col38 + '取餐号': 'table_or_pickup_no', # col39 + '桌牌号': None, # col40 + '订单金额(元)': None, # col41 + '营业额(元)': None, # col42 + '订单优惠(元)': None, # col43 + '订单收入(元)': None, # col44 + '标记': None, # col45 + '退菜数量': None, # col46 + '退菜金额(元)': None, # col47 + '敏感操作类型': None, # col48 + '单品备注': None, # col49 +} + +NUMERIC_FIELDS = {'sales_quantity', 'gross_amount', 'received_amount', 'unit_price', 'guest_count'} +TIMESTAMP_FIELDS = {'opened_at', 'closed_at', 'ordered_at'} + + +def import_dish_sales_may(conn, items_filepath, report_month): + """将5月品项销售明细导入dish_sales_details""" + fname = os.path.basename(items_filepath) + + cur = conn.cursor() + + # 幂等检查 + cur.execute( + "SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s", + (fname,) + ) + if cur.fetchone(): + print(f" {fname} 已导入过，跳过") + return 0 + + file_size = os.path.getsize(items_filepath) + + # 读取Excel：row0=标题, row1=日期范围, row2=列头, row3+=数据 + df = pd.read_excel(items_filepath, header=2, engine='openpyxl') + df = df.dropna(subset=[df.columns[0]]) + data_rows = df.values.tolist() + headers = list(df.columns) + + print(f" 数据行数: {len(data_rows)}, 列数: {df.shape[1]}") + if not data_rows: + return 0 + + # 构建列名→索引映射 + col_map = {} + for i, h in enumerate(headers): + h_str = str(h).strip() if h else '' + if h_str: + col_map[h_str] = i + + # 确定要插入的列 + cols_to_insert = {} + for header_name, db_col in ITEMS_FIELD_MAP.items(): + if db_col and header_name in col_map: + cols_to_insert[db_col] = col_map[header_name] + + # 插入import log + cur.execute( + """INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status) + VALUES (%s, %s, %s, 'importing')""", + (fname, file_size, len(data_rows)) + ) + + all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys()) + col_names = ','.join(all_cols) + + batch = [] + for idx, r in enumerate(data_rows): + source_row = idx + 4 + vals = [fname, source_row] + for db_col, excel_idx in cols_to_insert.items(): + v = r[excel_idx] if excel_idx < len(r) else None + if db_col in NUMERIC_FIELDS: + vals.append(to_num(v)) + elif db_col in TIMESTAMP_FIELDS: + vals.append(to_timestamp_str(v)) + else: + vals.append(to_text(v)) + batch.append(tuple(vals)) + + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + if (idx + 1) % 10000 == 0: + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + + # 更新log状态 + cur.execute( + "UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s", + (fname,) + ) + conn.commit() + + cur.close() + return len(data_rows) + + +def main(): + parser = argparse.ArgumentParser(description='5月账单数据导入(全渠道订单明细+品项销售明细)') + parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)') + parser.add_argument('--orders', required=True, help='全渠道订单明细Excel文件路径') + parser.add_argument('--items', required=True, help='品项销售明细Excel文件路径') + parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)') + parser.add_argument('--skip-orders', action='store_true', help='跳过订单明细导入') + parser.add_argument('--skip-items', action='store_true', help='跳过品项销售明细导入') + + args = parser.parse_args() + + print(f"\n=== 5月账单数据导入 ===") + print(f"月份: {args.month}") + print(f"数据库: {args.db}") + print(f"订单明细: {args.orders}") + print(f"品项明细: {args.items}") + + cfg = DB_CONFIG.copy() + cfg['dbname'] = args.db + conn = psycopg2.connect(**cfg) + conn.autocommit = False + + try: + # 先从品项明细构建门店编码映射 + store_code_map = {} + if not args.skip_items and os.path.isfile(args.items): + print("\n--- 构建门店编码映射 ---") + store_code_map = build_store_code_map(args.items) + + # 导入bill_records + if not args.skip_orders and os.path.isfile(args.orders): + print("\n--- 导入全渠道订单明细 → bill_records ---") + import_bill_records_may(conn, args.orders, store_code_map, args.month) + + # 导入dish_sales_details + if not args.skip_items and os.path.isfile(args.items): + print("\n--- 导入品项销售明细 → dish_sales_details ---") + import_dish_sales_may(conn, args.items, args.month) + + print("\n=== 5月账单数据导入完成 ===") + + except Exception as e: + conn.rollback() + print(f'错误: {e}', file=sys.stderr) + raise + finally: + conn.close() + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/import_derived_data.py b/db/pipeline/import_derived_data.py new file mode 100644 index 0000000..81ad39e --- /dev/null +++ b/db/pipeline/import_derived_data.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 +""" +派生分析数据导入脚本 +生成以下派生表： + - dish_diagnosis_snapshot (菜品诊断快照，从dish_cost_analysis_summary + BOM生成) + - central_kitchen_manufacturing_cost_pool (中央厨房制造成本池，从薪资+营业费用+完工报表派生) + +用法: + python3 import_derived_data.py --month 2026-04-01 --db bill_query_test +""" +import argparse +import datetime +import sys + +import psycopg2 + + +def generate_dish_diagnosis(conn, diag_date): + """生成菜品诊断快照""" + cur = conn.cursor() + + # 检查是否已有数据 + cur.execute("SELECT count(*) FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,)) + existing = cur.fetchone()[0] + if existing > 0: + print(f" dish_diagnosis_snapshot 已有 {existing} 行 (date={diag_date})，先删除再生成") + + cur.execute("DELETE FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,)) + + # 检查依赖表是否存在且有数据 + cur.execute("SELECT count(*) FROM public.dish_cost_analysis_summary") + summary_count = cur.fetchone()[0] + if summary_count == 0: + print(" 跳过: dish_cost_analysis_summary 无数据") + cur.close() + return 0 + + # 检查 analytics.fact_recipe_bom 是否存在 + cur.execute(""" + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = 'analytics' AND table_name = 'fact_recipe_bom' + ) + """) + has_bom = cur.fetchone()[0] + + bom_join = "" + bom_select = "0 AS cnt, 0 AS unique_cnt, 0 AS avg_waste" + if has_bom: + bom_select = "COALESCE(bom.cnt, 0), COALESCE(bom.unique_cnt, 0), COALESCE(round(bom.avg_waste::numeric, 2), 0)" + bom_join = """ + LEFT JOIN ( + SELECT b.sku_code, + count(*) AS cnt, + count(*) FILTER (WHERE r.ref_count = 1) AS unique_cnt, + sum(b.waste_rate * b.standard_gross_quantity) / nullif(sum(b.standard_gross_quantity), 0) AS avg_waste + FROM analytics.fact_recipe_bom b + LEFT JOIN ( + SELECT material_code, count(DISTINCT sku_code) AS ref_count + FROM analytics.fact_recipe_bom GROUP BY material_code + ) r ON r.material_code = b.material_code + GROUP BY b.sku_code + ) bom ON bom.sku_code = s.dish_code + """ + + sql = f""" + INSERT INTO public.dish_diagnosis_snapshot ( + diagnosis_date, dish_code, dish_name, category_l1, + sales_amount, sales_quantity, + theoretical_margin_pct, actual_margin_pct, cost_variance_amount, + cost_tier, bom_complexity_score, unique_material_count, waste_rate_avg, + diagnosis_type, diagnosis_detail, suggested_action, priority + ) + SELECT + '{diag_date}'::date, + s.dish_code, + s.dish_name, + s.category_level1, + round(s.sales_amount::numeric, 2), + round(s.sales_quantity::numeric, 2), + round(s.theoretical_margin_rate_pct::numeric, 2), + round(s.actual_margin_rate_pct::numeric, 2), + round(s.cost_variance_amount::numeric, 2), + CASE + WHEN s.actual_margin_rate_pct < 0 THEN '数据异常' + WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0 + AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN '紧急' + WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0 + AND (s.cost_variance_amount / s.theoretical_cost) > 0.1 THEN '整改' + WHEN s.cost_variance_amount > 0 THEN '关注' + ELSE '正常' + END, + {bom_select}, + CASE + WHEN s.actual_margin_rate_pct < 0 THEN '数据异常' + WHEN s.theoretical_margin_rate_pct < 0 THEN '负毛利' + WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN '低毛利-定价偏低' + WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '低毛利-成本超耗' + WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '高超耗-份量超标' + WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 100 THEN '高超耗-分摊异常' + WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN '配方复杂-低销量' + WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN '独有原料风险' + WHEN s.cost_variance_amount > 0 THEN '成本差异' + ELSE '正常' + END, + CASE + WHEN s.actual_margin_rate_pct < 0 THEN '实际毛利率为负，需先核查BOM/单位/分摊' + WHEN s.theoretical_margin_rate_pct < 0 THEN '理论毛利率为负，定价低于标准成本' + WHEN s.theoretical_margin_rate_pct < 30 THEN '理论毛利率低于30%，定价偏低' + WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '实际成本超理论，存在超耗' + WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '物料损耗率超过20%' + ELSE '成本基本正常' + END, + CASE + WHEN s.actual_margin_rate_pct < 0 THEN 'fix_data' + WHEN s.theoretical_margin_rate_pct < 0 THEN 'price_up' + WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN 'price_up' + WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN 'recipe_optimize' + WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN 'portion_reduce' + WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'delist' + WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'evaluate_delist' + WHEN s.cost_variance_amount > 0 THEN 'monitor' + ELSE 'keep' + END, + CASE + WHEN s.actual_margin_rate_pct < 0 THEN 'P0' + WHEN s.theoretical_margin_rate_pct < 0 THEN 'P0' + WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0 + AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN 'P0' + WHEN s.theoretical_margin_rate_pct < 50 THEN 'P1' + WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'P2' + WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'P2' + ELSE 'P3' + END + FROM public.dish_cost_analysis_summary s + {bom_join} + WHERE s.dish_code IS NOT NULL + """ + + cur.execute(sql) + inserted = cur.rowcount + conn.commit() + print(f" dish_diagnosis_snapshot 生成完成: {inserted} 行 (date={diag_date})") + cur.close() + return inserted + + +def generate_manufacturing_cost_pool(conn, report_month): + """生成中央厨房制造成本池""" + cur = conn.cursor() + + # 检查是否已有数据 + cur.execute("SELECT count(*) FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,)) + existing = cur.fetchone()[0] + if existing > 0: + print(f" central_kitchen_manufacturing_cost_pool 已有 {existing} 行，先删除再生成") + + cur.execute("DELETE FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,)) + + # 检查依赖表是否有数据 + cur.execute("SELECT count(*) FROM public.salary_detail_records") + salary_count = cur.fetchone()[0] + cur.execute("SELECT count(*) FROM public.operating_expense_records WHERE report_month = %s", (report_month,)) + expense_count = cur.fetchone()[0] + cur.execute("SELECT count(*) FROM public.central_kitchen_finished_receipt WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date", (report_month, report_month)) + ck_count = cur.fetchone()[0] + + if salary_count == 0 and expense_count == 0 and ck_count == 0: + print(" 跳过: 依赖表无数据") + cur.close() + return 0 + + rows = [] + + # 1. 直接人工：中央厨房生产人员工资 + if salary_count > 0: + cur.execute(""" + SELECT round(coalesce(sum(net_pay), 0)::numeric, 2) + FROM public.salary_detail_records + WHERE org_level1 LIKE '%加工配送中心%' OR org_level1 LIKE '%中央厨房%' + OR org_level2 LIKE '%加工配送中心%' OR org_level2 LIKE '%中央厨房%' + OR org_level3 LIKE '%加工配送中心%' OR org_level3 LIKE '%中央厨房%' + OR org_level4 LIKE '%加工配送中心%' OR org_level4 LIKE '%中央厨房%' + OR org_level5 LIKE '%加工配送中心%' OR org_level5 LIKE '%中央厨房%' + OR org_level6 LIKE '%加工配送中心%' OR org_level6 LIKE '%中央厨房%' + OR org_level7 LIKE '%加工配送中心%' OR org_level7 LIKE '%中央厨房%' + OR org_level8 LIKE '%加工配送中心%' OR org_level8 LIKE '%中央厨房%' + """) + salary_total = cur.fetchone()[0] + if salary_total and float(salary_total) > 0: + rows.append(( + report_month, '直接人工', '中央厨房生产人员工资', '薪资明细', + 'salary_detail_records:加工配送中心/中央厨房', + float(salary_total), 1.0, float(salary_total), + '中央厨房组织直接归集', False, True, + None + )) + + # 2. 制造费用：从营业费用中供应链部分按比例分摊 + if expense_count > 0: + # 中央厨房人数/加工配送中心人数比例 = 7/11 ≈ 0.6364 + ck_ratio = 0.63636364 + expense_items = [ + ('50302', '餐厅房租', 'operating_expense_records:供应链/50302'), + ('50307', '水费', 'operating_expense_records:供应链/50307'), + ('50308', '电费', 'operating_expense_records:供应链/50308'), + ('50310', '员工宿舍费用', 'operating_expense_records:供应链/50310'), + ('50315', '维修费', 'operating_expense_records:供应链/50315'), + ] + for acct_code, acct_name, source_ref in expense_items: + cur.execute(""" + SELECT round(coalesce(sum(amount), 0)::numeric, 2) + FROM public.operating_expense_records + WHERE report_month = %s AND account_code = %s + AND cost_unit_source_name LIKE '供应链%%' + """, (report_month, acct_code)) + amount = cur.fetchone()[0] + if amount and float(amount) > 0: + allocated = round(float(amount) * ck_ratio, 6) + rows.append(( + report_month, '制造费用', acct_name, '营业费用', + source_ref, float(amount), ck_ratio, allocated, + '中央厨房人数/加工配送中心人数（临时）', True, True, + '供应链共享费用，待电表、面积或工时数据后替换' + )) + + # 3. 源报表费用：完工报表费用成本 + if ck_count > 0: + cur.execute(""" + SELECT round(coalesce(sum(source_fee_cost), 0)::numeric, 4) + FROM public.central_kitchen_finished_receipt + WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date + """, (report_month, report_month)) + fee_total = cur.fetchone()[0] + if fee_total and float(fee_total) > 0: + rows.append(( + report_month, '源报表费用', '完工报表费用成本', '完工入库报表', + 'central_kitchen_finished_receipt.source_fee_cost', + float(fee_total), 1.0, float(fee_total), + '源报表原值', True, False, + '无水电人工等明细来源，为避免与工资和营业费用重复，暂不计入重建成本' + )) + + if rows: + from psycopg2.extras import execute_values + execute_values(cur, """ + INSERT INTO public.central_kitchen_manufacturing_cost_pool + (report_month, cost_type, cost_subtype, source_type, source_reference, + source_amount, central_kitchen_share_pct, allocated_amount, + allocation_method, is_provisional, include_in_rebuilt_cost, note) + VALUES %s + """, rows, page_size=100) + + conn.commit() + print(f" central_kitchen_manufacturing_cost_pool 生成完成: {len(rows)} 行") + cur.close() + return len(rows) + + +def main(): + parser = argparse.ArgumentParser(description='派生分析数据导入') + parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)') + parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)') + parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)') + + args = parser.parse_args() + + conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak') + conn.autocommit = False + + diag_date = args.diagnosis_date or datetime.date.today().isoformat() + + try: + print(f"\n=== 派生分析数据生成 ===") + print(f"数据库: {args.db}, 月份: {args.month}") + + print("\n--- 菜品诊断快照 ---") + generate_dish_diagnosis(conn, diag_date) + + print("\n--- 中央厨房制造成本池 ---") + generate_manufacturing_cost_pool(conn, args.month) + + print("\n=== 生成完成 ===") + + except Exception as e: + conn.rollback() + print(f"错误: {e}", file=sys.stderr) + raise + finally: + conn.close() + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/import_may_data.sh b/db/pipeline/import_may_data.sh new file mode 100755 index 0000000..ad5ca46 --- /dev/null +++ b/db/pipeline/import_may_data.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# ============================================================ +# 5月数据导入脚本 +# 用法: bash import_may_data.sh [数据库名] +# 默认数据库: bill_query_test +# ============================================================ +set -e + +DB="${1:-bill_query_test}" +MONTH="2026-05-01" +DIAG_DATE="2026-08-18" + +# 数据根目录 +DATA="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/5月" +DATA4="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/4月" +PIPELINE="/Users/freedak/Documents/AIDashboard/SBrainCO3/db/pipeline" +PY=python3 + +echo "============================================================" +echo " 5月数据导入 → 数据库: ${DB}" +echo " 月份: ${MONTH}" +echo "============================================================" + +# ============================================================ +# Step 1: 薪资 + 考勤 +# ============================================================ +echo "" +echo "=== [1/9] 薪资 + 考勤 ===" +${PY} "${PIPELINE}/import_salary_attendance.py" \ + --month "${MONTH}" \ + --db "${DB}" \ + --salary-file "${DATA}/营业数据/2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx" \ + --attendance-file "${DATA}/营业数据/2026年5月考勤数据表.xlsx" + +# ============================================================ +# Step 2: bill_records + dish_sales_details（5月专用脚本） +# 5月格式: 全渠道订单明细(51列) + 品项销售明细(50列) +# ============================================================ +echo "" +echo "=== [2/9] bill_records + 菜品销售明细 ===" +${PY} "${PIPELINE}/import_bill_records_may.py" \ + --month "${MONTH}" \ + --orders "${DATA}/营业数据/正餐事业部_全渠道订单明细_20260730_1112_1785381233074.xlsx" \ + --items "${DATA}/营业数据/正餐事业部_品项销售明细_20260730_1128_1785382191936.xlsx" \ + --db "${DB}" + +# ============================================================ +# Step 4: 库存成本（盘点倒挤成本） +# ============================================================ +echo "" +echo "=== [4/9] 库存成本 ===" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset inventory \ + --month "${MONTH}" \ + --file "${DATA}/供应链数据/4.13盘点倒挤成本报表2026年5月.xlsx" \ + --db "${DB}" + +# ============================================================ +# Step 5: 菜品成本BOM +# ============================================================ +echo "" +echo "=== [5/9] 菜品成本BOM ===" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset dish_cost \ + --month "${MONTH}" \ + --file "${DATA}/营业数据/2026年5月菜品成本分析报表.xlsx" \ + --db "${DB}" + +# ============================================================ +# Step 6: 营业费用 +# ============================================================ +echo "" +echo "=== [6/9] 营业费用 ===" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset operating_expense \ + --month "${MONTH}" \ + --file "${DATA}/2026年5月营业费用分析.xls" \ + --db "${DB}" + +# ============================================================ +# Step 7: 中央厨房（4个子文件） +# ============================================================ +echo "" +echo "=== [7/9] 中央厨房 ===" +echo " [7a] 完工入库" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset central_kitchen --ck-type finished \ + --month "${MONTH}" \ + --file "${DATA}/中央厨房/2026-05-01--2026-05-31 23 59 59完工入库统计分析报表.xlsx" \ + --db "${DB}" + +echo " [7b] 配方耗用" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset central_kitchen --ck-type recipe \ + --month "${MONTH}" \ + --file "${DATA}/中央厨房/按配方导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \ + --db "${DB}" + +echo " [7c] 货品耗用" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset central_kitchen --ck-type material \ + --month "${MONTH}" \ + --file "${DATA}/中央厨房/按货品导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \ + --db "${DB}" + +echo " [7d] 加工单价" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset central_kitchen --ck-type processing \ + --month "${MONTH}" \ + --file "${DATA}/中央厨房/加工单价分析报表.xlsx" \ + --db "${DB}" + +# ============================================================ +# Step 8: 配送明细（2个文件） +# ============================================================ +echo "" +echo "=== [8/9] 配送明细 ===" +echo " [8a] 1-15日" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset distribution \ + --month "${MONTH}" \ + --file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月1-15日.xlsx" \ + --db "${DB}" + +echo " [8b] 16-31日" +${PY} "${PIPELINE}/import_monthly_data.py" \ + --dataset distribution \ + --month "${MONTH}" \ + --file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月16-31日.xlsx" \ + --db "${DB}" + +# ============================================================ +# Step 9: 门店位置与映射表（复用4月各店信息文件） +# ============================================================ +echo "" +echo "=== [9/9] 门店位置与映射表 ===" +${PY} "${PIPELINE}/import_store_location.py" \ + --file "${DATA4}/各店信息新（20260508）.xls" \ + --db "${DB}" + +# ============================================================ +# Step 10: 派生分析数据 +# ============================================================ +echo "" +echo "=== [10] 派生分析数据 ===" +${PY} "${PIPELINE}/import_derived_data.py" \ + --month "${MONTH}" \ + --db "${DB}" \ + --diagnosis-date "${DIAG_DATE}" + +# ============================================================ +# Step 11: 物化视图刷新 +# ============================================================ +echo "" +echo "=== [11] 物化视图刷新 ===" +${PY} "${PIPELINE}/refresh_materialized_views.py" \ + --db "${DB}" + +# ============================================================ +# Step 12: 验证 +# ============================================================ +echo "" +echo "=== [12] 数据一致性验证 ===" +${PY} "${PIPELINE}/verify.py" \ + --db "${DB}" || true + +echo "" +echo "============================================================" +echo " 5月数据导入完成" +echo "============================================================" diff --git a/db/pipeline/import_monthly_data.py b/db/pipeline/import_monthly_data.py new file mode 100644 index 0000000..d1ca63f --- /dev/null +++ b/db/pipeline/import_monthly_data.py @@ -0,0 +1,1301 @@ +#!/usr/bin/env python3 +""" +通用月度数据导入脚本 +支持: 库存成本、菜品销售明细、菜品成本BOM、营业费用、中央厨房、配送明细、采购货品 +用法: python3 import_monthly_data.py --dataset inventory --month 2026-04-01 --file --db bill_query_test +""" +import argparse +import hashlib +import os +import sys +import openpyxl +import pandas as pd +import psycopg2 +from psycopg2.extras import execute_values +from datetime import datetime + +DB_CONFIG = { + 'host': 'localhost', + 'port': 5432, + 'user': 'freedak', + 'password': '', +} + +def file_sha256(filepath): + h = hashlib.sha256() + with open(filepath, 'rb') as f: + for chunk in iter(lambda: f.read(8192), b''): + h.update(chunk) + return h.hexdigest() + +def to_num(val): + if val is None or val == '': + return None + if isinstance(val, float) and pd.isna(val): + return None + try: + f = float(val) + import math + if math.isinf(f) or math.isnan(f): + return None + return f + except (ValueError, TypeError): + return None + +def to_text(val): + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + s = str(val).strip() + if s.lower() == 'nan' or s.lower() == 'none': + return None + return s if s else None + +def to_date(val): + """Convert value to date string YYYY-MM-DD or None""" + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + if isinstance(val, (datetime,)): + return val.strftime('%Y-%m-%d') + s = str(val).strip() + if not s or s.lower() == 'nan' or s.lower() == 'none': + return None + # Try pandas Timestamp + try: + ts = pd.Timestamp(s) + if pd.isna(ts): + return None + return ts.strftime('%Y-%m-%d') + except (ValueError, TypeError): + return s # Return as-is, let DB handle it + +def clean_percent(val): + """Convert percentage value: string '81.23%' -> 81.23, float 0.8123 -> 81.23""" + if val is None: + return None + if isinstance(val, float) and pd.isna(val): + return None + if isinstance(val, str): + s = val.strip().replace('%', '') + if not s or s.lower() == 'nan': + return None + try: + return float(s) + except ValueError: + return None + num = float(val) + return num * 100 if abs(num) <= 1 else num + +def connect(dbname='bill_query'): + cfg = DB_CONFIG.copy() + cfg['dbname'] = dbname + return psycopg2.connect(**cfg) + +# ============================================================ +# 库存成本导入 +# ============================================================ +def import_inventory_cost(conn, filepath, report_month): + """库存倒挤成本导入""" + print(f"\n=== 库存成本导入 ===") + print(f"文件: {filepath}") + print(f"月份: {report_month}") + + sha = file_sha256(filepath) + source_file = os.path.basename(filepath) + + wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True) + ws = wb.active + + # Row 2: group headers, Row 3: sub-headers, Row 4+: data + rows = list(ws.iter_rows(min_row=4, values_only=True)) + data_rows = [r for r in rows if r[4] is not None and str(r[4]).strip()] + print(f"数据行数: {len(data_rows)}") + + cur = conn.cursor() + # Check if already imported + cur.execute( + "SELECT 1 FROM public.inventory_cost_import_log WHERE source_file = %s AND report_month = %s", + (source_file, report_month) + ) + if cur.fetchone(): + print("已导入过，跳过") + wb.close() + return + + # Insert import log + cur.execute( + """INSERT INTO public.inventory_cost_import_log (source_file, report_month, source_rows, imported_rows, status) + VALUES (%s, %s, %s, %s, 'importing')""", + (source_file, report_month, len(data_rows), len(data_rows)) + ) + + # Batch insert + insert_sql = """INSERT INTO public.inventory_cost_records ( + report_month, source_file, source_row, + brand, region, cost_unit_name, cost_unit_code, item_name, specification, + major_category, minor_category, finance_category, item_code, unit, + opening_quantity, opening_unit_cost, opening_amount, opening_tax, opening_tax_inclusive, + purchase_quantity, purchase_unit_cost, purchase_amount, purchase_tax, purchase_tax_inclusive, + ending_quantity, ending_unit_cost, ending_amount, ending_tax, ending_tax_inclusive, + consumption_quantity, consumption_unit_cost, consumption_amount, consumption_tax, consumption_tax_inclusive, + conversion_precision_amount, conversion_precision_tax_inclusive, + return_loss_amount, return_loss_tax_inclusive + ) VALUES %s""" + + batch = [] + batch_size = 500 + for idx, r in enumerate(data_rows): + source_row = idx + 4 + consumption_amount = to_num(r[28]) + vals = ( + report_month, source_file, source_row, + to_text(r[0]), to_text(r[1]), to_text(r[2]), to_text(r[3]), + to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]), + to_text(r[9]), to_text(r[10]), + to_num(r[11]), to_num(r[12]), to_num(r[13]), to_num(r[14]), to_num(r[15]), + to_num(r[16]), to_num(r[17]), to_num(r[18]), to_num(r[19]), to_num(r[20]), + to_num(r[21]), to_num(r[22]), to_num(r[23]), to_num(r[24]), to_num(r[25]), + to_num(r[26]), to_num(r[27]), consumption_amount, to_num(r[29]), to_num(r[30]), + to_num(r[31]), to_num(r[32]), + to_num(r[33]), to_num(r[34]), + ) + batch.append(vals) + if len(batch) >= batch_size: + execute_values(cur, insert_sql, batch, page_size=500) + conn.commit() + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values(cur, insert_sql, batch, page_size=500) + conn.commit() + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + + # Update log status + cur.execute( + "UPDATE public.inventory_cost_import_log SET status = 'completed' WHERE source_file = %s AND report_month = %s", + (source_file, report_month) + ) + conn.commit() + wb.close() + print(f"库存成本导入完成: {len(data_rows)} 行") + +# ============================================================ +# 菜品销售明细导入 +# ============================================================ +def import_dish_sales(conn, filepath, report_month, source_file_label=None): + """菜品销售明细导入（支持单文件或目录）""" + print(f"\n=== 菜品销售明细导入 ===") + + files = [] + if os.path.isdir(filepath): + for f in sorted(os.listdir(filepath)): + if f.endswith('.xlsx') and not f.startswith('.'): + files.append(os.path.join(filepath, f)) + else: + files.append(filepath) + + print(f"文件数: {len(files)}") + + cur = conn.cursor() + total_imported = 0 + + for file_idx, fpath in enumerate(files): + fname = os.path.basename(fpath) + print(f"\n[{file_idx+1}/{len(files)}] {fname}") + + file_size = os.path.getsize(fpath) + + # Check if already imported + cur.execute( + "SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s", + (fname,) + ) + if cur.fetchone(): + print(f" 已导入过, 跳过") + continue + + # Use pandas to read Excel (handles read_only issues with these files) + # Header is on row 3 (0-indexed: header=2) + df = pd.read_excel(fpath, header=2, engine='openpyxl') + # Drop rows where first column is empty + df = df.dropna(subset=[df.columns[0]]) + data_rows = df.values.tolist() + headers = list(df.columns) + print(f" 数据行数: {len(data_rows)}") + + if not data_rows: + continue + + # Insert import log + cur.execute( + """INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status) + VALUES (%s, %s, %s, 'importing')""", + (fname, file_size, len(data_rows)) + ) + + # Map headers to columns by name + col_map = {} + for i, h in enumerate(headers): + if h and str(h).strip(): + col_map[str(h).strip()] = i + + # Build column mapping based on known header names + field_mapping = { + '门店编码': 'store_code', + '门店名称': 'store_name', + '门店': 'store_name', + '账单号': 'bill_no', + '订单号': 'bill_no', + '人数': 'guest_count', + '开台时间': 'opened_at', + '结账时间': 'closed_at', + '下单时间': 'ordered_at', + '菜品名称': 'dish_name', + '品项名称': 'dish_name', + '分类': 'category_level1', + '菜品分类': 'category_level1', + '出品部门': 'production_department', + '销量': 'sales_quantity', + '金额': 'gross_amount', + '实收金额': 'received_amount', + '实收': 'received_amount', + '单价': 'unit_price', + '单位': 'unit', + '业务类型': 'business_type', + '桌台/取餐号': 'table_or_pickup_no', + '台号': 'table_or_pickup_no', + '做法': 'preparation_method', + } + + # Determine actual columns + cols_to_insert = {} + for h_name, db_col in field_mapping.items(): + if h_name in col_map: + cols_to_insert[db_col] = col_map[h_name] + + # source_file + source_row + mapped columns + all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys()) + col_names = ','.join(all_cols) + + numeric_cols = {'guest_count', 'sales_quantity', 'gross_amount', 'received_amount', 'unit_price'} + + batch = [] + for idx, r in enumerate(data_rows): + source_row = idx + 4 # data starts at row 4 (header=3) + vals = [fname, source_row] + for db_col, excel_idx in cols_to_insert.items(): + v = r[excel_idx] if excel_idx < len(r) else None + if db_col in numeric_cols: + vals.append(to_num(v)) + else: + vals.append(to_text(v)) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + + total_imported += len(data_rows) + + # Update log status + cur.execute( + "UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s", + (fname,) + ) + conn.commit() + + print(f"\n菜品销售明细导入完成: 共 {total_imported} 行") + +# ============================================================ +# 菜品成本/BOM导入 +# ============================================================ +def import_dish_cost(conn, filepath, report_month): + """菜品成本分析导入：同时导入summary和detail两张表 + Excel结构：Row0=标题, Row1=主表头, Row2=子表头(原料区), Row3+=数据 + 每个菜品有多行原料明细，菜品列(col0-12)只在首行填充""" + print(f"\n=== 菜品成本/BOM导入 ===") + print(f"文件: {filepath}") + + sha = file_sha256(filepath) + source_file = os.path.basename(filepath) + + cur = conn.cursor() + cur.execute( + "SELECT 1 FROM public.dish_cost_analysis_import_log WHERE source_file = %s AND file_sha256 = %s", + (source_file, sha) + ) + if cur.fetchone(): + print("已导入过，跳过") + return + + # Read with pandas + raw_df = pd.read_excel(filepath, header=None, engine='openpyxl') + print(f" 总行数: {len(raw_df)}") + + # Headers: Row 1 (main) + Row 2 (sub for material detail cols 13+) + main_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[1]] + sub_headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[2]] + + # Build combined headers + headers = [] + for i in range(len(main_headers)): + m = main_headers[i] + s = sub_headers[i] + if m and s and m != s: + headers.append(f"{m}_{s}") + elif m: + headers.append(m) + elif s: + headers.append(s) + else: + headers.append(f'col_{i}') + + data_start = 3 + df = raw_df.iloc[data_start:].copy() + + # Identify summary rows: col 0 originally non-empty (first row of each dish) + # Detail rows: col 13 non-empty (material name) + summary_mask = df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan') + detail_mask = df[13].notna() & (df[13].astype(str).str.strip() != '') & (df[13].astype(str).str.strip() != 'nan') + + summary_df = df[summary_mask].copy() + detail_df = df[detail_mask].copy() + + # Filter out 合计 rows + summary_df = summary_df[~summary_df[0].astype(str).str.strip().isin(['合计', '总计'])] + + print(f" 菜品汇总行数: {len(summary_df)}") + print(f" 原料明细行数: {len(detail_df)}") + + if len(summary_df) == 0: + print("无菜品汇总数据，跳过") + return + + # Calculate totals for log + sales_amount_total = to_num(summary_df[7].sum()) if 7 < len(summary_df.columns) else 0 + theoretical_cost_total = to_num(summary_df[8].sum()) if 8 < len(summary_df.columns) else 0 + actual_cost_total = to_num(summary_df[9].sum()) if 9 < len(summary_df.columns) else 0 + cost_variance_total = to_num(summary_df[12].sum()) if 12 < len(summary_df.columns) else 0 + + # Parse report period from filename or use report_month + import re + date_match = re.search(r'(\d{4})年(\d{1,2})月', source_file) + if date_match: + year, month = date_match.groups() + period_start = f"{year}-{int(month):02d}-01" + period_end = (pd.Timestamp(period_start) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d') + else: + period_start = report_month + period_end = (pd.Timestamp(report_month) + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d') + + # Insert import log + cur.execute( + """INSERT INTO public.dish_cost_analysis_import_log + (source_file, file_sha256, sheet_name, workbook_data_rows, + dish_summary_rows, material_detail_rows, + sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total, + report_period_start, report_period_end, import_status, report_month) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'loading', %s) RETURNING import_id""", + (source_file, sha, 'Worksheet', len(df), len(summary_df), len(detail_df), + sales_amount_total, theoretical_cost_total, actual_cost_total, cost_variance_total, + period_start, period_end, report_month) + ) + import_id = cur.fetchone()[0] + + # Import summary rows + summary_mapping = { + 'dish_name': 0, # 菜品名称 + 'dish_code': 1, # 菜品编码 + 'price': 2, # 售价 + 'category_level1': 3, # 菜品大类 + 'category_level2': 4, # 菜品小类 + 'dish_unit': 5, # 菜品单位 + 'sales_quantity': 6, # 销售数量 + 'sales_amount': 7, # 销售金额(元) + 'theoretical_cost': 8, # 理论成本(元) + 'actual_cost': 9, # 实际成本(元) + 'theoretical_margin_rate_pct': 10, # 理论毛利率 + 'actual_margin_rate_pct': 11, # 实际毛利率 + 'cost_variance_amount': 12, # 成本差异金额(元) + } + + summary_cols = ['import_id', 'source_row', 'dish_sequence'] + list(summary_mapping.keys()) + summary_col_names = ','.join(summary_cols) + + # Fields that need clean_percent vs to_num vs to_text + summary_percent_fields = {'theoretical_margin_rate_pct', 'actual_margin_rate_pct'} + summary_text_fields = {'dish_name', 'dish_code', 'category_level1', 'category_level2', 'dish_unit'} + + batch = [] + for idx, (row_idx, r) in enumerate(summary_df.iterrows()): + source_row = int(row_idx) + data_start + 1 # 1-indexed Excel row + vals = [import_id, source_row, idx + 1] # dish_sequence + for db_col, excel_idx in summary_mapping.items(): + v = r[excel_idx] if excel_idx < len(r) else None + if v is None or (isinstance(v, float) and pd.isna(v)): + vals.append(None) + elif db_col in summary_percent_fields: + vals.append(clean_percent(v)) + elif db_col in summary_text_fields: + vals.append(to_text(v)) + else: + vals.append(to_num(v)) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" Summary: 已导入 {idx + 1}/{len(summary_df)} 行") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.dish_cost_analysis_summary ({summary_col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" Summary: 已导入 {len(summary_df)}/{len(summary_df)} 行") + + # Get summary_ids for detail linking + cur.execute("SELECT summary_id, source_row FROM public.dish_cost_analysis_summary WHERE import_id = %s ORDER BY source_row", (import_id,)) + summary_id_map = {r[1]: r[0] for r in cur.fetchall()} + + # Import detail rows + # Detail columns: col13=原料, col14=单位, col15=规格, col16=属性, + # col17=理论数量, col18=理论金额, col19=实际数量, col20=实际金额, + # col21=损耗数量, col22=损耗金额, col23=数量量比 + detail_mapping = { + 'material_name': 13, + 'material_unit': 14, + 'specification': 15, + 'material_type': 16, + 'theoretical_quantity': 17, + 'theoretical_amount': 18, + 'actual_quantity': 19, + 'actual_amount': 20, + 'loss_quantity': 21, + 'loss_amount': 22, + 'loss_quantity_rate_pct': 23, + } + + detail_cols = ['import_id', 'summary_id', 'source_row', 'material_sequence'] + list(detail_mapping.keys()) + ['theoretical_quantity_per_dish', 'actual_quantity_per_dish'] + detail_col_names = ','.join(detail_cols) + + # For each detail row, find the parent summary's source_row + # The parent dish's source_row is the first non-empty col0 at or before this row + # We already forward-filled col0, so we can match by looking up the original row + # Actually, we need to find which summary row each detail row belongs to + # Since we forward-filled, the dish_name (col0) tells us which dish it belongs to + + # Build a map: dish_name -> summary source_row + dish_to_summary_row = {} + for idx, (row_idx, r) in enumerate(summary_df.iterrows()): + source_row = int(row_idx) + data_start + 1 + dish_name = str(r[0]).strip() if pd.notna(r[0]) else '' + if dish_name: + dish_to_summary_row[dish_name] = source_row + + # For detail rows, find parent dish by looking backward in original data for nearest non-empty col0 + # Build a sorted list of summary source_rows for binary search + summary_source_rows = sorted(summary_id_map.keys()) + + batch = [] + mat_seq = 0 + current_dish = None + for idx, (row_idx, r) in enumerate(detail_df.iterrows()): + source_row = int(row_idx) + data_start + 1 + excel_row_idx = int(row_idx) # 0-indexed in raw_df + + # Find parent dish: look backward from current row for nearest non-empty col0 + parent_dish_name = None + for back_idx in range(excel_row_idx, data_start - 1, -1): + val = raw_df.iloc[back_idx, 0] + if pd.notna(val) and str(val).strip() and str(val).strip() not in ('合计', '总计', 'nan'): + parent_dish_name = str(val).strip() + break + + if parent_dish_name != current_dish: + current_dish = parent_dish_name + mat_seq = 0 + mat_seq += 1 + + # Find summary_id + parent_source_row = dish_to_summary_row.get(parent_dish_name) + summary_id = summary_id_map.get(parent_source_row) if parent_source_row else None + + vals = [import_id, summary_id, source_row, mat_seq] + # Get sales_quantity from parent summary for per_dish calculation + parent_sales_qty = None + if parent_source_row: + for _, srow in summary_df.iterrows(): + if int(srow.name) + data_start + 1 == parent_source_row: + parent_sales_qty = to_num(srow[6]) if pd.notna(srow[6]) else None + break + + for db_col, excel_idx in detail_mapping.items(): + v = r[excel_idx] if excel_idx < len(r) else None + if v is None or (isinstance(v, float) and pd.isna(v)): + vals.append(None) + elif db_col == 'loss_quantity_rate_pct': + vals.append(clean_percent(v)) + elif db_col in ('material_name', 'material_unit', 'specification', 'material_type'): + vals.append(to_text(v)) + else: + vals.append(to_num(v)) + + # Calculate per_dish quantities + theoretical_qty = vals[-len(detail_mapping) + 6] # theoretical_quantity in vals + actual_qty = vals[-len(detail_mapping) + 8] # actual_quantity in vals + # Actually let's just get them directly + theoretical_qty = to_num(r[17]) if pd.notna(r[17]) else None + actual_qty = to_num(r[19]) if pd.notna(r[19]) else None + if theoretical_qty is not None and parent_sales_qty not in (None, 0): + vals.append(theoretical_qty / parent_sales_qty) + else: + vals.append(None) + if actual_qty is not None and parent_sales_qty not in (None, 0): + vals.append(actual_qty / parent_sales_qty) + else: + vals.append(None) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" Detail: 已导入 {idx + 1}/{len(detail_df)} 行") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.dish_cost_analysis_material_detail ({detail_col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" Detail: 已导入 {len(detail_df)}/{len(detail_df)} 行") + + # Update log status + cur.execute( + "UPDATE public.dish_cost_analysis_import_log SET import_status = 'success' WHERE import_id = %s", + (import_id,) + ) + conn.commit() + print(f"菜品成本导入完成: {len(summary_df)} 菜品, {len(detail_df)} 原料明细") + +# ============================================================ +# 营业费用导入 +# ============================================================ +def import_operating_expense(conn, filepath, report_month, expense_type='operating'): + """营业费用导入：宽表转长表，每行=科目×门店""" + print(f"\n=== 营业费用导入 ({expense_type}) ===") + print(f"文件: {filepath}") + + sha = file_sha256(filepath) + source_file = os.path.basename(filepath) + + cur = conn.cursor() + cur.execute( + "SELECT 1 FROM public.operating_expense_import_log WHERE source_file = %s AND file_sha256 = %s", + (source_file, sha) + ) + if cur.fetchone(): + print("已导入过，跳过") + return + + # Read .xls file with pandas + raw_df = pd.read_excel(filepath, header=None, engine='xlrd') + print(f" 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}") + + # Row 0: headers (科目编码, 科目名称, 统计方式, 方向3, 合计金额, 01 大钟寺金额, ...) + headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]] + + # Data starts from row 1 + df = raw_df.iloc[1:].copy() + + # Filter out summary rows (col 0 = '费用科目' or empty) + df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != '费用科目')] + + print(f" 科目行数: {len(df)}") + + # Parse store columns: col 5+ are store columns (skip col 0-4: code, name, direction, direction3, total) + # Extract store code and name from header + store_cols = [] + for ci in range(5, len(headers)): + h = headers[ci] + if not h: + continue + # Parse "01 大钟寺金额" -> code="01", name="大钟寺" + import re + m = re.match(r'([A-Z]\d+)\s+(.+?)金额$', h) + if m: + store_cols.append((ci, m.group(1), m.group(2))) + else: + # Keep as-is with full header + store_cols.append((ci, None, h)) + + print(f" 门店列数: {len(store_cols)}") + + # Calculate total amount for log + total_amount = 0 + for _, r in df.iterrows(): + v = to_num(r[4]) if pd.notna(r[4]) else 0 # col 4 = 合计金额 + total_amount += v if v else 0 + + # Insert import log + cur.execute( + """INSERT INTO public.operating_expense_import_log + (report_month, source_file, source_sheet, file_sha256, source_total_amount, imported_record_count) + VALUES (%s, %s, %s, %s, %s, 0) RETURNING import_id""", + (report_month, source_file, 'Worksheet', sha, total_amount) + ) + import_id = cur.fetchone()[0] + + # Insert records: long format (one row per account × store) + insert_cols = ['import_id', 'report_month', 'source_file', 'source_sheet', + 'source_row', 'source_column', 'account_code', 'account_name', + 'accounting_direction', 'is_summary_account', 'parent_account_code', + 'cost_unit_source_header', 'cost_unit_source_code', + 'cost_unit_source_name', 'amount'] + col_names = ','.join(insert_cols) + + batch = [] + total_records = 0 + + for row_idx, (_, r) in enumerate(df.iterrows()): + account_code = to_text(r[0]) + account_name = to_text(r[1]) + direction = to_text(r[2]) # 统计方式 (借方/贷方) + excel_row = int(r.name) + 1 # 1-indexed Excel row + + # Determine if summary account: 7-digit codes (5032401) are sub-accounts of 5-digit (50324) + is_summary = False + parent_code = None + if account_code and len(account_code) == 7: + parent_code = account_code[:5] + is_summary = False + elif account_code and len(account_code) == 5: + is_summary = True + + for col_idx, store_code, store_name in store_cols: + amount = to_num(r[col_idx]) if col_idx < len(r) and pd.notna(r[col_idx]) else None + if amount is None: + continue # Skip NULL amounts, but keep 0 amounts + + vals = ( + import_id, + report_month, + source_file, + 'Worksheet', + excel_row, + col_idx + 1, # 1-indexed column + account_code, + account_name, + direction, + is_summary, + parent_code, + headers[col_idx], # full header text + store_code, + store_name, + amount, + ) + batch.append(vals) + total_records += 1 + + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {total_records} 条记录") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.operating_expense_records ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {total_records} 条记录") + + # Update log + cur.execute( + "UPDATE public.operating_expense_import_log SET imported_record_count = %s WHERE import_id = %s", + (total_records, import_id) + ) + conn.commit() + print(f"营业费用导入完成: {total_records} 条记录") + +# ============================================================ +# 中央厨房导入 (4表) +# ============================================================ +def import_central_kitchen(conn, filepath, report_month, ck_type='finished'): + """中央厨房导入: finished/recipe/material/processing + 使用pandas读取，精确映射列，raw_data存储原始行JSON""" + import json + type_names = { + 'finished': '完工入库', + 'recipe': '配方耗用', + 'material': '材料日报', + 'processing': '加工单价', + } + table_map = { + 'finished': 'central_kitchen_finished_receipt', + 'recipe': 'central_kitchen_recipe_consumption', + 'material': 'central_kitchen_material_daily', + 'processing': 'central_kitchen_processing_cost', + } + table_name = table_map[ck_type] + + print(f"\n=== 中央厨房-{type_names.get(ck_type, ck_type)} 导入 ===") + print(f"文件: {filepath}") + + sha = file_sha256(filepath) + source_file = os.path.basename(filepath) + + cur = conn.cursor() + cur.execute( + "SELECT 1 FROM public.central_kitchen_import_log WHERE source_file = %s AND file_sha256 = %s", + (source_file, sha) + ) + if cur.fetchone(): + print("已导入过，跳过") + return + + # Header row varies by type: + # finished: row 2 (0-indexed) with sub-header row 3 + # recipe: row 0 + # material: row 0 with sub-header row 1 + # processing: row 2 + header_rows = { + 'finished': 2, # 0-indexed, merged header rows 2+3 + 'recipe': 0, + 'material': 1, # skip row 0 (group header), use row 1 + 'processing': 2, + } + hr = header_rows[ck_type] + + # Read raw without header to get all rows + raw_df = pd.read_excel(filepath, header=None, engine='openpyxl') + + # Build headers from the appropriate row(s) + if ck_type == 'finished': + # Row 2 has group headers, row 3 has sub-headers + # Don't fill forward - empty cells stay empty (merged cells) + h1 = raw_df.iloc[2].fillna('').astype(str).str.strip() + h2 = raw_df.iloc[3].fillna('').astype(str).str.strip() + headers = [] + for i in range(len(h1)): + g = h1[i] + s = h2[i] + if g and s: + headers.append(f"{g}_{s}") + elif g: + headers.append(g) + elif s: + headers.append(s) + else: + headers.append(f'col_{i}') # Placeholder for empty headers + data_start = 4 + elif ck_type == 'material': + # Row 0 has group headers, row 1 has sub-headers + h1 = raw_df.iloc[0].fillna('').astype(str).str.strip() + h2 = raw_df.iloc[1].fillna('').astype(str).str.strip() + headers = [] + current_group = '' + for i in range(len(h1)): + g = h1[i] if h1[i] else current_group + if g: + current_group = g + s = h2[i] if h2[i] else '' + headers.append(f"{g}_{s}" if s else g) + data_start = 2 + else: + # processing: row 2 (main) + row 3 (sub) — like finished + h1 = raw_df.iloc[hr].fillna('').astype(str).str.strip() + h2 = raw_df.iloc[hr + 1].fillna('').astype(str).str.strip() + headers = [] + for i in range(len(h1)): + g = h1[i] + s = h2[i] + if g and s and g != s: + headers.append(f"{g}_{s}") + elif g and g != 'nan': + headers.append(g) + elif s and s != 'nan': + headers.append(s) + else: + headers.append(f'col_{i}') + data_start = hr + 2 + + # Extract data rows + df = raw_df.iloc[data_start:] + df = df.dropna(subset=[df.columns[0]]) + # Filter out 合计/total rows by checking all string cells for '合计' + mask = pd.Series([True] * len(df), index=df.index) + for ci in range(min(15, df.shape[1])): + mask = mask & ~df[ci].astype(str).str.strip().isin(['合计', '总计', 'Total', 'total']) + # Filter out footer rows (e.g. "制表人:xxx") + mask = mask & ~df[0].astype(str).str.strip().str.startswith('制表人') + mask = mask & ~df[0].astype(str).str.strip().str.startswith('打印') + mask = mask & ~df[0].astype(str).str.strip().str.startswith('第') + df = df[mask] + data_rows = df.values.tolist() + print(f"数据行数: {len(data_rows)}") + + if not data_rows: + return + + # Insert import log + cur.execute( + """INSERT INTO public.central_kitchen_import_log + (report_month, dataset_type, source_file, source_sheet, file_sha256, + workbook_rows, imported_rows, excluded_rows, status) + VALUES (%s, %s, %s, %s, %s, %s, %s, 0, 'loading') RETURNING import_id""", + (report_month, ck_type, source_file, 'Worksheet', sha, len(data_rows), len(data_rows)) + ) + import_id = cur.fetchone()[0] + + # Get DB columns (exclude record_id, import_id, source_row, raw_data) + cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{table_name}' ORDER BY ordinal_position") + db_cols = [r[0] for r in cur.fetchall()] + insertable_cols = [c for c in db_cols if c not in ('record_id', 'import_id', 'source_row', 'raw_data')] + + # Build header name -> excel index map (keep first occurrence) + col_map = {} + for i, h in enumerate(headers): + if h and str(h).strip(): + key = str(h).strip() + if key not in col_map: + col_map[key] = i + + # Type-specific column mappings (db_col -> list of possible header names) + mappings = { + 'finished': { + 'receipt_date': ['入库日期'], + 'workshop': ['col_1'], + 'processing_warehouse': ['加工仓库'], + 'recipe_name': ['配方名称'], + 'product_name': ['货品名称'], + 'specification': ['规格'], + 'unit': ['库存单位'], + 'return_quantity': ['退库数量'], + 'return_amount': ['退库金额'], + 'inbound_avg_unit_price': ['平均单价'], + 'inbound_quantity': ['入库数量'], + 'inbound_amount': ['入库金额'], + 'raw_material_cost': ['加工利润分析_原料成本', '原料成本'], + 'source_fee_cost': ['费用成本'], + 'source_processing_profit': ['增值金额（加工利润）'], + 'source_margin_rate': ['毛利率'], + 'source_cost_rate': ['成本率'], + 'theoretical_inbound_quantity': ['加工达成率分析_理论入库数量', '理论入库数量'], + 'actual_inbound_quantity': ['实际入库数量'], + 'inbound_difference_quantity': ['差异数量'], + 'achievement_rate': ['加工达成率'], + 'standard_expected_quantity': ['加工应产量分析_标准应产量', '标准应产量'], + 'average_expected_quantity': ['平均应产量'], + 'expected_quantity_variance_rate': ['应产量差异量比'], + 'water_cost': ['水费'], + 'electricity_cost': ['电费'], + 'gas_cost': ['燃气费'], + 'labor_cost': ['人工费'], + 'other_cost': ['其他费用'], + 'detailed_fee_total': ['费用合计'], + }, + 'recipe': { + 'business_date': ['日期'], + 'processing_warehouse': ['加工仓库'], + 'finance_category_major': ['财务大类'], + 'finance_category_minor': ['财务小类'], + 'category_major': ['所属大类'], + 'category_minor': ['所属小类'], + 'item_id': ['货品ID'], + 'item_code': ['编码'], + 'item_name': ['货品名称'], + 'specification': ['规格'], + 'unit': ['库存单位'], + 'unit_price_excl_tax': ['未税单价'], + 'unit_price': ['单价'], + 'recipe_name': ['配方名称'], + 'standard_usage': ['标准用量'], + 'actual_average_usage': ['实际平均用量'], + 'planned_output_quantity': ['计划加工数量'], + 'actual_output_quantity': ['实际加工数量'], + 'output_unit': ['加工数量库存单位'], + 'theoretical_quantity': ['理论用量'], + 'net_quantity': ['净料用量'], + 'theoretical_amount': ['理论金额'], + 'issue_quantity': ['领用数量'], + 'issue_amount': ['领用金额'], + 'source_inventory_cost_quantity': ['成本数量（进销存）'], + 'source_inventory_cost_amount': ['成本金额（进销存）'], + 'actual_vs_theory_quantity': ['实际与理论差异数量'], + 'actual_vs_theory_amount': ['实际与理论差异金额'], + 'actual_vs_theory_rate': ['实际差异量比'], + 'issue_vs_theory_quantity': ['领用与理论差异数量'], + 'issue_vs_theory_amount': ['领用与理论差异金额'], + 'issue_vs_theory_rate': ['领用差异量比'], + 'actual_yield': ['实际出成率'], + 'recipe_yield': ['配方出成率'], + 'yield_difference': ['出成率差异'], + }, + 'material': { + 'business_date': ['日期'], + 'finance_category_major': ['财务大类'], + 'finance_category_minor': ['财务小类'], + 'category_major': ['所属大类'], + 'category_minor': ['所属小类'], + 'item_id': ['货品ID'], + 'item_code': ['编码'], + 'item_name': ['货品名称'], + 'specification': ['规格'], + 'unit': ['库存单位'], + }, + 'processing': { + 'report_month': ['report_month'], # Will be set from parameter + 'item_id': ['货品ID'], + 'recipe_name': ['配方名称'], + 'product_code': ['货品编码'], + 'product_name': ['货品名称'], + 'category_major': ['所属大类'], + 'category_minor': ['所属小类'], + 'finance_category_major': ['财务大类'], + 'finance_category_minor': ['财务小类'], + 'specification': ['规格'], + 'unit': ['单位'], + 'inbound_quantity': ['入库数量'], + 'inbound_avg_unit_price': ['入库均价'], + 'theoretical_unit_cost': ['理论单价'], + 'standard_unit_cost': ['标准单价'], + 'actual_unit_cost': ['实际单价'], + 'theoretical_cost': ['理论成本'], + 'standard_cost': ['标准成本'], + 'actual_cost': ['实际成本'], + }, + } + + field_mapping = mappings[ck_type] + cols_to_insert = {} + fixed_values = {} # Values not from Excel (e.g. report_month) + for db_col, h_names in field_mapping.items(): + if db_col not in insertable_cols: + continue + found = False + for h_name in h_names: + if h_name in col_map: + cols_to_insert[db_col] = col_map[h_name] + found = True + break + if not found and db_col == 'report_month': + fixed_values['report_month'] = report_month + + # Determine numeric columns from DB schema + cur.execute(f"SELECT column_name, data_type FROM information_schema.columns WHERE table_name='{table_name}'") + col_types = {r[0]: r[1] for r in cur.fetchall()} + numeric_db_cols = {c for c, t in col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')} + date_db_cols = {c for c, t in col_types.items() if t == 'date'} + + all_cols = ['import_id', 'source_row'] + list(cols_to_insert.keys()) + list(fixed_values.keys()) + ['raw_data'] + col_names = ','.join(all_cols) + + batch = [] + for idx, r in enumerate(data_rows): + source_row = idx + data_start + 1 # 1-indexed Excel row + vals = [import_id, source_row] + for db_col, excel_idx in cols_to_insert.items(): + v = r[excel_idx] if excel_idx < len(r) else None + if db_col in numeric_db_cols: + vals.append(to_num(v) if v is not None else None) + elif db_col in date_db_cols: + vals.append(to_date(v)) + else: + vals.append(to_text(v)) + # Fixed values (not from Excel) + for fv_col in fixed_values: + if fv_col in date_db_cols: + vals.append(to_date(fixed_values[fv_col])) + else: + vals.append(fixed_values[fv_col]) + # raw_data: store entire row as JSON + raw_dict = {} + for hi, hv in enumerate(headers): + if hv and hi < len(r): + rv = r[hi] + if pd.notna(rv): + raw_dict[str(hv)] = str(rv) if not isinstance(rv, (int, float)) else rv + vals.append(json.dumps(raw_dict, ensure_ascii=False)) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.{table_name} ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + + # Update log status + cur.execute( + "UPDATE public.central_kitchen_import_log SET status = 'success', completed_at = now() WHERE import_id = %s", + (import_id,) + ) + conn.commit() + print(f"中央厨房{type_names[ck_type]}导入完成: {len(data_rows)} 行") + +# ============================================================ +# 配送明细导入 +# ============================================================ +def import_distribution(conn, filepath, report_month): + """配送明细导入（支持单文件或目录）""" + import json + print(f"\n=== 配送明细导入 ===") + + files = [] + if os.path.isdir(filepath): + for f in sorted(os.listdir(filepath)): + if f.endswith('.xlsx') and not f.startswith('.'): + files.append(os.path.join(filepath, f)) + else: + files.append(filepath) + + print(f"文件数: {len(files)}") + + cur = conn.cursor() + total_imported = 0 + + for file_idx, fpath in enumerate(files): + fname = os.path.basename(fpath) + print(f"\n[{file_idx+1}/{len(files)}] {fname}") + + sha = file_sha256(fpath) + + cur.execute( + "SELECT 1 FROM public.distribution_import_log WHERE source_file = %s AND file_sha256 = %s", + (fname, sha) + ) + if cur.fetchone(): + print(" 已导入过, 跳过") + continue + + # Read with pandas, header on row 0 + raw_df = pd.read_excel(fpath, header=None, engine='openpyxl') + headers = [str(v).strip() if pd.notna(v) else '' for v in raw_df.iloc[0]] + df = raw_df.iloc[1:] + df = df.dropna(subset=[df.columns[0]]) + data_rows = df.values.tolist() + print(f" 数据行数: {len(data_rows)}") + + if not data_rows: + continue + + # Parse period from filename or use report_month + import re + date_match = re.search(r'(\d{4})年(\d{1,2})月(\d{1,2})日.*?(\d{1,2})日', fname) + if date_match: + year, month, start_day, end_day = date_match.groups() + period_start = f"{year}-{int(month):02d}-{int(start_day):02d}" + period_end = f"{year}-{int(month):02d}-{int(end_day):02d}" + else: + # Use full month + period_start = report_month + # Last day of month + rm_date = pd.Timestamp(report_month) + period_end = (rm_date + pd.offsets.MonthEnd(0)).strftime('%Y-%m-%d') + + cur.execute( + """INSERT INTO public.distribution_import_log + (source_file, source_sheet, file_sha256, workbook_rows, summary_rows_excluded, + imported_rows, report_month, period_start, period_end, status) + VALUES (%s, %s, %s, %s, 0, %s, %s, %s, %s, 'loading') RETURNING import_id""", + (fname, 'Worksheet', sha, len(data_rows), len(data_rows), report_month, period_start, period_end) + ) + import_id = cur.fetchone()[0] + + # Get DB columns + cur.execute("SELECT column_name, data_type FROM information_schema.columns WHERE table_name='distribution_detail_records' ORDER BY ordinal_position") + db_col_types = {r[0]: r[1] for r in cur.fetchall()} + insertable_cols = [c for c in db_col_types if c not in ('record_id', 'import_id', 'source_row', 'raw_data')] + + # Build header map + col_map = {} + for i, h in enumerate(headers): + if h and str(h).strip(): + col_map[str(h).strip()] = i + + # Known mappings for distribution based on actual Excel headers + field_mapping = { + 'business_date': ['日期', '业务日期'], + 'distribution_center_code': ['配送中心编码'], + 'distribution_center_name': ['配送中心'], + 'distribution_center_company': ['配送中心所属公司'], + 'store_company': ['门店所属公司'], + 'store_type': ['门店类型'], + 'store_brand': ['门店品牌'], + 'region_level1': ['一级区域'], + 'terminal_region': ['末级区域'], + 'store_code': ['门店编码'], + 'store_name': ['门店'], + 'store_tax_entity': ['门店财税主体'], + 'original_batch_unit_price': ['原始批次单价'], + 'cost_avg_unit_price': ['成本平均单价'], + 'cost_total_amount': ['成本总金额'], + 'team': ['班组'], + 'cost_excl_tax_amount': ['成本未含税金额'], + 'cost_tax_amount': ['成本税额'], + 'cost_excl_tax_unit_price': ['成本未含税单价'], + 'outbound_tax_amount': ['配出税额'], + 'distribution_sales_tax_rate': ['配送销售税点'], + 'route': ['路线'], + 'profit_amount': ['利润'], + 'profit_excl_tax_amount': ['未含税利润', '利润未含税'], + 'distribution_type': ['配送类型'], + 'logistics_attribute': ['物流属性'], + 'item_id': ['货品ID'], + 'item_name': ['货品名称', '货品'], + 'item_alias': ['货品别名'], + 'specification': ['规格'], + 'item_purpose': ['货品用途', '用途'], + 'unit': ['单位'], + 'normal_quantity': ['正常品数量', '正常数量'], + 'gift_quantity': ['赠品数量'], + 'total_quantity': ['总数量', '合计数量'], + 'outbound_avg_unit_price': ['配出平均单价', '配出均价'], + 'outbound_total_amount': ['配出总金额', '配出金额'], + 'outbound_margin_rate': ['配出毛利率'], + 'outbound_excl_tax_unit_price': ['配出未含税单价'], + 'outbound_excl_tax_amount': ['配出未含税金额'], + 'document_no': ['单号'], + 'requisition_no': ['要货单号', '申请单号'], + 'payment_method': ['支付方式', '结算方式'], + 'value_added_coefficient': ['增值系数'], + 'audit_time': ['审核时间'], + 'inbound_batch': ['入库批次'], + 'inbound_batch_tax_rate': ['入库批次税点', '入库批次税率'], + 'custom_batch_code': ['自定义批次码', '自定义批次编码'], + 'invoice_type': ['发票种类', '发票类型'], + 'production_time': ['生产时间'], + 'shelf_life': ['保质期'], + 'supplier_batch': ['供应商批次'], + 'supplier_code': ['供应商编码'], + 'supplier_category': ['供应商类别', '供应商分类'], + 'supplier_name': ['供应商'], + 'supplier_contact': ['供应商联系人'], + 'supplier_phone': ['供应商联系电话', '供应商电话'], + 'supplier_address': ['供应商地址'], + 'from_warehouse': ['调出仓库', '发货仓库'], + 'to_warehouse': ['调入仓库', '收货仓库'], + 'item_code': ['货品编码'], + 'item_barcode': ['货品条形码', '货品条码'], + 'minor_category': ['所属小类', '小类'], + 'major_category': ['所属大类', '大类'], + 'internal_brand': ['内部品牌'], + 'finance_category': ['财务分类'], + 'created_time': ['制单时间', '创建时间'], + 'creator': ['制单人', '创建人'], + 'auditor': ['审核人'], + 'arrival_time': ['到货时间'], + 'document_remark': ['单据备注'], + 'document_item_remark': ['单据项备注'], + 'reason_remark': ['原因备注'], + 'business_date': ['业务日期'], + 'contact_address': ['联系地址'], + 'contact_phone': ['联系电话'], + 'contact_person': ['联系人'], + } + + cols_to_insert = {} + for db_col, h_names in field_mapping.items(): + if db_col not in insertable_cols: + continue + for h_name in h_names: + if h_name in col_map: + cols_to_insert[db_col] = col_map[h_name] + break + + numeric_db_cols = {c for c, t in db_col_types.items() if t in ('numeric', 'integer', 'bigint', 'double precision', 'real')} + date_db_cols = {c for c, t in db_col_types.items() if t == 'date'} + + all_cols = ['import_id', 'source_row', 'report_month', 'source_file', 'source_sheet'] + list(cols_to_insert.keys()) + col_names = ','.join(all_cols) + + batch = [] + for idx, r in enumerate(data_rows): + source_row = idx + 2 # 1-indexed, data starts at row 2 + vals = [import_id, source_row, report_month, fname, 'Worksheet'] + for db_col, excel_idx in cols_to_insert.items(): + v = r[excel_idx] if excel_idx < len(r) else None + if db_col in numeric_db_cols: + vals.append(to_num(v) if v is not None else None) + elif db_col in date_db_cols: + vals.append(to_date(v)) + else: + vals.append(to_text(v)) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {idx + 1}/{len(data_rows)} 行") + batch = [] + + if batch: + execute_values(cur, f"INSERT INTO public.distribution_detail_records ({col_names}) VALUES %s", batch, page_size=500) + conn.commit() + print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行") + + total_imported += len(data_rows) + + # Update log status + cur.execute( + "UPDATE public.distribution_import_log SET status = 'success', completed_at = now() WHERE import_id = %s", + (import_id,) + ) + conn.commit() + + print(f"\n配送明细导入完成: 共 {total_imported} 行") + +# ============================================================ +# Main +# ============================================================ +def main(): + parser = argparse.ArgumentParser(description='月度数据导入工具') + parser.add_argument('--dataset', required=True, + choices=['inventory', 'dish_sales', 'dish_cost', 'operating_expense', + 'central_kitchen', 'distribution'], + help='数据域') + parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)') + parser.add_argument('--file', required=True, help='文件路径或目录') + parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)') + parser.add_argument('--ck-type', default='finished', + choices=['finished', 'recipe', 'material', 'processing'], + help='中央厨房子类型') + parser.add_argument('--expense-type', default='operating', + choices=['operating', 'management'], + help='费用类型') + + args = parser.parse_args() + + conn = connect(args.db) + conn.autocommit = False + + try: + if args.dataset == 'inventory': + import_inventory_cost(conn, args.file, args.month) + elif args.dataset == 'dish_sales': + import_dish_sales(conn, args.file, args.month) + elif args.dataset == 'dish_cost': + import_dish_cost(conn, args.file, args.month) + elif args.dataset == 'operating_expense': + import_operating_expense(conn, args.file, args.month, args.expense_type) + elif args.dataset == 'central_kitchen': + import_central_kitchen(conn, args.file, args.month, args.ck_type) + elif args.dataset == 'distribution': + import_distribution(conn, args.file, args.month) + + print("\n=== 导入完成 ===") + except Exception as e: + conn.rollback() + print(f"错误: {e}", file=sys.stderr) + raise + finally: + conn.close() + +if __name__ == '__main__': + main() diff --git a/db/import_salary_attendance.py b/db/pipeline/import_salary_attendance.py similarity index 63% rename from db/import_salary_attendance.py rename to db/pipeline/import_salary_attendance.py index d32d4bf..deca7d6 100644 --- a/db/import_salary_attendance.py +++ b/db/pipeline/import_salary_attendance.py @@ -3,12 +3,13 @@ 薪资拆分明细表 + 考勤数据表 导入脚本 遵循现有 import_log + records 模式 """ -import openpyxl import hashlib import psycopg2 import os import sys -from datetime import datetime +import argparse +import pandas as pd +from psycopg2.extras import execute_values DB_CONFIG = { 'host': 'localhost', @@ -18,9 +19,9 @@ DB_CONFIG = { 'password': '', } -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -SALARY_FILE = os.path.join(BASE_DIR, '马兰拉面数据分析', '薪资拆分明细表_脱敏.xlsx') -ATTENDANCE_FILE = os.path.join(BASE_DIR, '马兰拉面数据分析', '考勤数据表-脱敏.xlsx') +BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SALARY_FILE = os.path.join(BASE_DIR, '数据', '4月', '薪资拆分明细表_脱敏.xlsx') +ATTENDANCE_FILE = os.path.join(BASE_DIR, '数据', '4月', '考勤数据表-脱敏.xlsx') def file_sha256(filepath): h = hashlib.sha256() @@ -31,34 +32,42 @@ def file_sha256(filepath): def to_num(val): if val is None or val == '': - return 0 + return None + if isinstance(val, float) and pd.isna(val): + return None try: return float(val) except (ValueError, TypeError): - return 0 + return None def to_text(val): if val is None: return None + if isinstance(val, float) and pd.isna(val): + return None s = str(val).strip() + if s.lower() == 'nan' or s.lower() == 'none': + return None return s if s else None -def import_salary(conn, report_month='2026-04-01'): - print(f'导入薪资拆分明细表: {SALARY_FILE}') - sha = file_sha256(SALARY_FILE) - wb = openpyxl.load_workbook(SALARY_FILE, read_only=True, data_only=True) - ws = wb[wb.sheetnames[0]] - - # Row 1: title, Row 2: merged title, Row 3: headers, Row 4+: data - rows = list(ws.iter_rows(min_row=4, values_only=True)) - data_rows = [r for r in rows if r[0] is not None and r[9] is not None and str(r[9]).strip()] +def import_salary(conn, report_month='2026-04-01', salary_file=None): + filepath = salary_file or SALARY_FILE + print(f'导入薪资拆分明细表: {filepath}') + sha = file_sha256(filepath) + + # Read with pandas (row 3 = headers, row 4+ = data) + raw_df = pd.read_excel(filepath, header=None, engine='openpyxl') + print(f' 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}') + + # Data starts from row 3 (0-indexed), filter rows with employee_code (col 9) + df = raw_df.iloc[3:].copy() + df = df[df[9].notna() & (df[9].astype(str).str.strip() != '') & (df[9].astype(str).str.strip() != 'nan')] + data_rows = df.values.tolist() print(f' 数据行数: {len(data_rows)}') - - report_month = report_month - source_file = os.path.basename(SALARY_FILE) + + source_file = os.path.basename(filepath) cur = conn.cursor() - # Check if already imported cur.execute( 'SELECT import_id FROM public.salary_import_log WHERE report_month = %s AND source_file = %s AND file_sha256 = %s', (report_month, source_file, sha) @@ -66,10 +75,8 @@ def import_salary(conn, report_month='2026-04-01'): existing = cur.fetchone() if existing: print(f' 已导入过, import_id={existing[0]}, 跳过') - wb.close() return - # Insert import log cur.execute( '''INSERT INTO public.salary_import_log (report_month, source_file, file_sha256, workbook_rows, imported_rows) VALUES (%s, %s, %s, %s, %s) RETURNING import_id''', @@ -78,9 +85,8 @@ def import_salary(conn, report_month='2026-04-01'): import_id = cur.fetchone()[0] print(f' import_id={import_id}') - # Batch insert - insert_sql = '''INSERT INTO public.salary_detail_records ( - import_id, source_row, + # Column mapping: use execute_values for batch insert + col_names = '''import_id, source_row, org_level1, org_level2, org_level3, org_level4, org_level5, org_level6, org_level7, org_level8, employee_code, salary_period, position, work_type, employment_type, hire_date, leave_date, salary_standard, base_wage, overtime_subsidy, social_subsidy, position_wage, tenure_wage, @@ -100,13 +106,11 @@ def import_salary(conn, report_month='2026-04-01'): external_subsidy, external_other_deduction, external_gross, pension_deduction, medical_deduction, unemployment_deduction, external_social_total, external_tax, external_net, internal_tax, internal_net, - external_unit, attendance_remark, employment_type_orig, salary_category - ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)''' + external_unit, attendance_remark, employment_type_orig, salary_category''' batch = [] - batch_size = 500 for idx, r in enumerate(data_rows): - source_row = idx + 4 # Excel row number + source_row = idx + 4 vals = [ import_id, source_row, to_text(r[1]), to_text(r[2]), to_text(r[3]), to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]), @@ -130,34 +134,36 @@ def import_salary(conn, report_month='2026-04-01'): to_num(r[90]), to_num(r[91]), to_num(r[92]), to_num(r[93]), to_num(r[94]), to_text(r[95]), to_text(r[96]), to_text(r[97]), to_text(r[98]), ] - batch.append(vals) - if len(batch) >= batch_size: - cur.executemany(insert_sql, batch) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f'INSERT INTO public.salary_detail_records ({col_names}) VALUES %s', batch, page_size=500) conn.commit() print(f' 已导入 {idx + 1}/{len(data_rows)} 行') batch = [] if batch: - cur.executemany(insert_sql, batch) + execute_values(cur, f'INSERT INTO public.salary_detail_records ({col_names}) VALUES %s', batch, page_size=500) conn.commit() print(f' 已导入 {len(data_rows)}/{len(data_rows)} 行') - wb.close() print(f' 薪资明细导入完成: {len(data_rows)} 行') -def import_attendance(conn, report_month='2026-04-01'): - print(f'\n导入考勤数据表: {ATTENDANCE_FILE}') - sha = file_sha256(ATTENDANCE_FILE) - wb = openpyxl.load_workbook(ATTENDANCE_FILE, read_only=True, data_only=True) - ws = wb[wb.sheetnames[0]] - - # Row 1: headers, Row 2+: data - rows = list(ws.iter_rows(min_row=2, values_only=True)) - data_rows = [r for r in rows if r[0] is not None] +def import_attendance(conn, report_month='2026-04-01', attendance_file=None): + filepath = attendance_file or ATTENDANCE_FILE + print(f'\n导入考勤数据表: {filepath}') + sha = file_sha256(filepath) + + # Read with pandas (row 0 = headers, row 1+ = data) + raw_df = pd.read_excel(filepath, header=None, engine='openpyxl') + print(f' 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}') + + # Data starts from row 1 + df = raw_df.iloc[1:].copy() + df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan')] + data_rows = df.values.tolist() print(f' 数据行数: {len(data_rows)}') - - report_month = report_month - source_file = os.path.basename(ATTENDANCE_FILE) + + source_file = os.path.basename(filepath) cur = conn.cursor() cur.execute( @@ -167,7 +173,6 @@ def import_attendance(conn, report_month='2026-04-01'): existing = cur.fetchone() if existing: print(f' 已导入过, import_id={existing[0]}, 跳过') - wb.close() return cur.execute( @@ -178,44 +183,52 @@ def import_attendance(conn, report_month='2026-04-01'): import_id = cur.fetchone()[0] print(f' import_id={import_id}') - # 33 columns: 工号, 岗位, 所在部门, day_01..day_30 + # 34 columns: import_id, source_row, employee_code, position, department, day_01..day_31 cols = ['import_id', 'source_row', 'employee_code', 'position', 'department'] - cols += [f'day_{str(i).zfill(2)}' for i in range(1, 31)] - placeholders = ', '.join(['%s'] * len(cols)) - col_names = ', '.join(cols) - insert_sql = f'INSERT INTO public.attendance_records ({col_names}) VALUES ({placeholders})' + cols += [f'day_{str(i).zfill(2)}' for i in range(1, 32)] + col_names = ','.join(cols) batch = [] - batch_size = 500 for idx, r in enumerate(data_rows): source_row = idx + 2 vals = [import_id, source_row, to_text(r[0]), to_text(r[1]), to_text(r[2])] - # Days 1-30 (columns 3-32) - for d in range(30): + # Days 1-31 (columns 3-33) + for d in range(31): vals.append(to_text(r[3 + d]) if 3 + d < len(r) else None) - batch.append(vals) - if len(batch) >= batch_size: - cur.executemany(insert_sql, batch) + batch.append(tuple(vals)) + if len(batch) >= 500: + execute_values(cur, f'INSERT INTO public.attendance_records ({col_names}) VALUES %s', batch, page_size=500) conn.commit() print(f' 已导入 {idx + 1}/{len(data_rows)} 行') batch = [] if batch: - cur.executemany(insert_sql, batch) + execute_values(cur, f'INSERT INTO public.attendance_records ({col_names}) VALUES %s', batch, page_size=500) conn.commit() print(f' 已导入 {len(data_rows)}/{len(data_rows)} 行') - wb.close() print(f' 考勤数据导入完成: {len(data_rows)} 行') def main(): - report_month = sys.argv[1] if len(sys.argv) > 1 else '2026-04-01' - print(f'导入月份: {report_month}') - conn = psycopg2.connect(**DB_CONFIG) + parser = argparse.ArgumentParser(description='薪资考勤数据导入') + parser.add_argument('--month', default='2026-04-01', help='报告月份') + parser.add_argument('--db', default='bill_query', help='数据库名') + parser.add_argument('--skip-salary', action='store_true', help='跳过薪资导入') + parser.add_argument('--skip-attendance', action='store_true', help='跳过考勤导入') + parser.add_argument('--salary-file', default=None, help='薪资Excel文件路径 (默认使用内置路径)') + parser.add_argument('--attendance-file', default=None, help='考勤Excel文件路径 (默认使用内置路径)') + args = parser.parse_args() + + print(f'导入月份: {args.month}, 数据库: {args.db}') + cfg = DB_CONFIG.copy() + cfg['dbname'] = args.db + conn = psycopg2.connect(**cfg) conn.autocommit = False try: - import_salary(conn, report_month) - import_attendance(conn, report_month) + if not args.skip_salary: + import_salary(conn, args.month, args.salary_file) + if not args.skip_attendance: + import_attendance(conn, args.month, args.attendance_file) print('\n=== 导入完成 ===') except Exception as e: conn.rollback() diff --git a/db/pipeline/import_store_location.py b/db/pipeline/import_store_location.py new file mode 100644 index 0000000..0198813 --- /dev/null +++ b/db/pipeline/import_store_location.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python3 +""" +门店位置与映射表导入脚本 +从 各店信息新（20260508）.xls 导入以下表： + - store_location_source_rows (原始行) + - store_location_master (门店主表，含地理编码) + - sales_store_location_mapping (销售门店→位置映射) + +同时导入静态映射表： + - store_name_mapping (薪资名称↔账单名称映射，8条手工数据) + - inventory_store_mapping (库存成本单位→销售门店映射) + - operating_expense_store_mapping (费用单位→销售门店映射) + +用法: + python3 import_store_location.py --file <各店信息.xls> --db bill_query_test + python3 import_store_location.py --file <各店信息.xls> --db bill_query_test --skip-mappings +""" +import argparse +import math +import os +import re +import sys +from datetime import date, datetime +from difflib import SequenceMatcher + +import pandas as pd +import psycopg2 +from psycopg2.extras import Json, execute_values + + +# ============================================================ +# 静态数据 +# ============================================================ + +# WGS84行政区中心点（低精度兜底） +ADMIN_CENTROIDS = { + "北京市东城区": (39.92855, 116.41637), + "北京市西城区": (39.91231, 116.36679), + "北京市朝阳区": (39.92149, 116.44355), + "北京市海淀区": (39.95933, 116.29845), + "北京市丰台区": (39.85856, 116.28625), + "北京市石景山区": (39.90569, 116.22298), + "北京市昌平区": (40.22077, 116.23128), + "北京市大兴区": (39.72684, 116.34159), + "北京市通州区": (39.90249, 116.65643), + "北京市房山区": (39.74788, 116.14327), + "北京市怀柔区": (40.31600, 116.63170), + "北京市北京经济技术开发区": (39.79500, 116.50600), + "上海市浦东新区": (31.22114, 121.54409), + "上海市青浦区": (31.15074, 121.12417), + "浙江省杭州市余杭区": (30.41875, 120.29940), + "陕西省西安市": (34.34157, 108.93977), +} + +MANUAL_ALIAS = { + "火锅北三环店": "火锅", + "双安总店": "双安", + "温泉店": "温泉西部马华", + "生命园路店": "北大生命园", + "永丰悦界店": "永丰路", + "哈马尔罕总部基地店": "哈马尔罕", +} + +# 薪资名称 → 账单名称 手工映射 +STORE_NAME_MAPPING = [ + ("双安店", "双安总店"), + ("安宁庄快手店", "安宁庄快手"), + ("海淀大街店", "海淀大街"), + ("百子湾店", "百子湾路店"), + ("哈马尔罕大钟寺店", "大钟寺店"), + ("大钟寺店", "大钟寺店"), + ("阿里疆（温泉路店）", "温泉店"), + ("温泉店", "温泉店"), +] + + +# ============================================================ +# 工具函数 +# ============================================================ + +def clean(v): + if v is None: + return None + s = str(v).strip() + return s or None + + +def normalize_name(v): + s = clean(v) or "" + s = re.sub(r"[（(].*?[）)]", "", s) + for token in ("西部马华", "牛肉面", "餐饮店", "餐厅", "总店"): + s = s.replace(token, "") + s = re.sub(r"店$", "", s) + return re.sub(r"\s+", "", s) + + +def parse_date(v): + if isinstance(v, datetime): + return v.date() + if isinstance(v, date): + return v + if isinstance(v, (int, float)): + try: + return (datetime(1899, 12, 30) + datetime.timedelta(days=float(v))).date() + except Exception: + return None + s = clean(v) + if not s or s in {"长期", "未开业"}: + return None + try: + return datetime.fromisoformat(s).date() + except ValueError: + pass + for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d"): + try: + return datetime.strptime(s, fmt).date() + except ValueError: + pass + return None + + +def parse_area(v): + if isinstance(v, (int, float)): + return float(v), "numeric" + s = clean(v) + if not s: + return None, "missing" + nums = [float(x) for x in re.findall(r"\d+(?:\.\d+)?", s)] + if len(nums) >= 2 and ("-" in s or "至" in s or "~" in s): + return sum(nums[:2]) / 2, "range_midpoint" + if nums: + return nums[0], "text_numeric" + return None, "unparsed" + + +def parse_admin(address): + a = clean(address) or "" + if "上海市" in a: + province, city = "上海市", "上海市" + elif "浙江省" in a or "杭州市" in a: + province, city = "浙江省", "杭州市" + elif "陕西省" in a or "西安市" in a: + province, city = "陕西省", "西安市" + else: + province, city = "北京市", "北京市" + + district = None + candidates = ["东城区", "西城区", "朝阳区", "海淀区", "丰台区", "石景山区", + "昌平区", "大兴区", "通州区", "房山区", "怀柔区", + "浦东新区", "青浦区", "余杭区"] + for item in candidates: + if item in a: + district = item + break + if "北京经济技术开发区" in a: + district = "北京经济技术开发区" + return province, city, district + + +def wgs84_to_gcj02(lat, lon): + if lat is None or lon is None: + return None, None + if not (72.004 <= lon <= 137.8347 and 0.8293 <= lat <= 55.8271): + return lat, lon + a, ee = 6378245.0, 0.00669342162296594323 + dlat = _transform_lat(lon - 105.0, lat - 35.0) + dlon = _transform_lon(lon - 105.0, lat - 35.0) + radlat = lat / 180.0 * math.pi + magic = math.sin(radlat) + magic = 1 - ee * magic * magic + sqrtmagic = math.sqrt(magic) + dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * math.pi) + dlon = (dlon * 180.0) / (a / sqrtmagic * math.cos(radlat) * math.pi) + return lat + dlat, lon + dlon + + +def _transform_lat(x, y): + ret = -100.0 + 2.0*x + 3.0*y + 0.2*y*y + 0.1*x*y + 0.2*math.sqrt(abs(x)) + ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0 + ret += (20.0*math.sin(y*math.pi) + 40.0*math.sin(y/3.0*math.pi))*2.0/3.0 + ret += (160.0*math.sin(y/12.0*math.pi) + 320*math.sin(y*math.pi/30.0))*2.0/3.0 + return ret + + +def _transform_lon(x, y): + ret = 300.0 + x + 2.0*y + 0.1*x*x + 0.1*x*y + 0.1*math.sqrt(abs(x)) + ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0 + ret += (20.0*math.sin(x*math.pi) + 40.0*math.sin(x/3.0*math.pi))*2.0/3.0 + ret += (150.0*math.sin(x/12.0*math.pi) + 300.0*math.sin(x/30.0*math.pi))*2.0/3.0 + return ret + + +def fallback_geocode(province, city, district, address): + if not address: + return None, None, None, "pending_no_address", 0.0 + key = f"{city}{district}" if district else city + if province not in {"北京市", "上海市"} and city: + key = f"{province}{city}{district or ''}" + coord = ADMIN_CENTROIDS.get(key) + if coord: + precision = "district_centroid" if district else "city_centroid" + confidence = 0.25 if district else 0.10 + return coord[0], coord[1], key, precision, confidence + city_key = f"{province}{city}" if province != city else city + coord = ADMIN_CENTROIDS.get(city_key) + if coord: + return coord[0], coord[1], city_key, "city_centroid", 0.10 + return None, None, None, "pending_exact_geocode", 0.0 + + +# ============================================================ +# 导入函数 +# ============================================================ + +def import_store_location(conn, filepath): + """导入门店位置主表和源行表""" + source_file = os.path.basename(filepath) + print(f"\n=== 门店位置导入 ===") + print(f"文件: {source_file}") + + cur = conn.cursor() + + # 幂等检查 + cur.execute("SELECT 1 FROM public.store_location_master WHERE source_file = %s LIMIT 1", (source_file,)) + if cur.fetchone(): + print(" 位置数据已导入过，跳过位置导入") + # 仍然需要重建销售门店映射 + _rebuild_sales_mapping(conn) + cur.close() + return + + # 读取Excel + df = pd.read_excel(filepath, header=0, engine='xlrd') + print(f" 总行数: {len(df)}") + + # 清理数据 + raw_rows = [] + numbered = [] + for idx, row in df.iterrows(): + row_no = idx + 2 # 1-indexed from row 2 + seq = row.iloc[0] + seq_int = int(seq) if isinstance(seq, (int, float)) and not pd.isna(seq) else None + raw = { + "row": row_no, "seq": seq_int, + "name": clean(row.iloc[1]) if len(row) > 1 else None, + "company": clean(row.iloc[2]) if len(row) > 2 else None, + "brand": clean(row.iloc[3]) if len(row) > 3 else None, + "address": clean(row.iloc[4]) if len(row) > 4 else None, + "area": clean(row.iloc[5]) if len(row) > 5 else None, + "opened": clean(row.iloc[6]) if len(row) > 6 else None, + "lease": clean(row.iloc[7]) if len(row) > 7 else None, + "license": clean(row.iloc[8]) if len(row) > 8 else None, + "note": clean(row.iloc[9]) if len(row) > 9 else None, + } + raw_rows.append(raw) + if seq_int is not None: + numbered.append(raw) + + print(f" 编号门店数: {len(numbered)}") + + # 清理旧数据 + cur.execute("TRUNCATE public.sales_store_location_mapping RESTART IDENTITY CASCADE") + cur.execute("DELETE FROM public.store_location_master WHERE source_file = %s OR source_file = 'sales_db_placeholder'", (source_file,)) + cur.execute("DELETE FROM public.store_location_source_rows WHERE source_file = %s", (source_file,)) + + # 导入源行 + source_values = [( + source_file, r["row"], r["seq"], r["name"], r["company"], r["brand"], r["address"], + r["area"], r["opened"], r["lease"], r["license"], r["note"] + ) for r in raw_rows] + execute_values(cur, """ + INSERT INTO public.store_location_source_rows + (source_file, source_row, source_store_no, store_short_name, company_name, brand_name, + business_address, area_raw, opened_raw, lease_expiry_raw, license_raw, note) + VALUES %s + """, source_values, page_size=500) + print(f" 源行导入: {len(source_values)} 行") + + # 构建主表数据 + masters = [] + for r in numbered: + area_sqm, area_method = parse_area(r["area"]) + province, city, district = parse_admin(r["address"]) + lat, lon, display, precision, confidence = fallback_geocode(province, city, district, r["address"]) + gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon) + name = r["name"] or f"未命名门店{r['seq']}" + if "停业" in name: + status = "停业" + elif r["opened"] == "未开业": + status = "未开业" + else: + status = "在册" + masters.append({ + **r, "area_sqm": area_sqm, "area_method": area_method, + "opened_date": parse_date(r["opened"]), "lease_date": parse_date(r["lease"]), + "license_date": parse_date(r["license"]), "status": status, + "province": province, "city": city, "district": district, + "lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon, + "display": display, "precision": precision, "confidence": confidence, + }) + + # 占位门店 + placeholders = [ + {"name": "甄选商城店", "address": None, "province": None, "city": None, "district": None, + "status": "线上虚拟门店", "precision": "not_applicable", "display": None, "confidence": 0.0}, + {"name": "西安含光店", "address": "陕西省西安市含光路（具体门牌待补）", + "province": "陕西省", "city": "西安市", "district": None, + "status": "地址待补", "precision": "city_centroid", "display": "陕西省西安市", "confidence": 0.10}, + ] + for i, p in enumerate(placeholders, start=1): + coord = ADMIN_CENTROIDS.get(f"{p['province']}{p['city']}") if p["province"] else None + lat, lon = coord if coord else (None, None) + gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon) + masters.append({ + "row": None, "seq": 9000 + i, "name": p["name"], "company": None, + "brand": "西部马华牛肉面", "address": p["address"], "area": None, + "opened": None, "lease": None, "license": None, "note": "经营数据占位记录", + "area_sqm": None, "area_method": "missing", "opened_date": None, + "lease_date": None, "license_date": None, "status": p["status"], + "province": p["province"], "city": p["city"], "district": p["district"], + "lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon, + "display": p["display"], "precision": p["precision"], "confidence": p["confidence"], + }) + + # 导入主表 + location_ids = {} + for m in masters: + provider = "offline_admin_centroid" if m["lat"] is not None else "none" + geocode_status = "fallback_low_precision" if m["lat"] is not None else m["precision"] + src_file = source_file if m["seq"] < 9000 else "sales_db_placeholder" + cur.execute(""" + INSERT INTO public.store_location_master + (source_file, source_row, source_store_no, store_short_name, company_name, brand_name, + business_address, area_raw, area_sqm, area_parse_method, opened_raw, opened_date, + lease_expiry_raw, lease_expiry_date, license_raw, license_date, operating_status, + province, city, district, geocode_query, geocode_provider, geocode_status, + geocode_precision, geocode_confidence, geocode_display_name, + latitude_wgs84, longitude_wgs84, latitude_gcj02, longitude_gcj02, geocode_raw, geocoded_at) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, + %s,%s,%s,%s,%s, now()) + RETURNING location_id + """, ( + src_file, m["row"], m["seq"], m["name"], m["company"], m["brand"], m["address"], + m["area"], m["area_sqm"], m["area_method"], m["opened"], m["opened_date"], + m["lease"], m["lease_date"], m["license"], m["license_date"], m["status"], + m["province"], m["city"], m["district"], m["address"], + provider, geocode_status, m["precision"], m["confidence"], m["display"], + m["lat"], m["lon"], m["gcj_lat"], m["gcj_lon"], + Json({"notice": "行政区中心点兜底，不是门店精确坐标"}) if m["lat"] is not None else None + )) + location_ids[m["name"]] = cur.fetchone()[0] + + print(f" 主表导入: {len(masters)} 家门店") + + # 从DB读取location_ids用于映射 + cur.execute("SELECT location_id, store_short_name FROM public.store_location_master") + location_ids = {name: lid for lid, name in cur.fetchall()} + + _rebuild_sales_mapping(conn, location_ids, masters) + + conn.commit() + cur.close() + print(f"门店位置导入完成") + + +def _rebuild_sales_mapping(conn, location_ids=None, masters=None): + """重建销售门店→位置映射表""" + cur = conn.cursor() + + # 如果没有传入location_ids，从DB读取 + if location_ids is None: + cur.execute("SELECT location_id, store_short_name FROM public.store_location_master") + location_ids = {name: lid for lid, name in cur.fetchall()} + + if not location_ids: + print(" 跳过销售门店映射: 无位置数据") + cur.close() + return + + # 清理旧映射 + cur.execute("TRUNCATE public.sales_store_location_mapping") + + # 获取销售门店列表 + try: + cur.execute("SELECT store_code, store_name FROM analytics.v_store_scorecard ORDER BY store_name") + sales_stores = cur.fetchall() + except Exception: + sales_stores = [] + print(" 警告: analytics.v_store_scorecard 不存在，跳过销售门店映射") + cur.close() + return + + # 构建候选列表 + if masters: + source_candidates = [(m["name"], normalize_name(m["name"])) for m in masters] + else: + source_candidates = [(name, normalize_name(name)) for name in location_ids.keys()] + + mapped = [] + for store_code, store_name in sales_stores: + target_name = MANUAL_ALIAS.get(store_name) + method = "manual_alias" if target_name else "normalized_name" + score = 1.0 if target_name else 0.0 + if not target_name: + nn = normalize_name(store_name) + exact = [x for x in source_candidates if x[1] == nn and nn] + if exact: + target_name, _ = exact[0] + score = 1.0 + else: + ranked = [] + for candidate, cn in source_candidates: + s = SequenceMatcher(None, nn, cn).ratio() + if nn and cn and (nn in cn or cn in nn): + s = max(s, 0.92) + ranked.append((s, candidate)) + if ranked: + score, target_name = max(ranked) + method = "fuzzy_name" + location_id = location_ids.get(target_name) + if not location_id or score < 0.60: + print(f" 警告: 经营门店无法映射: {store_code} {store_name} -> {target_name} score={score}") + continue + status = "confirmed" if method in {"manual_alias", "normalized_name"} or score >= 0.85 else "reviewed_low_confidence" + note = None if status == "confirmed" else "名称相似匹配，建议业务复核" + mapped.append((store_code, store_name, location_id, method, score, status, note)) + + if mapped: + execute_values(cur, """ + INSERT INTO public.sales_store_location_mapping + (sales_store_code, sales_store_name, location_id, mapping_method, mapping_confidence, mapping_status, review_note) + VALUES %s + """, mapped, page_size=500) + conn.commit() + print(f" 销售门店映射: {len(mapped)} 家") + cur.close() + + +def import_store_name_mapping(conn): + """导入薪资名称↔账单名称映射（静态8条）""" + cur = conn.cursor() + cur.execute("SELECT count(*) FROM public.store_name_mapping") + if cur.fetchone()[0] > 0: + print("store_name_mapping已有数据，跳过") + cur.close() + return + + execute_values( + cur, + "INSERT INTO public.store_name_mapping (salary_name, bill_name) VALUES %s", + STORE_NAME_MAPPING, + page_size=100 + ) + conn.commit() + print(f"store_name_mapping导入完成: {len(STORE_NAME_MAPPING)} 条") + cur.close() + + +def import_inventory_store_mapping(conn): + """从库存成本数据推导库存门店映射""" + cur = conn.cursor() + cur.execute("SELECT count(*) FROM public.inventory_store_mapping") + if cur.fetchone()[0] > 0: + print("inventory_store_mapping已有数据，跳过") + cur.close() + return + + # 从库存成本记录中提取唯一的成本单位编码和名称 + cur.execute(""" + SELECT DISTINCT cost_unit_code, cost_unit_name + FROM public.inventory_cost_records + WHERE report_month = '2026-04-01' + AND cost_unit_code IS NOT NULL + ORDER BY cost_unit_code + """) + cost_units = cur.fetchall() + + mappings = [] + for code, name in cost_units: + # 成本单位编码与销售门店编码相同的直接映射 + mappings.append(( + code, name, code, name, + '经营门店', True, '门店编码精确匹配', 100.00, None + )) + + if mappings: + execute_values(cur, """ + INSERT INTO public.inventory_store_mapping + (cost_unit_code, cost_unit_name, sales_store_code, sales_store_name, + unit_type, include_in_operating_cost, mapping_method, mapping_confidence, review_note) + VALUES %s + """, mappings, page_size=500) + conn.commit() + print(f"inventory_store_mapping导入完成: {len(mappings)} 条") + else: + print("inventory_store_mapping: 无库存成本数据可推导") + + cur.close() + + +def import_operating_expense_store_mapping(conn): + """从营业费用数据推导费用门店映射""" + cur = conn.cursor() + cur.execute("SELECT count(*) FROM public.operating_expense_store_mapping") + if cur.fetchone()[0] > 0: + print("operating_expense_store_mapping已有数据，跳过") + cur.close() + return + + # 从营业费用记录中提取唯一的成本单位名称 + cur.execute(""" + SELECT DISTINCT cost_unit_source_name + FROM public.operating_expense_records + WHERE report_month = '2026-04-01' + AND cost_unit_source_name IS NOT NULL + ORDER BY cost_unit_source_name + """) + expense_units = cur.fetchall() + + # 尝试从销售数据获取门店列表做匹配 + try: + cur.execute("SELECT store_code, store_name FROM analytics.dim_store ORDER BY store_code") + sales_stores = cur.fetchall() + except Exception: + sales_stores = [] + + mappings = [] + for (source_name,) in expense_units: + # 标准化名称：去掉"金额"后缀 + normalized = re.sub(r"金额$", "", source_name).strip() + # 尝试精确匹配 + matched_code = None + matched_name = None + method = "unmapped" + confidence = 0.0 + + for sc, sn in sales_stores: + if normalized == sn or normalized in sn or sn in normalized: + matched_code = sc + matched_name = sn + method = "标准名称精确匹配" + confidence = 100.0 + break + + if not matched_code: + # 尝试编码前缀匹配 + m = re.match(r"^(\d+)\s+", normalized) + if m: + prefix = m.group(1) + for sc, sn in sales_stores: + if sc == prefix: + matched_code = sc + matched_name = sn + method = "编码前缀匹配" + confidence = 90.0 + break + + mappings.append(( + source_name, normalized, + matched_code, matched_name, + '经营门店', matched_code is not None, + method if matched_code else "待人工确认", + confidence, + None if matched_code else "需人工确认匹配关系" + )) + + if mappings: + execute_values(cur, """ + INSERT INTO public.operating_expense_store_mapping + (cost_unit_source_name, normalized_cost_unit_name, sales_store_code, sales_store_name, + unit_type, include_in_operating_analysis, mapping_method, mapping_confidence, review_note) + VALUES %s + """, mappings, page_size=500) + conn.commit() + print(f"operating_expense_store_mapping导入完成: {len(mappings)} 条") + else: + print("operating_expense_store_mapping: 无费用数据可推导") + + cur.close() + + +# ============================================================ +# Main +# ============================================================ + +def main(): + parser = argparse.ArgumentParser(description='门店位置与映射表导入') + parser.add_argument('--file', required=True, help='各店信息Excel文件路径') + parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)') + parser.add_argument('--skip-location', action='store_true', help='跳过门店位置导入') + parser.add_argument('--skip-mappings', action='store_true', help='跳过映射表导入') + + args = parser.parse_args() + + conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak') + conn.autocommit = False + + try: + if not args.skip_location: + import_store_location(conn, args.file) + + if not args.skip_mappings: + print("\n=== 映射表导入 ===") + import_store_name_mapping(conn) + import_inventory_store_mapping(conn) + import_operating_expense_store_mapping(conn) + + print("\n=== 导入完成 ===") + + except Exception as e: + conn.rollback() + print(f"错误: {e}", file=sys.stderr) + raise + finally: + conn.close() + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/refresh_materialized_views.py b/db/pipeline/refresh_materialized_views.py new file mode 100644 index 0000000..89727e6 --- /dev/null +++ b/db/pipeline/refresh_materialized_views.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" +物化视图按依赖顺序刷新 +用法: python3 refresh_materialized_views.py --db bill_query_test +""" +import argparse +import psycopg2 +import time +import sys + +from config import DB_CONFIG, MATVIEW_LAYERS + + +def refresh_all(dbname, verbose=True): + conn = psycopg2.connect( + host=DB_CONFIG['host'], + port=DB_CONFIG['port'], + dbname=dbname, + user=DB_CONFIG['user'], + password=DB_CONFIG['password'] or None, + ) + conn.autocommit = True + cur = conn.cursor() + + total = sum(len(layer) for layer in MATVIEW_LAYERS) + done = 0 + failed = [] + + for layer_idx, layer in enumerate(MATVIEW_LAYERS): + if verbose: + print(f"\n--- L{layer_idx}: {len(layer)} 个物化视图 ---") + for mv in layer: + done += 1 + t0 = time.time() + try: + cur.execute(f'REFRESH MATERIALIZED VIEW analytics."{mv}"') + elapsed = time.time() - t0 + if verbose: + print(f" [{done}/{total}] OK {mv} ({elapsed:.1f}s)") + except Exception as e: + elapsed = time.time() - t0 + err_msg = str(e).strip().split('\n')[0] + if verbose: + print(f" [{done}/{total}] FAIL {mv} ({elapsed:.1f}s): {err_msg}") + failed.append((mv, err_msg)) + conn.rollback() + + cur.close() + conn.close() + + if failed: + print(f"\n=== 完成: {total - len(failed)}/{total} 成功, {len(failed)} 失败 ===") + for mv, err in failed: + print(f" FAIL {mv}: {err}") + # 重试失败项（可能因依赖顺序问题，第二轮能成功） + if len(failed) > 0: + print(f"\n--- 重试 {len(failed)} 个失败项 ---") + conn = psycopg2.connect( + host=DB_CONFIG['host'], + port=DB_CONFIG['port'], + dbname=dbname, + user=DB_CONFIG['user'], + password=DB_CONFIG['password'] or None, + ) + conn.autocommit = True + cur = conn.cursor() + retry_failed = [] + for mv, _ in failed: + t0 = time.time() + try: + cur.execute(f'REFRESH MATERIALIZED VIEW analytics."{mv}"') + elapsed = time.time() - t0 + print(f" OK {mv} ({elapsed:.1f}s)") + except Exception as e: + err_msg = str(e).strip().split('\n')[0] + print(f" FAIL {mv}: {err_msg}") + retry_failed.append(mv) + conn.rollback() + cur.close() + conn.close() + if retry_failed: + print(f"\n最终失败: {retry_failed}") + return 1 + else: + print(f"\n=== 全部 {total} 个物化视图刷新成功 ===") + return 0 + else: + print(f"\n=== 全部 {total} 个物化视图刷新成功 ===") + return 0 + + +def main(): + parser = argparse.ArgumentParser(description='物化视图按依赖顺序刷新') + parser.add_argument('--db', default='bill_query_test', help='目标数据库') + args = parser.parse_args() + + print(f"\n=== 物化视图刷新 ===") + print(f"数据库: {args.db}") + rc = refresh_all(args.db) + sys.exit(rc) + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/run_all.py b/db/pipeline/run_all.py new file mode 100644 index 0000000..5dbe142 --- /dev/null +++ b/db/pipeline/run_all.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +""" +一键全量导入 + 物化视图刷新 +用法: python3 run_all.py --month 2026-04-01 --db bill_query_test + python3 run_all.py --month 2026-04-01 --db bill_query_test --skip-import # 仅刷新物化 + python3 run_all.py --month 2026-04-01 --db bill_query_test --skip-views # 仅导入 + python3 run_all.py --month 2026-04-01 --db bill_query_test --only bill_records # 仅导入某类 +""" +import argparse +import os +import subprocess +import sys +import time + +from config import DB_CONFIG, get_data_sources, PIPELINE_DIR + + +def run_script(name, cmd, cwd=None): + """运行子进程并实时输出""" + print(f"\n{'='*60}") + print(f" 执行: {name}") + print(f" 命令: {' '.join(cmd)}") + print(f"{'='*60}") + t0 = time.time() + result = subprocess.run(cmd, cwd=cwd or PIPELINE_DIR) + elapsed = time.time() - t0 + status = "成功" if result.returncode == 0 else f"失败(rc={result.returncode})" + print(f" → {name}: {status} ({elapsed:.1f}s)") + return result.returncode == 0 + + +def main(): + parser = argparse.ArgumentParser(description='全量数据导入与物化刷新流水线') + parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)') + parser.add_argument('--db', default='bill_query_test', help='目标数据库') + parser.add_argument('--skip-import', action='store_true', help='跳过数据导入，仅刷新物化视图') + parser.add_argument('--skip-views', action='store_true', help='跳过物化视图刷新') + parser.add_argument('--only', default=None, + help='仅导入指定数据集: bill_records, monthly, salary, store_location, derived') + parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)') + args = parser.parse_args() + + sources = get_data_sources(args.month) + if not sources: + print(f"错误: 无法找到月份 {args.month} 的数据源") + sys.exit(1) + + print(f"\n{'#'*60}") + print(f" 数据导入与物化刷新流水线") + print(f" 月份: {args.month}") + print(f" 数据库: {args.db}") + print(f" 数据源根: {sources.get('bill_records', 'N/A')}") + print(f"{'#'*60}") + + py = sys.executable + results = [] + + # ============================================================ + # Step 1: 数据导入 + # ============================================================ + if not args.skip_import: + target = args.only + + # 1.1 薪资+考勤 + if target in (None, 'salary'): + cmd = [py, 'import_salary_attendance.py', '--month', args.month, '--db', args.db] + sal_file = sources.get('salary') + att_file = sources.get('attendance') + if sal_file and os.path.isfile(sal_file): + cmd.extend(['--salary-file', sal_file]) + if att_file and os.path.isfile(att_file): + cmd.extend(['--attendance-file', att_file]) + ok = run_script("薪资+考勤导入", cmd) + results.append(('薪资+考勤', ok)) + + # 1.2 账单查询原始数据（bill_records + bill_columns） + if target in (None, 'bill_records'): + bill_dir = sources.get('bill_records') + if bill_dir and os.path.exists(bill_dir): + ok = run_script("账单查询导入(bill_records)", [ + py, 'import_bill_records.py', + '--month', args.month, + '--file', bill_dir, + '--db', args.db, + ]) + results.append(('bill_records', ok)) + else: + print(f"\n 跳过 bill_records: 路径不存在 {bill_dir}") + results.append(('bill_records', False)) + + # 1.3 月度业务数据（库存/销售/成本/费用/中央厨房/配送） + if target in (None, 'monthly'): + monthly_datasets = [ + ('inventory', 'inventory', None), + ('dish_sales', 'dish_sales', None), + ('dish_cost', 'dish_cost', None), + ('operating_expense', 'operating_expense', None), + ('central_kitchen_finished', 'central_kitchen', 'finished'), + ('central_kitchen_recipe', 'central_kitchen', 'recipe'), + ('central_kitchen_material', 'central_kitchen', 'material'), + ('central_kitchen_processing', 'central_kitchen', 'processing'), + ('distribution', 'distribution', None), + ('distribution_2', 'distribution', None), + ] + + for src_key, dataset, ck_type in monthly_datasets: + filepath = sources.get(src_key) + if not filepath or not os.path.exists(filepath): + print(f"\n 跳过 {dataset}/{ck_type or ''}: 文件不存在 {filepath}") + results.append((f'{dataset}/{ck_type or src_key}', False)) + continue + + cmd = [ + py, 'import_monthly_data.py', + '--dataset', dataset, + '--month', args.month, + '--file', filepath, + '--db', args.db, + ] + if ck_type: + cmd.extend(['--ck-type', ck_type]) + + label = f"月度数据-{dataset}" + (f"-{ck_type}" if ck_type else f"({os.path.basename(filepath)})") + ok = run_script(label, cmd) + results.append((label, ok)) + + # 1.4 门店位置与映射表 + if target in (None, 'store_location'): + store_file = sources.get('store_location') + if store_file and os.path.exists(store_file): + ok = run_script("门店位置与映射表导入", [ + py, 'import_store_location.py', + '--file', store_file, + '--db', args.db, + ]) + results.append(('门店位置', ok)) + else: + print(f"\n 跳过门店位置: 文件不存在 {store_file}") + results.append(('门店位置', False)) + + # 1.5 派生分析数据 + if target in (None, 'derived'): + diag_date = args.diagnosis_date or '' + cmd = [ + py, 'import_derived_data.py', + '--month', args.month, + '--db', args.db, + ] + if diag_date: + cmd.extend(['--diagnosis-date', diag_date]) + ok = run_script("派生分析数据生成", cmd) + results.append(('派生数据', ok)) + + # ============================================================ + # Step 2: 物化视图刷新 + # ============================================================ + if not args.skip_views: + ok = run_script("物化视图刷新", [ + py, 'refresh_materialized_views.py', + '--db', args.db, + ], cwd=PIPELINE_DIR) + results.append(('物化视图', ok)) + + # ============================================================ + # Step 3: 验证 + # ============================================================ + ok = run_script("数据一致性验证", [ + py, 'verify.py', + '--db', args.db, + ], cwd=PIPELINE_DIR) + results.append(('验证', ok)) + + # ============================================================ + # 汇总 + # ============================================================ + print(f"\n{'#'*60}") + print(f" 流水线执行汇总") + print(f"{'#'*60}") + all_ok = True + for name, ok in results: + status = "✓ 成功" if ok else "✗ 失败" + print(f" {status} {name}") + if not ok: + all_ok = False + + if all_ok: + print(f"\n 全部完成!") + sys.exit(0) + else: + print(f"\n 部分失败，请检查上方日志") + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/verify.py b/db/pipeline/verify.py new file mode 100644 index 0000000..b47825e --- /dev/null +++ b/db/pipeline/verify.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +测试库与正式库行数对比验证 +用法: python3 verify.py --db bill_query_test [--prod bill_query] +""" +import argparse +import psycopg2 +import sys + +from config import DB_CONFIG + + +def get_table_counts(dbname): + conn = psycopg2.connect( + host=DB_CONFIG['host'], + port=DB_CONFIG['port'], + dbname=dbname, + user=DB_CONFIG['user'], + password=DB_CONFIG['password'] or None, + ) + cur = conn.cursor() + cur.execute(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_type = 'BASE TABLE' + ORDER BY table_name + """) + tables = [r[0] for r in cur.fetchall()] + + counts = {} + for tbl in tables: + cur.execute(f"SELECT count(*) FROM public.{tbl}") + counts[tbl] = cur.fetchone()[0] + + # 物化视图行数 + cur.execute(""" + SELECT matviewname + FROM pg_matviews + WHERE schemaname = 'analytics' + ORDER BY matviewname + """) + mvs = [r[0] for r in cur.fetchall()] + mv_counts = {} + for mv in mvs: + try: + cur.execute(f'SELECT count(*) FROM analytics."{mv}"') + mv_counts[mv] = cur.fetchone()[0] + except Exception: + mv_counts[mv] = -1 + conn.rollback() + + cur.close() + conn.close() + return counts, mv_counts + + +def main(): + parser = argparse.ArgumentParser(description='测试库与正式库数据一致性验证') + parser.add_argument('--db', default='bill_query_test', help='测试库名') + parser.add_argument('--prod', default='bill_query', help='正式库名') + args = parser.parse_args() + + print(f"\n=== 数据一致性验证 ===") + print(f"正式库: {args.prod}") + print(f"测试库: {args.db}") + + prod_counts, prod_mv = get_table_counts(args.prod) + test_counts, test_mv = get_table_counts(args.db) + + # 基础表对比 + print(f"\n--- 基础表 ({len(prod_counts)} 张) ---") + ok = 0 + diff = 0 + for tbl in sorted(prod_counts.keys()): + p = prod_counts[tbl] + t = test_counts.get(tbl, 'MISSING') + flag = 'OK' if t == p else 'DIFF' + if flag == 'OK': + ok += 1 + else: + diff += 1 + if flag != 'OK' or True: # 显示全部 + print(f" {flag:4s} {tbl:45s} prod={p:<12s} test={t}") + + print(f"\n 基础表: {ok} OK, {diff} DIFF") + + # 物化视图对比 + print(f"\n--- 物化视图 ({len(prod_mv)} 个) ---") + mv_ok = 0 + mv_diff = 0 + for mv in sorted(prod_mv.keys()): + p = prod_mv[mv] + t = test_mv.get(mv, 'MISSING') + flag = 'OK' if t == p else 'DIFF' + if flag == 'OK': + mv_ok += 1 + else: + mv_diff += 1 + print(f" {flag:4s} {mv:45s} prod={p:<12s} test={t}") + + print(f"\n 物化视图: {mv_ok} OK, {mv_diff} DIFF") + print(f"\n=== 总计: {ok + mv_ok} OK, {diff + mv_diff} DIFF ===") + + if diff + mv_diff > 0: + sys.exit(1) + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/db/pipeline/verify_import.py b/db/pipeline/verify_import.py new file mode 100644 index 0000000..7fee27f --- /dev/null +++ b/db/pipeline/verify_import.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +""" +对比验证脚本：比较测试库与正式库的4月数据 +用法: python3 verify_import.py +""" +import psycopg2 + +def connect(dbname): + return psycopg2.connect(host='localhost', port=5432, dbname=dbname, user='freedak') + +def compare_table(test_cur, prod_cur, table_name, where_clause='1=1', key_cols=None, sum_cols=None): + """对比两个库中同一表的数据""" + print(f"\n--- {table_name} ---") + + # Row count + test_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}") + test_count = test_cur.fetchone()[0] + prod_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}") + prod_count = prod_cur.fetchone()[0] + + match = "✅" if test_count == prod_count else "❌" + print(f" 行数: 测试库={test_count}, 正式库={prod_count} {match}") + + # Sum comparison + if sum_cols: + for col in sum_cols: + test_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}") + test_sum = test_cur.fetchone()[0] + prod_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}") + prod_sum = prod_cur.fetchone()[0] + match = "✅" if abs(float(test_sum or 0) - float(prod_sum or 0)) < 0.01 else "❌" + print(f" {col}合计: 测试库={test_sum}, 正式库={prod_sum} {match}") + + return test_count == prod_count + +def main(): + test_conn = connect('bill_query_test') + prod_conn = connect('bill_query') + test_cur = test_conn.cursor() + prod_cur = prod_conn.cursor() + + print("=" * 60) + print(" 数据对比验证: 测试库 vs 正式库 (4月数据)") + print("=" * 60) + + results = [] + + # 1. 库存成本 + results.append(compare_table( + test_cur, prod_cur, 'inventory_cost_records', + where_clause="report_month = '2026-04-01'", + sum_cols=['consumption_amount', 'opening_amount', 'ending_amount', 'purchase_amount'] + )) + + # 2. 菜品销售明细 (测试库只导入了1个文件，按文件对比) + test_cur.execute("SELECT DISTINCT source_file FROM public.dish_sales_details") + test_files = [r[0] for r in test_cur.fetchall()] + for sf in test_files: + results.append(compare_table( + test_cur, prod_cur, 'dish_sales_details', + where_clause=f"source_file = '{sf}'", + sum_cols=['gross_amount', 'received_amount'] + )) + + # 3. 菜品成本 + results.append(compare_table( + test_cur, prod_cur, 'dish_cost_analysis_summary', + where_clause="1=1", + sum_cols=['theoretical_cost', 'actual_cost', 'sales_amount'] + )) + # 3b. 菜品成本原料明细 + results.append(compare_table( + test_cur, prod_cur, 'dish_cost_analysis_material_detail', + where_clause="1=1", + sum_cols=['theoretical_amount', 'actual_amount'] + )) + + # 4. 营业费用 + results.append(compare_table( + test_cur, prod_cur, 'operating_expense_records', + where_clause="report_month = '2026-04-01'", + sum_cols=['amount'] + )) + + # 5. 中央厨房 + ck_tables = { + 'central_kitchen_finished_receipt': ('receipt_date', ['amount', 'quantity']), + 'central_kitchen_recipe_consumption': ('business_date', ['theoretical_quantity', 'theoretical_amount', 'issue_quantity', 'issue_amount']), + 'central_kitchen_material_daily': ('business_date', ['amount']), + 'central_kitchen_processing_cost': ('report_month', ['theoretical_cost', 'actual_cost']), + } + for tbl, (date_col, sums) in ck_tables.items(): + # Check which sum cols actually exist + test_cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{tbl}'") + actual_cols = [r[0] for r in test_cur.fetchall()] + valid_sums = [c for c in sums if c in actual_cols] + if date_col == 'report_month': + where = f"{date_col} = '2026-04-01'" + else: + where = f"{date_col} >= '2026-04-01' AND {date_col} < '2026-05-01'" + results.append(compare_table( + test_cur, prod_cur, tbl, + where_clause=where, + sum_cols=valid_sums if valid_sums else None + )) + + # 6. 配送明细 + results.append(compare_table( + test_cur, prod_cur, 'distribution_detail_records', + where_clause="report_month = '2026-04-01'", + sum_cols=['cost_total_amount', 'outbound_total_amount', 'total_quantity'] + )) + + # 7. 薪资 + results.append(compare_table( + test_cur, prod_cur, 'salary_detail_records', + where_clause="1=1", + sum_cols=['net_pay', 'gross_pay'] + )) + + # 8. 考勤 + results.append(compare_table( + test_cur, prod_cur, 'attendance_records', + where_clause="1=1", + sum_cols=None + )) + + # Summary + print("\n" + "=" * 60) + passed = sum(results) + total = len(results) + print(f" 通过: {passed}/{total}") + if passed == total: + print(" ✅ 所有对比通过") + else: + print(f" ❌ {total - passed} 项对比失败") + print("=" * 60) + + test_cur.close() + prod_cur.close() + test_conn.close() + prod_conn.close() + +if __name__ == '__main__': + main()