feat: 新增db/pipeline数据导入管线，适配5月数据格式

- 整合所有导入脚本到db/pipeline/目录
- config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构
- import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details)
- import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径
- import_may_data.sh: 5月一键导入脚本(12步全流程)
- run_all.py: 一键全量导入+物化视图刷新编排
- refresh_materialized_views.py: 按依赖顺序刷新38个物化视图
- verify.py/verify_import.py: 数据一致性验证
- README.md: 管线文档

.gitignore

@@ -18,3 +18,4 @@ backups/
 .playwright-cli/
 .tmp/
 数据/
+__pycache__/

db/pipeline/README.md

@@ -0,0 +1,67 @@
+# 数据导入与物化刷新流水线
+
+## 目录结构
+
+```
+db/pipeline/
+├── README.md                      ← 本文件
+├── config.py                      ← 数据源路径与数据库配置
+├── run_all.py                     ← 一键全量导入+物化刷新
+├── refresh_materialized_views.py  ← 物化视图按依赖顺序刷新
+├── verify.py                      ← 测试库与正式库行数对比验证
+├── verify_import.py               ← 详细数据对比验证
+├── import_bill_records.py         ← 账单查询原始数据导入（4月格式，196列宽表）
+├── import_bill_records_may.py     ← 5月专用账单导入（51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details）
+├── import_monthly_data.py         ← 月度业务数据导入（库存/销售/成本/费用/中央厨房/配送）
+├── import_salary_attendance.py    ← 薪资考勤数据导入
+├── import_store_location.py       ← 门店位置与映射表导入
+└── import_derived_data.py         ← 派生分析数据生成
+```
+
+## 导入脚本
+
+| 脚本 | 数据表 | 数据源 |
+|------|--------|--------|
+| `import_bill_records.py` | bill_records, bill_columns | 账单查询 Excel 目录（17个文件） |
+| `import_monthly_data.py` | inventory_cost_records, dish_sales_details, dish_cost_analysis_*, operating_expense_records, central_kitchen_*, distribution_detail_records | 各业务 Excel 文件 |
+| `import_salary_attendance.py` | salary_detail_records, attendance_records | 薪资/考勤 Excel |
+| `import_store_location.py` | store_location_master, store_location_source_rows, sales_store_location_mapping, store_name_mapping, inventory_store_mapping, operating_expense_store_mapping | 各店信息 Excel |
+| `import_derived_data.py` | dish_diagnosis_snapshot, central_kitchen_manufacturing_cost_pool | 从已有表派生 |
+
+## 物化视图刷新顺序（按依赖分层）
+
+| 层级 | 物化视图 | 依赖 |
+|------|----------|------|
+| L0 | bill_fact | 基础表 |
+| L1 | mv_daily_revenue, mv_dish_basket_monthly, mv_dish_pair_summary_monthly, mv_dish_sku_summary_monthly, mv_dish_store_summary_monthly, mv_district_site_benchmark_monthly, mv_inventory_cost_classified_monthly, mv_site_segment_benchmark_monthly, mv_store_*_monthly (12个), mv_store_scorecard | bill_fact |
+| L1 | dish_sales_april | bill_fact |
+| L2 | dish_basket_april, dish_category_summary_april, dish_member_sku_april, dish_sku_summary_april, dish_store_sku_april | dish_sales_april |
+| L2 | mv_dish_sku_abc_monthly | mv_dish_sku_summary_monthly |
+| L2 | v_store_benchmark, v_store_category_mix, v_store_platform_economics | bill_fact |
+| L3 | dish_store_summary_april | dish_basket_april |
+| L3 | v_store_action_list | v_store_benchmark, v_store_category_mix, v_store_platform_economics |
+| L3 | mv_store_action_priority_deep_april, mv_store_theoretical_actual_cost_april | bill_fact / dish_store_summary_april |
+| L4 | v_store_execution_priority | v_store_action_list |
+
+## 使用方法
+
+### 一键全量导入
+```bash
+python3 db/pipeline/run_all.py --month 2026-04-01 --db bill_query_test
+```
+
+### 仅刷新物化视图
+```bash
+python3 db/pipeline/refresh_materialized_views.py --db bill_query_test
+```
+
+### 验证数据一致性
+```bash
+python3 db/pipeline/verify.py --db bill_query_test
+```
+
+### 单独导入某类数据
+参见各脚本 `--help`，例如：
+```bash
+python3 db/pipeline/import_monthly_data.py --dataset inventory --month 2026-04-01 --file <path> --db bill_query_test
+```

db/pipeline/config.py

@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""
+数据导入流水线配置
+统一管理数据源路径、数据库连接参数、导入脚本路径
+"""
+import os
+import glob
+import sys
+
+# ============================================================
+# 路径常量
+# ============================================================
+PIPELINE_DIR = os.path.dirname(os.path.abspath(__file__))
+DB_DIR = os.path.dirname(PIPELINE_DIR)
+BASE_DIR = os.path.dirname(DB_DIR)
+
+# 数据源根目录（西部马华数据分析 下有完整的业务数据）
+DATA_ROOT = os.path.join(os.path.dirname(BASE_DIR), '西部马华数据分析')
+# SBrainCO3 数据目录（部分文件只在这里有）
+DATA_SBRAIN = os.path.join(BASE_DIR, '数据')
+
+# ============================================================
+# 数据库配置
+# ============================================================
+DB_CONFIG = {
+    'host': 'localhost',
+    'port': 5432,
+    'user': 'freedak',
+    'password': '',
+}
+
+# ============================================================
+# 月份配置
+# ============================================================
+MONTH_CONFIG = {
+    '2026-04-01': {
+        'month_dir': '4月',
+        'date_range': '2026-04-01--2026-04-30',
+        'dist_half1': '2026年4月1-15日',
+        'dist_half2': '2026年4月16-30日',
+        'inventory_file': '4.13盘点倒挤成本报表2026年4月明细.xlsx',
+        'dish_cost_file': '菜品成本分析报表.xlsx',
+        'expense_file': '2026年4月营业费用分析.xls',
+        'ck_subdir': '中央厨房4月',
+        'salary_file': '薪资拆分明细表_脱敏.xlsx',
+        'attendance_file': '考勤数据表-脱敏.xlsx',
+        'store_info_file': '各店信息新（20260508）.xls',
+        'bill_records_subdir': '账单查询',
+        'dish_sales_subdir': '账单查询-菜品销售明细表',
+        'data_subdirs': {},  # 4月文件平铺在月份目录下
+    },
+    '2026-05-01': {
+        'month_dir': '5月',
+        'date_range': '2026-05-01--2026-05-31',
+        'dist_half1': '2026年5月1-15日',
+        'dist_half2': '2026年5月16-31日',
+        'inventory_file': '4.13盘点倒挤成本报表2026年5月.xlsx',
+        'dish_cost_file': '2026年5月菜品成本分析报表.xlsx',
+        'expense_file': '2026年5月营业费用分析.xls',
+        'ck_subdir': '中央厨房',
+        'salary_file': '2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx',
+        'attendance_file': '2026年5月考勤数据表.xlsx',
+        'store_info_file': None,  # 5月无新各店信息文件，复用4月
+        'bill_records_subdir': None,  # 5月无独立账单查询目录
+        'dish_sales_subdir': '账单查询-菜品销售明细表',
+        'data_subdirs': {  # 5月文件在子目录中
+            'inventory': '供应链数据',
+            'dish_cost': '营业数据',
+            'operating_expense': '',
+            'distribution': '供应链数据',
+            'salary': '营业数据',
+            'attendance': '营业数据',
+            'central_kitchen': '中央厨房',
+            'dish_sales': '营业数据',
+        },
+    },
+}
+
+
+def _find_file(base_dirs, pattern):
+    """在多个候选目录中查找文件"""
+    for d in base_dirs:
+        if not d:
+            continue
+        path = os.path.join(d, pattern)
+        if os.path.isfile(path):
+            return path
+        matches = glob.glob(os.path.join(d, pattern))
+        if matches:
+            return matches[0]
+    return None
+
+
+def get_data_sources(month):
+    """
+    根据月份返回数据源路径字典
+    month: 'YYYY-MM-01' 格式
+    """
+    cfg = MONTH_CONFIG.get(month)
+    if not cfg:
+        print(f"警告: 未知月份 {month}，请在 config.py MONTH_CONFIG 中添加配置")
+        return {}
+
+    month_dir = cfg['month_dir']
+    data_dir = os.path.join(DATA_ROOT, month_dir)
+    sbrain_data = os.path.join(DATA_SBRAIN, month_dir)
+    subdirs = cfg.get('data_subdirs', {})
+
+    sources = {}
+
+    def resolve(subdir_key, filename):
+        """在 data_dir 和 sbrain_data 中按子目录结构查找文件"""
+        sub = subdirs.get(subdir_key, '')
+        candidates = []
+        if sub:
+            candidates.append(os.path.join(sbrain_data, sub, filename))
+            candidates.append(os.path.join(data_dir, sub, filename))
+        candidates.append(os.path.join(sbrain_data, filename))
+        candidates.append(os.path.join(data_dir, filename))
+        for c in candidates:
+            if os.path.isfile(c):
+                return c
+        return candidates[0]
+
+    def resolve_dir(subdir_key, dirname):
+        """在 data_dir 和 sbrain_data 中按子目录结构查找目录"""
+        sub = subdirs.get(subdir_key, '')
+        candidates = []
+        if sub:
+            candidates.append(os.path.join(sbrain_data, sub, dirname))
+            candidates.append(os.path.join(data_dir, sub, dirname))
+        candidates.append(os.path.join(sbrain_data, dirname))
+        candidates.append(os.path.join(data_dir, dirname))
+        for c in candidates:
+            if os.path.isdir(c):
+                return c
+        return candidates[0]
+
+    # 1. 账单查询（bill_records）- 目录
+    bill_subdir = cfg.get('bill_records_subdir')
+    if bill_subdir:
+        sources['bill_records'] = resolve_dir('bill_records', bill_subdir)
+    else:
+        sources['bill_records'] = None
+
+    # 2. 菜品销售明细 - 目录
+    sources['dish_sales'] = resolve_dir('dish_sales', cfg['dish_sales_subdir'])
+
+    # 3. 库存成本
+    sources['inventory'] = resolve('inventory', cfg['inventory_file'])
+
+    # 4. 菜品成本BOM
+    sources['dish_cost'] = resolve('dish_cost', cfg['dish_cost_file'])
+
+    # 5. 营业费用
+    sources['operating_expense'] = resolve('operating_expense', cfg['expense_file'])
+
+    # 6. 中央厨房（4个子文件）
+    ck_subdir = cfg['ck_subdir']
+    ck_dir_sbrain = os.path.join(sbrain_data, ck_subdir)
+    ck_dir_data = os.path.join(data_dir, ck_subdir)
+    ck_dir = ck_dir_sbrain if os.path.isdir(ck_dir_sbrain) else ck_dir_data
+    date_range = cfg['date_range']
+    sources['central_kitchen_finished'] = _find_file(
+        [ck_dir], f'{date_range} 23 59 59完工入库统计分析报表.xlsx'
+    ) or os.path.join(ck_dir, f'{date_range} 23 59 59完工入库统计分析报表.xlsx')
+    sources['central_kitchen_recipe'] = _find_file(
+        [ck_dir], f'按配方导出-货品实际与理论耗用({date_range}).xlsx'
+    ) or os.path.join(ck_dir, f'按配方导出-货品实际与理论耗用({date_range}).xlsx')
+    sources['central_kitchen_material'] = _find_file(
+        [ck_dir], f'按货品导出-货品实际与理论耗用({date_range}).xlsx'
+    ) or os.path.join(ck_dir, f'按货品导出-货品实际与理论耗用({date_range}).xlsx')
+    sources['central_kitchen_processing'] = _find_file(
+        [ck_dir], '加工单价分析报表.xlsx'
+    ) or os.path.join(ck_dir, '加工单价分析报表.xlsx')
+
+    # 7. 配送明细（2个文件）
+    sources['distribution'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half1"]}.xlsx')
+    sources['distribution_2'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half2"]}.xlsx')
+
+    # 8. 薪资+考勤
+    sources['salary'] = resolve('salary', cfg['salary_file'])
+    sources['attendance'] = resolve('attendance', cfg['attendance_file'])
+
+    # 9. 门店位置
+    store_info_file = cfg.get('store_info_file')
+    if store_info_file:
+        sources['store_location'] = resolve('', store_info_file)
+    else:
+        # 5月无新文件，复用4月
+        for p in [
+            os.path.join(DATA_SBRAIN, '4月', '各店信息新（20260508）.xls'),
+            os.path.join(DATA_ROOT, '4月', '各店信息新（20260508）.xls'),
+        ]:
+            if os.path.isfile(p):
+                sources['store_location'] = p
+                break
+        else:
+            sources['store_location'] = None
+
+    return sources
+
+
+# ============================================================
+# 物化视图刷新顺序（按依赖分层，同层可并行）
+# ============================================================
+MATVIEW_LAYERS = [
+    # L0: 基础物化视图，依赖基础表
+    ['bill_fact'],
+
+    # L1: 依赖 bill_fact 或基础表
+    [
+        'mv_daily_revenue',
+        'mv_dish_basket_monthly',
+        'mv_dish_pair_summary_monthly',
+        'mv_dish_sku_summary_monthly',
+        'mv_dish_store_summary_monthly',
+        'mv_district_site_benchmark_monthly',
+        'mv_inventory_cost_classified_monthly',
+        'mv_site_segment_benchmark_monthly',
+        'mv_store_action_priority_deep_monthly',
+        'mv_store_area_efficiency_monthly',
+        'mv_store_benchmark_composite_monthly',
+        'mv_store_category_mix_monthly',
+        'mv_store_deep_diagnosis_monthly',
+        'mv_store_member_opportunity_monthly',
+        'mv_store_overlap_risk_monthly',
+        'mv_store_platform_economics_monthly',
+        'mv_store_repeat_summary_monthly',
+        'mv_store_risk_rating_monthly',
+        'mv_store_scorecard',
+        'mv_store_site_profile_monthly',
+        'mv_store_site_replication_monthly',
+        'mv_store_theoretical_actual_cost_monthly',
+        'dish_sales_april',
+        'v_store_benchmark',
+        'v_store_category_mix',
+        'v_store_platform_economics',
+    ],
+
+    # L2: 依赖 L1 物化视图
+    [
+        'dish_basket_april',
+        'dish_category_summary_april',
+        'dish_member_sku_april',
+        'dish_sku_summary_april',
+        'dish_store_sku_april',
+        'mv_dish_sku_abc_monthly',
+        'v_store_action_list',
+    ],
+
+    # L3: 依赖 L2
+    [
+        'dish_store_summary_april',
+        'mv_store_action_priority_deep_april',
+        'mv_store_theoretical_actual_cost_april',
+        'v_store_execution_priority',
+    ],
+]
+
+
+def get_dbname():
+    """获取默认数据库名"""
+    return 'bill_query_test'

db/pipeline/import_bill_records.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""
+账单查询原始数据导入脚本
+从Excel文件导入bill_records和bill_columns表。
+
+用法:
+  python3 import_bill_records.py --month 2026-04-01 --file <文件或目录> --db bill_query_test
+  python3 import_bill_records.py --month 2026-04-01 --file <目录> --db bill_query_test --skip-columns
+"""
+import argparse
+import hashlib
+import os
+import sys
+
+import pandas as pd
+import psycopg2
+from psycopg2.extras import execute_values
+
+
+def file_sha256(filepath):
+    h = hashlib.sha256()
+    with open(filepath, 'rb') as f:
+        for chunk in iter(lambda: f.read(8192), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def to_text(val):
+    if val is None:
+        return None
+    if isinstance(val, float) and pd.isna(val):
+        return None
+    s = str(val).strip()
+    if s.lower() in ('nan', 'none', ''):
+        return None
+    return s
+
+
+def import_bill_columns(conn, filepaths):
+    """从第一个Excel文件提取列定义，导入bill_columns表"""
+    cur = conn.cursor()
+    cur.execute("SELECT count(*) FROM public.bill_columns")
+    existing = cur.fetchone()[0]
+    if existing > 0:
+        print(f"bill_columns已有{existing}行，跳过列定义导入")
+        return
+
+    if not filepaths:
+        print("无文件可用，跳过列定义导入")
+        return
+
+    fpath = filepaths[0]
+    df = pd.read_excel(fpath, header=None, nrows=2, engine='openpyxl')
+    main_headers = df.iloc[0].tolist()
+    sub_headers = df.iloc[1].tolist()
+
+    columns = []
+    for i, (mh, sh) in enumerate(zip(main_headers, sub_headers)):
+        col_name = f"c{i+1:03d}"
+        group_h = to_text(mh) or ''
+        sub_h = to_text(sh) or ''
+        # If group and sub are the same, only keep group
+        if group_h and sub_h and group_h == sub_h:
+            sub_h = ''
+        columns.append((col_name, i + 1, group_h, sub_h))
+
+    execute_values(
+        cur,
+        "INSERT INTO public.bill_columns (column_name, excel_position, group_header, sub_header) VALUES %s",
+        columns,
+        page_size=500
+    )
+    conn.commit()
+    print(f"bill_columns导入完成: {len(columns)} 列")
+    cur.close()
+
+
+def import_bill_records(conn, filepath, report_month):
+    """导入单个Excel文件的账单数据到bill_records"""
+    fname = os.path.basename(filepath)
+    sha = file_sha256(filepath)
+
+    cur = conn.cursor()
+
+    # 幂等检查
+    cur.execute(
+        "SELECT 1 FROM public.bill_records WHERE source_file = %s LIMIT 1",
+        (fname,)
+    )
+    if cur.fetchone():
+        print(f"  {fname} 已导入过，跳过")
+        cur.close()
+        return 0
+
+    # 读取Excel：row0=主表头, row1=子表头, 数据从row2开始
+    df = pd.read_excel(filepath, header=None, engine='openpyxl')
+    # 跳过前2行（表头）
+    data_df = df.iloc[2:]
+    # 过滤空行（第一列为空）
+    data_df = data_df.dropna(subset=[data_df.columns[0]])
+    data_rows = data_df.values.tolist()
+
+    print(f"  数据行数: {len(data_rows)}, 列数: {data_df.shape[1]}")
+
+    if not data_rows:
+        cur.close()
+        return 0
+
+    # 获取DB列名 (c001 ~ c196)
+    num_cols = data_df.shape[1]
+    col_names = [f"c{i+1:03d}" for i in range(num_cols)]
+    all_cols = ['source_file', 'source_row'] + col_names
+    col_str = ','.join(all_cols)
+
+    batch = []
+    for idx, row in enumerate(data_rows):
+        source_row = idx + 3  # 1-indexed, data starts at Excel row 3
+        vals = [fname, source_row]
+        for i in range(num_cols):
+            v = row[i] if i < len(row) else None
+            vals.append(to_text(v))
+        batch.append(tuple(vals))
+        if len(batch) >= 500:
+            execute_values(
+                cur,
+                f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
+                batch,
+                page_size=500
+            )
+            conn.commit()
+            if (idx + 1) % 10000 == 0:
+                print(f"  已导入 {idx + 1}/{len(data_rows)} 行")
+            batch = []
+
+    if batch:
+        execute_values(
+            cur,
+            f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
+            batch,
+            page_size=500
+        )
+        conn.commit()
+
+    print(f"  已导入 {len(data_rows)}/{len(data_rows)} 行")
+    cur.close()
+    return len(data_rows)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='账单查询原始数据导入')
+    parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
+    parser.add_argument('--file', required=True, help='Excel文件路径或目录')
+    parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
+    parser.add_argument('--skip-columns', action='store_true', help='跳过bill_columns导入')
+
+    args = parser.parse_args()
+
+    conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
+    conn.autocommit = False
+
+    # 收集文件
+    files = []
+    if os.path.isdir(args.file):
+        for f in sorted(os.listdir(args.file)):
+            if f.endswith('.xlsx') and not f.startswith('.'):
+                files.append(os.path.join(args.file, f))
+    else:
+        files.append(args.file)
+
+    print(f"\n=== 账单查询原始数据导入 ===")
+    print(f"文件数: {len(files)}")
+    print(f"数据库: {args.db}")
+
+    try:
+        # 导入列定义
+        if not args.skip_columns:
+            import_bill_columns(conn, files)
+
+        # 导入数据
+        total = 0
+        for idx, fpath in enumerate(files):
+            fname = os.path.basename(fpath)
+            print(f"\n[{idx+1}/{len(files)}] {fname}")
+            total += import_bill_records(conn, fpath, args.month)
+
+        print(f"\n账单数据导入完成: 共 {total} 行")
+        print("=== 导入完成 ===")
+
+    except Exception as e:
+        conn.rollback()
+        print(f"错误: {e}", file=sys.stderr)
+        raise
+    finally:
+        conn.close()
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/import_bill_records_may.py

@@ -0,0 +1,549 @@
+#!/usr/bin/env python3
+"""
+5月账单数据导入脚本
+将5月格式的"正餐事业部_全渠道订单明细"(51列)导入bill_records(196列结构)，
+将"正餐事业部_品项销售明细"(50列)导入dish_sales_details。
+
+5月数据格式与4月不同：
+- 4月: 账单查询196列宽表 + 菜品销售明细表(多文件)
+- 5月: 全渠道订单明细51列 + 品项销售明细50列 (各1个文件)
+
+用法:
+  python3 import_bill_records_may.py --month 2026-05-01 \
+    --orders <全渠道订单明细.xlsx> \
+    --items <品项销售明细.xlsx> \
+    --db bill_query_test
+"""
+import argparse
+import hashlib
+import os
+import re
+import sys
+
+import pandas as pd
+import psycopg2
+from psycopg2.extras import execute_values
+
+
+DB_CONFIG = {
+    'host': 'localhost',
+    'port': 5432,
+    'user': 'freedak',
+    'password': '',
+}
+
+
+def file_sha256(filepath):
+    h = hashlib.sha256()
+    with open(filepath, 'rb') as f:
+        for chunk in iter(lambda: f.read(8192), b''):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+def to_text(val):
+    if val is None:
+        return None
+    if isinstance(val, float) and pd.isna(val):
+        return None
+    s = str(val).strip()
+    if s.lower() in ('nan', 'none', '', '--'):
+        return None
+    return s
+
+
+def to_num(val):
+    if val is None:
+        return None
+    if isinstance(val, float) and pd.isna(val):
+        return None
+    s = str(val).strip().replace(',', '').replace('¥', '').replace('￥', '')
+    if s.lower() in ('nan', 'none', '', '--'):
+        return None
+    try:
+        return str(float(s))
+    except (ValueError, TypeError):
+        return None
+
+
+def to_timestamp_str(val):
+    """将日期/时间值转为字符串格式 YYYY/MM/DD HH:MM:SS"""
+    if val is None:
+        return None
+    if isinstance(val, float) and pd.isna(val):
+        return None
+    if isinstance(val, pd.Timestamp):
+        return val.strftime('%Y/%m/%d %H:%M:%S')
+    s = str(val).strip()
+    if s.lower() in ('nan', 'none', '', '--'):
+        return None
+    # 尝试解析各种日期格式
+    for fmt in ['%Y/%m/%d %H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y/%m/%d', '%Y-%m-%d']:
+        try:
+            ts = pd.to_datetime(s, format=fmt)
+            return ts.strftime('%Y/%m/%d %H:%M:%S')
+        except (ValueError, TypeError):
+            continue
+    # pandas通用解析
+    try:
+        ts = pd.to_datetime(s)
+        return ts.strftime('%Y/%m/%d %H:%M:%S')
+    except (ValueError, TypeError):
+        return s
+
+
+# ============================================================
+# 5月全渠道订单明细 → bill_records 列映射
+# 5月col索引 → bill_records cXXX
+# ============================================================
+ORDERS_COL_MAP = {
+    # 5月col: (bill_records col_name, converter)
+    0:  None,           # 省份 → 无对应
+    1:  None,           # 城市 → 无对应
+    2:  'c003',         # 门店名称 → c003 store_name
+    3:  None,           # 营业日期 → 无直接对应(bill_fact用closed_at)
+    4:  'c004',         # 餐段 → c004 meal_period
+    5:  'c005',         # 订单号 → c005 bill_no
+    6:  None,           # 原单号
+    7:  None,           # 外卖订单号
+    8:  None,           # 全渠道流水号
+    9:  'c008',         # 取餐号 → c008 table_no
+    10: None,           # 桌牌号
+    11: 'c006',         # 桌台区域 → c006 business_area
+    12: 'c009',         # 订单金额 → c009 consumption
+    13: None,           # 顾客实付
+    14: 'c114',         # 订单收入 → c114 received_total
+    15: 'c068',         # 订单优惠 → c068 discount_total
+    16: None,           # 订单来源
+    17: None,           # 订单子来源
+    18: None,           # 经营模式
+    19: None,           # 用餐方式
+    20: None,           # 宴会类型
+    21: 'c194',         # 订单状态 → c194 bill_status
+    22: None,           # 退单标识
+    23: 'c190',         # 下单员 → c190 waiter
+    24: 'c191',         # 收银员 → c191 cashier
+    25: None,           # 桌台提成人
+    26: None,           # 销售员 → c192 (不在bill_fact中，但可填)
+    27: 'c175',         # 创建时间 → c175 opened_at
+    28: 'c176',         # 完成时间 → c176 closed_at
+    29: None,           # 接单时间
+    30: None,           # 下单制作时间
+    31: None,           # 预定用餐时间
+    32: None,           # 订单备注
+    33: None,           # 桌台备注
+    34: 'c178',         # 用餐人数 → c178 guest_count
+    35: 'c179',         # 席数 → c179 table_count
+    36: None,           # 是否会员
+    37: None,           # 会员姓名
+    38: 'c185',         # 会员卡号 → c185 member_id
+    39: None,           # 会员手机号
+    40: None,           # 下单人手机号
+    41: None,           # 发票
+    42: None,           # 菜品金额
+    43: None,           # 菜品收入
+    44: None,           # 退单时间
+    45: None,           # 取消时间
+    46: None,           # 外卖订单已退金额
+    47: None,           # 敏感操作
+    48: None,           # 结账方式 → 需要解析支付方式
+    49: None,           # 企业版订单
+    50: None,           # 企业版订单是否开发票
+}
+
+# 结账方式解析 → bill_records支付方式列
+PAYMENT_PATTERNS = {
+    'c140': [r'现金'],                        # 现金
+    'c143': [r'支付宝'],                       # 支付宝支付实收
+    'c144': [r'微信'],                         # 微信支付实收
+    'c145': [r'美团.*实收', r'大众点评.*实收', r'新美大实收', r'一键买单'],  # 新美大实收
+    'c146': [r'云闪付'],                       # 云闪付实收
+    'c147': [r'抖音'],                         # 抖音券实收
+    'c151': [r'美团外卖'],                     # 美团外卖实收
+    'c152': [r'淘宝闪购', r'闪购'],            # 淘宝闪购实收
+    'c150': [r'京东外卖'],                     # 京东外卖实收
+    'c149': [r'挂账'],                         # 挂账消费
+}
+
+# 折扣/佣金解析
+DISCOUNT_PATTERNS = {
+    'c087': [r'会员积分'],                     # 会员积分抵现
+    'c088': [r'会员代金券'],                   # 会员代金券折扣
+    'c089': [r'会员预存'],                     # 会员预存折扣
+    'c095': [r'抖音.*折扣', r'抖音券折扣'],     # 抖音券折扣
+    'c097': [r'美团.*佣金'],                   # 美团外卖佣金
+    'c098': [r'淘宝.*佣金', r'闪购.*佣金'],    # 淘宝闪购佣金
+    'c100': [r'京东.*佣金'],                   # 京东外卖佣金
+}
+
+
+def parse_payment(payment_str):
+    """解析结账方式字段，返回 {cXXX: amount_str}"""
+    result = {}
+    if not payment_str or payment_str == '--':
+        return result
+
+    # 格式: "方式1 金额1,方式2 金额2" 或 "方式1 金额1"
+    parts = payment_str.split(',')
+    for part in parts:
+        part = part.strip()
+        # 提取金额（最后一个数字）
+        amount_match = re.search(r'([\d.]+)\s*$', part)
+        amount = amount_match.group(1) if amount_match else None
+
+        if not amount:
+            continue
+
+        # 匹配支付方式
+        for col, patterns in PAYMENT_PATTERNS.items():
+            if col in result:
+                continue
+            for pat in patterns:
+                if re.search(pat, part):
+                    if col in result:
+                        result[col] = str(float(result[col]) + float(amount))
+                    else:
+                        result[col] = amount
+                    break
+
+    return result
+
+
+def parse_discounts(discount_str):
+    """解析折扣字段"""
+    return parse_payment(discount_str)  # 同样的解析逻辑
+
+
+def build_store_code_map(items_filepath):
+    """从品项销售明细中构建 门店名称→机构编码 映射"""
+    df = pd.read_excel(items_filepath, header=2, engine='openpyxl')
+    df = df.dropna(subset=[df.columns[0]])
+
+    mapping = {}
+    if '门店名称' in df.columns and '机构编码' in df.columns:
+        for _, row in df[['门店名称', '机构编码']].drop_duplicates().iterrows():
+            name = str(row['门店名称']).strip()
+            code = str(row['机构编码']).strip()
+            if name and code and name != 'nan' and code != 'nan' and code != '--':
+                mapping[name] = code
+
+    print(f"  门店编码映射: {len(mapping)} 个门店")
+    for name, code in sorted(mapping.items()):
+        print(f"    {code} → {name}")
+
+    return mapping
+
+
+def import_bill_records_may(conn, orders_filepath, store_code_map, report_month):
+    """将5月全渠道订单明细导入bill_records"""
+    fname = os.path.basename(orders_filepath)
+    sha = file_sha256(orders_filepath)
+
+    cur = conn.cursor()
+
+    # 幂等检查
+    cur.execute(
+        "SELECT 1 FROM public.bill_records WHERE source_file = %s LIMIT 1",
+        (fname,)
+    )
+    if cur.fetchone():
+        print(f"  {fname} 已导入过，跳过")
+        cur.close()
+        return 0
+
+    # 读取Excel：row0=标题, row1=日期范围, row2=列头, row3+=数据
+    df = pd.read_excel(orders_filepath, header=2, engine='openpyxl')
+    df = df.dropna(subset=[df.columns[0]])
+    data_rows = df.values.tolist()
+
+    print(f"  数据行数: {len(data_rows)}, 列数: {df.shape[1]}")
+    if not data_rows:
+        cur.close()
+        return 0
+
+    # 构建196列的列名
+    all_col_names = [f"c{i+1:03d}" for i in range(196)]
+    insert_cols = ['source_file', 'source_row'] + all_col_names
+    col_str = ','.join(insert_cols)
+
+    # 反转映射: 5月col_index → cXXX
+    col_to_c = {}
+    for may_col, c_name in ORDERS_COL_MAP.items():
+        if c_name:
+            col_to_c[may_col] = c_name
+
+    batch = []
+    for idx, row in enumerate(data_rows):
+        source_row = idx + 4  # 数据从Excel第4行开始(1-indexed)
+        # 初始化196列全为None
+        c_vals = {f"c{i+1:03d}": None for i in range(196)}
+
+        # 映射已知字段
+        for may_idx, c_name in col_to_c.items():
+            if may_idx < len(row):
+                val = row[may_idx]
+                # 时间字段特殊处理
+                if c_name in ('c175', 'c176'):
+                    c_vals[c_name] = to_timestamp_str(val)
+                elif c_name in ('c009', 'c068', 'c114', 'c178', 'c179'):
+                    c_vals[c_name] = to_num(val)
+                else:
+                    c_vals[c_name] = to_text(val)
+
+        # 从门店名称查找门店编码 → c002
+        store_name = to_text(row[2]) if len(row) > 2 else None
+        if store_name and store_name in store_code_map:
+            c_vals['c002'] = store_code_map[store_name]
+
+        # 序号 → c001
+        c_vals['c001'] = str(idx + 1)
+
+        # 解析结账方式 → 支付方式列
+        payment_str = to_text(row[48]) if len(row) > 48 else None
+        if payment_str:
+            payments = parse_payment(payment_str)
+            for c_name, amount in payments.items():
+                c_vals[c_name] = amount
+
+        # 计算时长(分) → c177
+        opened = c_vals.get('c175')
+        closed = c_vals.get('c176')
+        if opened and closed:
+            try:
+                t1 = pd.to_datetime(opened, format='%Y/%m/%d %H:%M:%S')
+                t2 = pd.to_datetime(closed, format='%Y/%m/%d %H:%M:%S')
+                duration = (t2 - t1).total_seconds() / 60
+                if duration >= 0:
+                    c_vals['c177'] = str(int(duration))
+            except (ValueError, TypeError):
+                pass
+
+        # 构建行数据
+        vals = [fname, source_row] + [c_vals[f"c{i+1:03d}"] for i in range(196)]
+        batch.append(tuple(vals))
+
+        if len(batch) >= 500:
+            execute_values(
+                cur,
+                f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
+                batch,
+                page_size=500
+            )
+            conn.commit()
+            if (idx + 1) % 5000 == 0:
+                print(f"  已导入 {idx + 1}/{len(data_rows)} 行")
+            batch = []
+
+    if batch:
+        execute_values(
+            cur,
+            f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
+            batch,
+            page_size=500
+        )
+        conn.commit()
+
+    print(f"  已导入 {len(data_rows)}/{len(data_rows)} 行")
+    cur.close()
+    return len(data_rows)
+
+
+# ============================================================
+# 5月品项销售明细 → dish_sales_details 字段映射
+# ============================================================
+ITEMS_FIELD_MAP = {
+    '城市': None,                          # col0
+    '机构编码': 'store_code',               # col1
+    '门店名称': 'store_name',               # col2
+    '营业日期': None,                       # col3
+    '下单时间所属餐段': 'business_type',     # col4
+    '出品部门': 'production_department',    # col5
+    '菜品大类': 'category_level1',          # col6
+    '菜品小类': 'category_level2',          # col7
+    '菜品编码': None,                       # col8
+    '品项名称': 'dish_name',                # col9
+    '关联菜品名称': None,                   # col10
+    '商品别名': None,                       # col11
+    '品项类型': 'item_type',                # col12
+    '菜品类型': None,                       # col13
+    '菜品标签': None,                       # col14
+    '规格': None,                           # col15
+    '单位': 'unit',                         # col16
+    '关联做法': 'preparation_method',       # col17
+    '关联加料': None,                       # col18
+    '关联餐盒': None,                       # col19
+    '销售方式': None,                       # col20
+    '订单号': 'bill_no',                    # col21
+    '销售数量': 'sales_quantity',           # col22
+    '赠送数量': None,                       # col23
+    '销售金额(元)': 'gross_amount',         # col24
+    '赠送金额(元)': None,                   # col25
+    '优惠金额(元)': None,                   # col26
+    '品项收入(元)': 'received_amount',      # col27
+    '点菜时间': None,                       # col28
+    '下单时间': 'ordered_at',               # col29
+    '接单/结账/退菜时间': None,             # col30
+    '收银员': None,                         # col31
+    '点菜员': None,                         # col32
+    '下单人': None,                         # col33
+    '订单分类': None,                       # col34
+    '订单来源': None,                       # col35
+    '新订单来源': None,                     # col36
+    '订单子来源': None,                     # col37
+    '桌台区域': None,                       # col38
+    '取餐号': 'table_or_pickup_no',         # col39
+    '桌牌号': None,                         # col40
+    '订单金额(元)': None,                   # col41
+    '营业额(元)': None,                     # col42
+    '订单优惠(元)': None,                   # col43
+    '订单收入(元)': None,                   # col44
+    '标记': None,                           # col45
+    '退菜数量': None,                       # col46
+    '退菜金额(元)': None,                   # col47
+    '敏感操作类型': None,                   # col48
+    '单品备注': None,                       # col49
+}
+
+NUMERIC_FIELDS = {'sales_quantity', 'gross_amount', 'received_amount', 'unit_price', 'guest_count'}
+TIMESTAMP_FIELDS = {'opened_at', 'closed_at', 'ordered_at'}
+
+
+def import_dish_sales_may(conn, items_filepath, report_month):
+    """将5月品项销售明细导入dish_sales_details"""
+    fname = os.path.basename(items_filepath)
+
+    cur = conn.cursor()
+
+    # 幂等检查
+    cur.execute(
+        "SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s",
+        (fname,)
+    )
+    if cur.fetchone():
+        print(f"  {fname} 已导入过，跳过")
+        return 0
+
+    file_size = os.path.getsize(items_filepath)
+
+    # 读取Excel：row0=标题, row1=日期范围, row2=列头, row3+=数据
+    df = pd.read_excel(items_filepath, header=2, engine='openpyxl')
+    df = df.dropna(subset=[df.columns[0]])
+    data_rows = df.values.tolist()
+    headers = list(df.columns)
+
+    print(f"  数据行数: {len(data_rows)}, 列数: {df.shape[1]}")
+    if not data_rows:
+        return 0
+
+    # 构建列名→索引映射
+    col_map = {}
+    for i, h in enumerate(headers):
+        h_str = str(h).strip() if h else ''
+        if h_str:
+            col_map[h_str] = i
+
+    # 确定要插入的列
+    cols_to_insert = {}
+    for header_name, db_col in ITEMS_FIELD_MAP.items():
+        if db_col and header_name in col_map:
+            cols_to_insert[db_col] = col_map[header_name]
+
+    # 插入import log
+    cur.execute(
+        """INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status)
+           VALUES (%s, %s, %s, 'importing')""",
+        (fname, file_size, len(data_rows))
+    )
+
+    all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys())
+    col_names = ','.join(all_cols)
+
+    batch = []
+    for idx, r in enumerate(data_rows):
+        source_row = idx + 4
+        vals = [fname, source_row]
+        for db_col, excel_idx in cols_to_insert.items():
+            v = r[excel_idx] if excel_idx < len(r) else None
+            if db_col in NUMERIC_FIELDS:
+                vals.append(to_num(v))
+            elif db_col in TIMESTAMP_FIELDS:
+                vals.append(to_timestamp_str(v))
+            else:
+                vals.append(to_text(v))
+        batch.append(tuple(vals))
+
+        if len(batch) >= 500:
+            execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
+            conn.commit()
+            if (idx + 1) % 10000 == 0:
+                print(f"  已导入 {idx + 1}/{len(data_rows)} 行")
+            batch = []
+
+    if batch:
+        execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
+        conn.commit()
+
+    print(f"  已导入 {len(data_rows)}/{len(data_rows)} 行")
+
+    # 更新log状态
+    cur.execute(
+        "UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s",
+        (fname,)
+    )
+    conn.commit()
+
+    cur.close()
+    return len(data_rows)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='5月账单数据导入(全渠道订单明细+品项销售明细)')
+    parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
+    parser.add_argument('--orders', required=True, help='全渠道订单明细Excel文件路径')
+    parser.add_argument('--items', required=True, help='品项销售明细Excel文件路径')
+    parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
+    parser.add_argument('--skip-orders', action='store_true', help='跳过订单明细导入')
+    parser.add_argument('--skip-items', action='store_true', help='跳过品项销售明细导入')
+
+    args = parser.parse_args()
+
+    print(f"\n=== 5月账单数据导入 ===")
+    print(f"月份: {args.month}")
+    print(f"数据库: {args.db}")
+    print(f"订单明细: {args.orders}")
+    print(f"品项明细: {args.items}")
+
+    cfg = DB_CONFIG.copy()
+    cfg['dbname'] = args.db
+    conn = psycopg2.connect(**cfg)
+    conn.autocommit = False
+
+    try:
+        # 先从品项明细构建门店编码映射
+        store_code_map = {}
+        if not args.skip_items and os.path.isfile(args.items):
+            print("\n--- 构建门店编码映射 ---")
+            store_code_map = build_store_code_map(args.items)
+
+        # 导入bill_records
+        if not args.skip_orders and os.path.isfile(args.orders):
+            print("\n--- 导入全渠道订单明细 → bill_records ---")
+            import_bill_records_may(conn, args.orders, store_code_map, args.month)
+
+        # 导入dish_sales_details
+        if not args.skip_items and os.path.isfile(args.items):
+            print("\n--- 导入品项销售明细 → dish_sales_details ---")
+            import_dish_sales_may(conn, args.items, args.month)
+
+        print("\n=== 5月账单数据导入完成 ===")
+
+    except Exception as e:
+        conn.rollback()
+        print(f'错误: {e}', file=sys.stderr)
+        raise
+    finally:
+        conn.close()
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/import_derived_data.py

@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+"""
+派生分析数据导入脚本
+生成以下派生表：
+  - dish_diagnosis_snapshot (菜品诊断快照，从dish_cost_analysis_summary + BOM生成)
+  - central_kitchen_manufacturing_cost_pool (中央厨房制造成本池，从薪资+营业费用+完工报表派生)
+
+用法:
+  python3 import_derived_data.py --month 2026-04-01 --db bill_query_test
+"""
+import argparse
+import datetime
+import sys
+
+import psycopg2
+
+
+def generate_dish_diagnosis(conn, diag_date):
+    """生成菜品诊断快照"""
+    cur = conn.cursor()
+
+    # 检查是否已有数据
+    cur.execute("SELECT count(*) FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
+    existing = cur.fetchone()[0]
+    if existing > 0:
+        print(f"  dish_diagnosis_snapshot 已有 {existing} 行 (date={diag_date})，先删除再生成")
+
+    cur.execute("DELETE FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
+
+    # 检查依赖表是否存在且有数据
+    cur.execute("SELECT count(*) FROM public.dish_cost_analysis_summary")
+    summary_count = cur.fetchone()[0]
+    if summary_count == 0:
+        print("  跳过: dish_cost_analysis_summary 无数据")
+        cur.close()
+        return 0
+
+    # 检查 analytics.fact_recipe_bom 是否存在
+    cur.execute("""
+        SELECT EXISTS (
+            SELECT 1 FROM information_schema.tables
+            WHERE table_schema = 'analytics' AND table_name = 'fact_recipe_bom'
+        )
+    """)
+    has_bom = cur.fetchone()[0]
+
+    bom_join = ""
+    bom_select = "0 AS cnt, 0 AS unique_cnt, 0 AS avg_waste"
+    if has_bom:
+        bom_select = "COALESCE(bom.cnt, 0), COALESCE(bom.unique_cnt, 0), COALESCE(round(bom.avg_waste::numeric, 2), 0)"
+        bom_join = """
+            LEFT JOIN (
+                SELECT b.sku_code,
+                    count(*) AS cnt,
+                    count(*) FILTER (WHERE r.ref_count = 1) AS unique_cnt,
+                    sum(b.waste_rate * b.standard_gross_quantity) / nullif(sum(b.standard_gross_quantity), 0) AS avg_waste
+                FROM analytics.fact_recipe_bom b
+                LEFT JOIN (
+                    SELECT material_code, count(DISTINCT sku_code) AS ref_count
+                    FROM analytics.fact_recipe_bom GROUP BY material_code
+                ) r ON r.material_code = b.material_code
+                GROUP BY b.sku_code
+            ) bom ON bom.sku_code = s.dish_code
+        """
+
+    sql = f"""
+        INSERT INTO public.dish_diagnosis_snapshot (
+            diagnosis_date, dish_code, dish_name, category_l1,
+            sales_amount, sales_quantity,
+            theoretical_margin_pct, actual_margin_pct, cost_variance_amount,
+            cost_tier, bom_complexity_score, unique_material_count, waste_rate_avg,
+            diagnosis_type, diagnosis_detail, suggested_action, priority
+        )
+        SELECT
+            '{diag_date}'::date,
+            s.dish_code,
+            s.dish_name,
+            s.category_level1,
+            round(s.sales_amount::numeric, 2),
+            round(s.sales_quantity::numeric, 2),
+            round(s.theoretical_margin_rate_pct::numeric, 2),
+            round(s.actual_margin_rate_pct::numeric, 2),
+            round(s.cost_variance_amount::numeric, 2),
+            CASE
+                WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
+                WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
+                     AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN '紧急'
+                WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
+                     AND (s.cost_variance_amount / s.theoretical_cost) > 0.1 THEN '整改'
+                WHEN s.cost_variance_amount > 0 THEN '关注'
+                ELSE '正常'
+            END,
+            {bom_select},
+            CASE
+                WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
+                WHEN s.theoretical_margin_rate_pct < 0 THEN '负毛利'
+                WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN '低毛利-定价偏低'
+                WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '低毛利-成本超耗'
+                WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '高超耗-份量超标'
+                WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 100 THEN '高超耗-分摊异常'
+                WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN '配方复杂-低销量'
+                WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN '独有原料风险'
+                WHEN s.cost_variance_amount > 0 THEN '成本差异'
+                ELSE '正常'
+            END,
+            CASE
+                WHEN s.actual_margin_rate_pct < 0 THEN '实际毛利率为负，需先核查BOM/单位/分摊'
+                WHEN s.theoretical_margin_rate_pct < 0 THEN '理论毛利率为负，定价低于标准成本'
+                WHEN s.theoretical_margin_rate_pct < 30 THEN '理论毛利率低于30%，定价偏低'
+                WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '实际成本超理论，存在超耗'
+                WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '物料损耗率超过20%'
+                ELSE '成本基本正常'
+            END,
+            CASE
+                WHEN s.actual_margin_rate_pct < 0 THEN 'fix_data'
+                WHEN s.theoretical_margin_rate_pct < 0 THEN 'price_up'
+                WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN 'price_up'
+                WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN 'recipe_optimize'
+                WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN 'portion_reduce'
+                WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'delist'
+                WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'evaluate_delist'
+                WHEN s.cost_variance_amount > 0 THEN 'monitor'
+                ELSE 'keep'
+            END,
+            CASE
+                WHEN s.actual_margin_rate_pct < 0 THEN 'P0'
+                WHEN s.theoretical_margin_rate_pct < 0 THEN 'P0'
+                WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
+                     AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN 'P0'
+                WHEN s.theoretical_margin_rate_pct < 50 THEN 'P1'
+                WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'P2'
+                WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'P2'
+                ELSE 'P3'
+            END
+        FROM public.dish_cost_analysis_summary s
+        {bom_join}
+        WHERE s.dish_code IS NOT NULL
+    """
+
+    cur.execute(sql)
+    inserted = cur.rowcount
+    conn.commit()
+    print(f"  dish_diagnosis_snapshot 生成完成: {inserted} 行 (date={diag_date})")
+    cur.close()
+    return inserted
+
+
+def generate_manufacturing_cost_pool(conn, report_month):
+    """生成中央厨房制造成本池"""
+    cur = conn.cursor()
+
+    # 检查是否已有数据
+    cur.execute("SELECT count(*) FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
+    existing = cur.fetchone()[0]
+    if existing > 0:
+        print(f"  central_kitchen_manufacturing_cost_pool 已有 {existing} 行，先删除再生成")
+
+    cur.execute("DELETE FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
+
+    # 检查依赖表是否有数据
+    cur.execute("SELECT count(*) FROM public.salary_detail_records")
+    salary_count = cur.fetchone()[0]
+    cur.execute("SELECT count(*) FROM public.operating_expense_records WHERE report_month = %s", (report_month,))
+    expense_count = cur.fetchone()[0]
+    cur.execute("SELECT count(*) FROM public.central_kitchen_finished_receipt WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date", (report_month, report_month))
+    ck_count = cur.fetchone()[0]
+
+    if salary_count == 0 and expense_count == 0 and ck_count == 0:
+        print("  跳过: 依赖表无数据")
+        cur.close()
+        return 0
+
+    rows = []
+
+    # 1. 直接人工：中央厨房生产人员工资
+    if salary_count > 0:
+        cur.execute("""
+            SELECT round(coalesce(sum(net_pay), 0)::numeric, 2)
+            FROM public.salary_detail_records
+            WHERE org_level1 LIKE '%加工配送中心%' OR org_level1 LIKE '%中央厨房%'
+               OR org_level2 LIKE '%加工配送中心%' OR org_level2 LIKE '%中央厨房%'
+               OR org_level3 LIKE '%加工配送中心%' OR org_level3 LIKE '%中央厨房%'
+               OR org_level4 LIKE '%加工配送中心%' OR org_level4 LIKE '%中央厨房%'
+               OR org_level5 LIKE '%加工配送中心%' OR org_level5 LIKE '%中央厨房%'
+               OR org_level6 LIKE '%加工配送中心%' OR org_level6 LIKE '%中央厨房%'
+               OR org_level7 LIKE '%加工配送中心%' OR org_level7 LIKE '%中央厨房%'
+               OR org_level8 LIKE '%加工配送中心%' OR org_level8 LIKE '%中央厨房%'
+        """)
+        salary_total = cur.fetchone()[0]
+        if salary_total and float(salary_total) > 0:
+            rows.append((
+                report_month, '直接人工', '中央厨房生产人员工资', '薪资明细',
+                'salary_detail_records:加工配送中心/中央厨房',
+                float(salary_total), 1.0, float(salary_total),
+                '中央厨房组织直接归集', False, True,
+                None
+            ))
+
+    # 2. 制造费用：从营业费用中供应链部分按比例分摊
+    if expense_count > 0:
+        # 中央厨房人数/加工配送中心人数比例 = 7/11 ≈ 0.6364
+        ck_ratio = 0.63636364
+        expense_items = [
+            ('50302', '餐厅房租', 'operating_expense_records:供应链/50302'),
+            ('50307', '水费', 'operating_expense_records:供应链/50307'),
+            ('50308', '电费', 'operating_expense_records:供应链/50308'),
+            ('50310', '员工宿舍费用', 'operating_expense_records:供应链/50310'),
+            ('50315', '维修费', 'operating_expense_records:供应链/50315'),
+        ]
+        for acct_code, acct_name, source_ref in expense_items:
+            cur.execute("""
+                SELECT round(coalesce(sum(amount), 0)::numeric, 2)
+                FROM public.operating_expense_records
+                WHERE report_month = %s AND account_code = %s
+                  AND cost_unit_source_name LIKE '供应链%%'
+            """, (report_month, acct_code))
+            amount = cur.fetchone()[0]
+            if amount and float(amount) > 0:
+                allocated = round(float(amount) * ck_ratio, 6)
+                rows.append((
+                    report_month, '制造费用', acct_name, '营业费用',
+                    source_ref, float(amount), ck_ratio, allocated,
+                    '中央厨房人数/加工配送中心人数（临时）', True, True,
+                    '供应链共享费用，待电表、面积或工时数据后替换'
+                ))
+
+    # 3. 源报表费用：完工报表费用成本
+    if ck_count > 0:
+        cur.execute("""
+            SELECT round(coalesce(sum(source_fee_cost), 0)::numeric, 4)
+            FROM public.central_kitchen_finished_receipt
+            WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date
+        """, (report_month, report_month))
+        fee_total = cur.fetchone()[0]
+        if fee_total and float(fee_total) > 0:
+            rows.append((
+                report_month, '源报表费用', '完工报表费用成本', '完工入库报表',
+                'central_kitchen_finished_receipt.source_fee_cost',
+                float(fee_total), 1.0, float(fee_total),
+                '源报表原值', True, False,
+                '无水电人工等明细来源，为避免与工资和营业费用重复，暂不计入重建成本'
+            ))
+
+    if rows:
+        from psycopg2.extras import execute_values
+        execute_values(cur, """
+            INSERT INTO public.central_kitchen_manufacturing_cost_pool
+            (report_month, cost_type, cost_subtype, source_type, source_reference,
+             source_amount, central_kitchen_share_pct, allocated_amount,
+             allocation_method, is_provisional, include_in_rebuilt_cost, note)
+            VALUES %s
+        """, rows, page_size=100)
+
+    conn.commit()
+    print(f"  central_kitchen_manufacturing_cost_pool 生成完成: {len(rows)} 行")
+    cur.close()
+    return len(rows)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='派生分析数据导入')
+    parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
+    parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
+    parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)')
+
+    args = parser.parse_args()
+
+    conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
+    conn.autocommit = False
+
+    diag_date = args.diagnosis_date or datetime.date.today().isoformat()
+
+    try:
+        print(f"\n=== 派生分析数据生成 ===")
+        print(f"数据库: {args.db}, 月份: {args.month}")
+
+        print("\n--- 菜品诊断快照 ---")
+        generate_dish_diagnosis(conn, diag_date)
+
+        print("\n--- 中央厨房制造成本池 ---")
+        generate_manufacturing_cost_pool(conn, args.month)
+
+        print("\n=== 生成完成 ===")
+
+    except Exception as e:
+        conn.rollback()
+        print(f"错误: {e}", file=sys.stderr)
+        raise
+    finally:
+        conn.close()
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/import_may_data.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+# ============================================================
+# 5月数据导入脚本
+# 用法: bash import_may_data.sh [数据库名]
+# 默认数据库: bill_query_test
+# ============================================================
+set -e
+
+DB="${1:-bill_query_test}"
+MONTH="2026-05-01"
+DIAG_DATE="2026-08-18"
+
+# 数据根目录
+DATA="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/5月"
+DATA4="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/4月"
+PIPELINE="/Users/freedak/Documents/AIDashboard/SBrainCO3/db/pipeline"
+PY=python3
+
+echo "============================================================"
+echo "  5月数据导入 → 数据库: ${DB}"
+echo "  月份: ${MONTH}"
+echo "============================================================"
+
+# ============================================================
+# Step 1: 薪资 + 考勤
+# ============================================================
+echo ""
+echo "=== [1/9] 薪资 + 考勤 ==="
+${PY} "${PIPELINE}/import_salary_attendance.py" \
+  --month "${MONTH}" \
+  --db "${DB}" \
+  --salary-file "${DATA}/营业数据/2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx" \
+  --attendance-file "${DATA}/营业数据/2026年5月考勤数据表.xlsx"
+
+# ============================================================
+# Step 2: bill_records + dish_sales_details（5月专用脚本）
+# 5月格式: 全渠道订单明细(51列) + 品项销售明细(50列)
+# ============================================================
+echo ""
+echo "=== [2/9] bill_records + 菜品销售明细 ==="
+${PY} "${PIPELINE}/import_bill_records_may.py" \
+  --month "${MONTH}" \
+  --orders "${DATA}/营业数据/正餐事业部_全渠道订单明细_20260730_1112_1785381233074.xlsx" \
+  --items "${DATA}/营业数据/正餐事业部_品项销售明细_20260730_1128_1785382191936.xlsx" \
+  --db "${DB}"
+
+# ============================================================
+# Step 4: 库存成本（盘点倒挤成本）
+# ============================================================
+echo ""
+echo "=== [4/9] 库存成本 ==="
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset inventory \
+  --month "${MONTH}" \
+  --file "${DATA}/供应链数据/4.13盘点倒挤成本报表2026年5月.xlsx" \
+  --db "${DB}"
+
+# ============================================================
+# Step 5: 菜品成本BOM
+# ============================================================
+echo ""
+echo "=== [5/9] 菜品成本BOM ==="
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset dish_cost \
+  --month "${MONTH}" \
+  --file "${DATA}/营业数据/2026年5月菜品成本分析报表.xlsx" \
+  --db "${DB}"
+
+# ============================================================
+# Step 6: 营业费用
+# ============================================================
+echo ""
+echo "=== [6/9] 营业费用 ==="
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset operating_expense \
+  --month "${MONTH}" \
+  --file "${DATA}/2026年5月营业费用分析.xls" \
+  --db "${DB}"
+
+# ============================================================
+# Step 7: 中央厨房（4个子文件）
+# ============================================================
+echo ""
+echo "=== [7/9] 中央厨房 ==="
+echo "  [7a] 完工入库"
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset central_kitchen --ck-type finished \
+  --month "${MONTH}" \
+  --file "${DATA}/中央厨房/2026-05-01--2026-05-31 23 59 59完工入库统计分析报表.xlsx" \
+  --db "${DB}"
+
+echo "  [7b] 配方耗用"
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset central_kitchen --ck-type recipe \
+  --month "${MONTH}" \
+  --file "${DATA}/中央厨房/按配方导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \
+  --db "${DB}"
+
+echo "  [7c] 货品耗用"
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset central_kitchen --ck-type material \
+  --month "${MONTH}" \
+  --file "${DATA}/中央厨房/按货品导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \
+  --db "${DB}"
+
+echo "  [7d] 加工单价"
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset central_kitchen --ck-type processing \
+  --month "${MONTH}" \
+  --file "${DATA}/中央厨房/加工单价分析报表.xlsx" \
+  --db "${DB}"
+
+# ============================================================
+# Step 8: 配送明细（2个文件）
+# ============================================================
+echo ""
+echo "=== [8/9] 配送明细 ==="
+echo "  [8a] 1-15日"
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset distribution \
+  --month "${MONTH}" \
+  --file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月1-15日.xlsx" \
+  --db "${DB}"
+
+echo "  [8b] 16-31日"
+${PY} "${PIPELINE}/import_monthly_data.py" \
+  --dataset distribution \
+  --month "${MONTH}" \
+  --file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月16-31日.xlsx" \
+  --db "${DB}"
+
+# ============================================================
+# Step 9: 门店位置与映射表（复用4月各店信息文件）
+# ============================================================
+echo ""
+echo "=== [9/9] 门店位置与映射表 ==="
+${PY} "${PIPELINE}/import_store_location.py" \
+  --file "${DATA4}/各店信息新（20260508）.xls" \
+  --db "${DB}"
+
+# ============================================================
+# Step 10: 派生分析数据
+# ============================================================
+echo ""
+echo "=== [10] 派生分析数据 ==="
+${PY} "${PIPELINE}/import_derived_data.py" \
+  --month "${MONTH}" \
+  --db "${DB}" \
+  --diagnosis-date "${DIAG_DATE}"
+
+# ============================================================
+# Step 11: 物化视图刷新
+# ============================================================
+echo ""
+echo "=== [11] 物化视图刷新 ==="
+${PY} "${PIPELINE}/refresh_materialized_views.py" \
+  --db "${DB}"
+
+# ============================================================
+# Step 12: 验证
+# ============================================================
+echo ""
+echo "=== [12] 数据一致性验证 ==="
+${PY} "${PIPELINE}/verify.py" \
+  --db "${DB}" || true
+
+echo ""
+echo "============================================================"
+echo "  5月数据导入完成"
+echo "============================================================"

db/pipeline/import_salary_attendance.py

@@ -3,12 +3,13 @@
 薪资拆分明细表 + 考勤数据表 导入脚本
 遵循现有 import_log + records 模式
 """
-import openpyxl
 import hashlib
 import psycopg2
 import os
 import sys
-from datetime import datetime
+import argparse
+import pandas as pd
+from psycopg2.extras import execute_values
 
 DB_CONFIG = {
     'host': 'localhost',
@@ -18,9 +19,9 @@ DB_CONFIG = {
     'password': '',
 }
 
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-SALARY_FILE = os.path.join(BASE_DIR, '马兰拉面数据分析', '薪资拆分明细表_脱敏.xlsx')
+SALARY_FILE = os.path.join(BASE_DIR, '数据', '4月', '薪资拆分明细表_脱敏.xlsx')
-ATTENDANCE_FILE = os.path.join(BASE_DIR, '马兰拉面数据分析', '考勤数据表-脱敏.xlsx')
+ATTENDANCE_FILE = os.path.join(BASE_DIR, '数据', '4月', '考勤数据表-脱敏.xlsx')
 
 def file_sha256(filepath):
     h = hashlib.sha256()
@@ -31,34 +32,42 @@ def file_sha256(filepath):
 
 def to_num(val):
     if val is None or val == '':
-        return 0
+        return None
+    if isinstance(val, float) and pd.isna(val):
+        return None
     try:
         return float(val)
     except (ValueError, TypeError):
-        return 0
+        return None
 
 def to_text(val):
     if val is None:
         return None
+    if isinstance(val, float) and pd.isna(val):
+        return None
     s = str(val).strip()
+    if s.lower() == 'nan' or s.lower() == 'none':
+        return None
     return s if s else None
 
-def import_salary(conn, report_month='2026-04-01'):
+def import_salary(conn, report_month='2026-04-01', salary_file=None):
-    print(f'导入薪资拆分明细表: {SALARY_FILE}')
+    filepath = salary_file or SALARY_FILE
-    sha = file_sha256(SALARY_FILE)
+    print(f'导入薪资拆分明细表: {filepath}')
-    wb = openpyxl.load_workbook(SALARY_FILE, read_only=True, data_only=True)
+    sha = file_sha256(filepath)
-    ws = wb[wb.sheetnames[0]]
+    
-
+    # Read with pandas (row 3 = headers, row 4+ = data)
-    # Row 1: title, Row 2: merged title, Row 3: headers, Row 4+: data
+    raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
-    rows = list(ws.iter_rows(min_row=4, values_only=True))
+    print(f'  总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}')
-    data_rows = [r for r in rows if r[0] is not None and r[9] is not None and str(r[9]).strip()]
+    
+    # Data starts from row 3 (0-indexed), filter rows with employee_code (col 9)
+    df = raw_df.iloc[3:].copy()
+    df = df[df[9].notna() & (df[9].astype(str).str.strip() != '') & (df[9].astype(str).str.strip() != 'nan')]
+    data_rows = df.values.tolist()
     print(f'  数据行数: {len(data_rows)}')
-
+    
-    report_month = report_month
+    source_file = os.path.basename(filepath)
-    source_file = os.path.basename(SALARY_FILE)
 
     cur = conn.cursor()
-    # Check if already imported
     cur.execute(
         'SELECT import_id FROM public.salary_import_log WHERE report_month = %s AND source_file = %s AND file_sha256 = %s',
         (report_month, source_file, sha)
@@ -66,10 +75,8 @@ def import_salary(conn, report_month='2026-04-01'):
     existing = cur.fetchone()
     if existing:
         print(f'  已导入过, import_id={existing[0]}, 跳过')
-        wb.close()
         return
 
-    # Insert import log
     cur.execute(
         '''INSERT INTO public.salary_import_log (report_month, source_file, file_sha256, workbook_rows, imported_rows)
            VALUES (%s, %s, %s, %s, %s) RETURNING import_id''',
@@ -78,9 +85,8 @@ def import_salary(conn, report_month='2026-04-01'):
     import_id = cur.fetchone()[0]
     print(f'  import_id={import_id}')
 
-    # Batch insert
+    # Column mapping: use execute_values for batch insert
-    insert_sql = '''INSERT INTO public.salary_detail_records (
+    col_names = '''import_id, source_row,
-        import_id, source_row,
         org_level1, org_level2, org_level3, org_level4, org_level5, org_level6, org_level7, org_level8,
         employee_code, salary_period, position, work_type, employment_type, hire_date, leave_date,
         salary_standard, base_wage, overtime_subsidy, social_subsidy, position_wage, tenure_wage,
@@ -100,13 +106,11 @@ def import_salary(conn, report_month='2026-04-01'):
         external_subsidy, external_other_deduction, external_gross,
         pension_deduction, medical_deduction, unemployment_deduction,
         external_social_total, external_tax, external_net, internal_tax, internal_net,
-        external_unit, attendance_remark, employment_type_orig, salary_category
+        external_unit, attendance_remark, employment_type_orig, salary_category'''
-    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'''
 
     batch = []
-    batch_size = 500
     for idx, r in enumerate(data_rows):
-        source_row = idx + 4  # Excel row number
+        source_row = idx + 4
         vals = [
             import_id, source_row,
             to_text(r[1]), to_text(r[2]), to_text(r[3]), to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]),
@@ -130,34 +134,36 @@ def import_salary(conn, report_month='2026-04-01'):
             to_num(r[90]), to_num(r[91]), to_num(r[92]), to_num(r[93]), to_num(r[94]),
             to_text(r[95]), to_text(r[96]), to_text(r[97]), to_text(r[98]),
         ]
-        batch.append(vals)
+        batch.append(tuple(vals))
-        if len(batch) >= batch_size:
+        if len(batch) >= 500:
-            cur.executemany(insert_sql, batch)
+            execute_values(cur, f'INSERT INTO public.salary_detail_records ({col_names}) VALUES %s', batch, page_size=500)
             conn.commit()
             print(f'  已导入 {idx + 1}/{len(data_rows)} 行')
             batch = []
 
     if batch:
-        cur.executemany(insert_sql, batch)
+        execute_values(cur, f'INSERT INTO public.salary_detail_records ({col_names}) VALUES %s', batch, page_size=500)
         conn.commit()
         print(f'  已导入 {len(data_rows)}/{len(data_rows)} 行')
 
-    wb.close()
     print(f'  薪资明细导入完成: {len(data_rows)} 行')
 
-def import_attendance(conn, report_month='2026-04-01'):
+def import_attendance(conn, report_month='2026-04-01', attendance_file=None):
-    print(f'\n导入考勤数据表: {ATTENDANCE_FILE}')
+    filepath = attendance_file or ATTENDANCE_FILE
-    sha = file_sha256(ATTENDANCE_FILE)
+    print(f'\n导入考勤数据表: {filepath}')
-    wb = openpyxl.load_workbook(ATTENDANCE_FILE, read_only=True, data_only=True)
+    sha = file_sha256(filepath)
-    ws = wb[wb.sheetnames[0]]
+    
-
+    # Read with pandas (row 0 = headers, row 1+ = data)
-    # Row 1: headers, Row 2+: data
+    raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
-    rows = list(ws.iter_rows(min_row=2, values_only=True))
+    print(f'  总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}')
-    data_rows = [r for r in rows if r[0] is not None]
+    
+    # Data starts from row 1
+    df = raw_df.iloc[1:].copy()
+    df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan')]
+    data_rows = df.values.tolist()
     print(f'  数据行数: {len(data_rows)}')
-
+    
-    report_month = report_month
+    source_file = os.path.basename(filepath)
-    source_file = os.path.basename(ATTENDANCE_FILE)
 
     cur = conn.cursor()
     cur.execute(
@@ -167,7 +173,6 @@ def import_attendance(conn, report_month='2026-04-01'):
     existing = cur.fetchone()
     if existing:
         print(f'  已导入过, import_id={existing[0]}, 跳过')
-        wb.close()
         return
 
     cur.execute(
@@ -178,44 +183,52 @@ def import_attendance(conn, report_month='2026-04-01'):
     import_id = cur.fetchone()[0]
     print(f'  import_id={import_id}')
 
-    # 33 columns: 工号, 岗位, 所在部门, day_01..day_30
+    # 34 columns: import_id, source_row, employee_code, position, department, day_01..day_31
     cols = ['import_id', 'source_row', 'employee_code', 'position', 'department']
-    cols += [f'day_{str(i).zfill(2)}' for i in range(1, 31)]
+    cols += [f'day_{str(i).zfill(2)}' for i in range(1, 32)]
-    placeholders = ', '.join(['%s'] * len(cols))
+    col_names = ','.join(cols)
-    col_names = ', '.join(cols)
-    insert_sql = f'INSERT INTO public.attendance_records ({col_names}) VALUES ({placeholders})'
 
     batch = []
-    batch_size = 500
     for idx, r in enumerate(data_rows):
         source_row = idx + 2
         vals = [import_id, source_row, to_text(r[0]), to_text(r[1]), to_text(r[2])]
-        # Days 1-30 (columns 3-32)
+        # Days 1-31 (columns 3-33)
-        for d in range(30):
+        for d in range(31):
             vals.append(to_text(r[3 + d]) if 3 + d < len(r) else None)
-        batch.append(vals)
+        batch.append(tuple(vals))
-        if len(batch) >= batch_size:
+        if len(batch) >= 500:
-            cur.executemany(insert_sql, batch)
+            execute_values(cur, f'INSERT INTO public.attendance_records ({col_names}) VALUES %s', batch, page_size=500)
             conn.commit()
             print(f'  已导入 {idx + 1}/{len(data_rows)} 行')
             batch = []
 
     if batch:
-        cur.executemany(insert_sql, batch)
+        execute_values(cur, f'INSERT INTO public.attendance_records ({col_names}) VALUES %s', batch, page_size=500)
         conn.commit()
         print(f'  已导入 {len(data_rows)}/{len(data_rows)} 行')
 
-    wb.close()
     print(f'  考勤数据导入完成: {len(data_rows)} 行')
 
 def main():
-    report_month = sys.argv[1] if len(sys.argv) > 1 else '2026-04-01'
+    parser = argparse.ArgumentParser(description='薪资考勤数据导入')
-    print(f'导入月份: {report_month}')
+    parser.add_argument('--month', default='2026-04-01', help='报告月份')
-    conn = psycopg2.connect(**DB_CONFIG)
+    parser.add_argument('--db', default='bill_query', help='数据库名')
+    parser.add_argument('--skip-salary', action='store_true', help='跳过薪资导入')
+    parser.add_argument('--skip-attendance', action='store_true', help='跳过考勤导入')
+    parser.add_argument('--salary-file', default=None, help='薪资Excel文件路径 (默认使用内置路径)')
+    parser.add_argument('--attendance-file', default=None, help='考勤Excel文件路径 (默认使用内置路径)')
+    args = parser.parse_args()
+    
+    print(f'导入月份: {args.month}, 数据库: {args.db}')
+    cfg = DB_CONFIG.copy()
+    cfg['dbname'] = args.db
+    conn = psycopg2.connect(**cfg)
     conn.autocommit = False
     try:
-        import_salary(conn, report_month)
+        if not args.skip_salary:
-        import_attendance(conn, report_month)
+            import_salary(conn, args.month, args.salary_file)
+        if not args.skip_attendance:
+            import_attendance(conn, args.month, args.attendance_file)
         print('\n=== 导入完成 ===')
     except Exception as e:
         conn.rollback()

db/pipeline/import_store_location.py

@@ -0,0 +1,619 @@
+#!/usr/bin/env python3
+"""
+门店位置与映射表导入脚本
+从 各店信息新（20260508）.xls 导入以下表：
+  - store_location_source_rows (原始行)
+  - store_location_master (门店主表，含地理编码)
+  - sales_store_location_mapping (销售门店→位置映射)
+
+同时导入静态映射表：
+  - store_name_mapping (薪资名称↔账单名称映射，8条手工数据)
+  - inventory_store_mapping (库存成本单位→销售门店映射)
+  - operating_expense_store_mapping (费用单位→销售门店映射)
+
+用法:
+  python3 import_store_location.py --file <各店信息.xls> --db bill_query_test
+  python3 import_store_location.py --file <各店信息.xls> --db bill_query_test --skip-mappings
+"""
+import argparse
+import math
+import os
+import re
+import sys
+from datetime import date, datetime
+from difflib import SequenceMatcher
+
+import pandas as pd
+import psycopg2
+from psycopg2.extras import Json, execute_values
+
+
+# ============================================================
+# 静态数据
+# ============================================================
+
+# WGS84行政区中心点（低精度兜底）
+ADMIN_CENTROIDS = {
+    "北京市东城区": (39.92855, 116.41637),
+    "北京市西城区": (39.91231, 116.36679),
+    "北京市朝阳区": (39.92149, 116.44355),
+    "北京市海淀区": (39.95933, 116.29845),
+    "北京市丰台区": (39.85856, 116.28625),
+    "北京市石景山区": (39.90569, 116.22298),
+    "北京市昌平区": (40.22077, 116.23128),
+    "北京市大兴区": (39.72684, 116.34159),
+    "北京市通州区": (39.90249, 116.65643),
+    "北京市房山区": (39.74788, 116.14327),
+    "北京市怀柔区": (40.31600, 116.63170),
+    "北京市北京经济技术开发区": (39.79500, 116.50600),
+    "上海市浦东新区": (31.22114, 121.54409),
+    "上海市青浦区": (31.15074, 121.12417),
+    "浙江省杭州市余杭区": (30.41875, 120.29940),
+    "陕西省西安市": (34.34157, 108.93977),
+}
+
+MANUAL_ALIAS = {
+    "火锅北三环店": "火锅",
+    "双安总店": "双安",
+    "温泉店": "温泉西部马华",
+    "生命园路店": "北大生命园",
+    "永丰悦界店": "永丰路",
+    "哈马尔罕总部基地店": "哈马尔罕",
+}
+
+# 薪资名称 → 账单名称 手工映射
+STORE_NAME_MAPPING = [
+    ("双安店", "双安总店"),
+    ("安宁庄快手店", "安宁庄快手"),
+    ("海淀大街店", "海淀大街"),
+    ("百子湾店", "百子湾路店"),
+    ("哈马尔罕大钟寺店", "大钟寺店"),
+    ("大钟寺店", "大钟寺店"),
+    ("阿里疆（温泉路店）", "温泉店"),
+    ("温泉店", "温泉店"),
+]
+
+
+# ============================================================
+# 工具函数
+# ============================================================
+
+def clean(v):
+    if v is None:
+        return None
+    s = str(v).strip()
+    return s or None
+
+
+def normalize_name(v):
+    s = clean(v) or ""
+    s = re.sub(r"[（(].*?[）)]", "", s)
+    for token in ("西部马华", "牛肉面", "餐饮店", "餐厅", "总店"):
+        s = s.replace(token, "")
+    s = re.sub(r"店$", "", s)
+    return re.sub(r"\s+", "", s)
+
+
+def parse_date(v):
+    if isinstance(v, datetime):
+        return v.date()
+    if isinstance(v, date):
+        return v
+    if isinstance(v, (int, float)):
+        try:
+            return (datetime(1899, 12, 30) + datetime.timedelta(days=float(v))).date()
+        except Exception:
+            return None
+    s = clean(v)
+    if not s or s in {"长期", "未开业"}:
+        return None
+    try:
+        return datetime.fromisoformat(s).date()
+    except ValueError:
+        pass
+    for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d"):
+        try:
+            return datetime.strptime(s, fmt).date()
+        except ValueError:
+            pass
+    return None
+
+
+def parse_area(v):
+    if isinstance(v, (int, float)):
+        return float(v), "numeric"
+    s = clean(v)
+    if not s:
+        return None, "missing"
+    nums = [float(x) for x in re.findall(r"\d+(?:\.\d+)?", s)]
+    if len(nums) >= 2 and ("-" in s or "至" in s or "~" in s):
+        return sum(nums[:2]) / 2, "range_midpoint"
+    if nums:
+        return nums[0], "text_numeric"
+    return None, "unparsed"
+
+
+def parse_admin(address):
+    a = clean(address) or ""
+    if "上海市" in a:
+        province, city = "上海市", "上海市"
+    elif "浙江省" in a or "杭州市" in a:
+        province, city = "浙江省", "杭州市"
+    elif "陕西省" in a or "西安市" in a:
+        province, city = "陕西省", "西安市"
+    else:
+        province, city = "北京市", "北京市"
+
+    district = None
+    candidates = ["东城区", "西城区", "朝阳区", "海淀区", "丰台区", "石景山区",
+                  "昌平区", "大兴区", "通州区", "房山区", "怀柔区",
+                  "浦东新区", "青浦区", "余杭区"]
+    for item in candidates:
+        if item in a:
+            district = item
+            break
+    if "北京经济技术开发区" in a:
+        district = "北京经济技术开发区"
+    return province, city, district
+
+
+def wgs84_to_gcj02(lat, lon):
+    if lat is None or lon is None:
+        return None, None
+    if not (72.004 <= lon <= 137.8347 and 0.8293 <= lat <= 55.8271):
+        return lat, lon
+    a, ee = 6378245.0, 0.00669342162296594323
+    dlat = _transform_lat(lon - 105.0, lat - 35.0)
+    dlon = _transform_lon(lon - 105.0, lat - 35.0)
+    radlat = lat / 180.0 * math.pi
+    magic = math.sin(radlat)
+    magic = 1 - ee * magic * magic
+    sqrtmagic = math.sqrt(magic)
+    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * math.pi)
+    dlon = (dlon * 180.0) / (a / sqrtmagic * math.cos(radlat) * math.pi)
+    return lat + dlat, lon + dlon
+
+
+def _transform_lat(x, y):
+    ret = -100.0 + 2.0*x + 3.0*y + 0.2*y*y + 0.1*x*y + 0.2*math.sqrt(abs(x))
+    ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0
+    ret += (20.0*math.sin(y*math.pi) + 40.0*math.sin(y/3.0*math.pi))*2.0/3.0
+    ret += (160.0*math.sin(y/12.0*math.pi) + 320*math.sin(y*math.pi/30.0))*2.0/3.0
+    return ret
+
+
+def _transform_lon(x, y):
+    ret = 300.0 + x + 2.0*y + 0.1*x*x + 0.1*x*y + 0.1*math.sqrt(abs(x))
+    ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0
+    ret += (20.0*math.sin(x*math.pi) + 40.0*math.sin(x/3.0*math.pi))*2.0/3.0
+    ret += (150.0*math.sin(x/12.0*math.pi) + 300.0*math.sin(x/30.0*math.pi))*2.0/3.0
+    return ret
+
+
+def fallback_geocode(province, city, district, address):
+    if not address:
+        return None, None, None, "pending_no_address", 0.0
+    key = f"{city}{district}" if district else city
+    if province not in {"北京市", "上海市"} and city:
+        key = f"{province}{city}{district or ''}"
+    coord = ADMIN_CENTROIDS.get(key)
+    if coord:
+        precision = "district_centroid" if district else "city_centroid"
+        confidence = 0.25 if district else 0.10
+        return coord[0], coord[1], key, precision, confidence
+    city_key = f"{province}{city}" if province != city else city
+    coord = ADMIN_CENTROIDS.get(city_key)
+    if coord:
+        return coord[0], coord[1], city_key, "city_centroid", 0.10
+    return None, None, None, "pending_exact_geocode", 0.0
+
+
+# ============================================================
+# 导入函数
+# ============================================================
+
+def import_store_location(conn, filepath):
+    """导入门店位置主表和源行表"""
+    source_file = os.path.basename(filepath)
+    print(f"\n=== 门店位置导入 ===")
+    print(f"文件: {source_file}")
+
+    cur = conn.cursor()
+
+    # 幂等检查
+    cur.execute("SELECT 1 FROM public.store_location_master WHERE source_file = %s LIMIT 1", (source_file,))
+    if cur.fetchone():
+        print("  位置数据已导入过，跳过位置导入")
+        # 仍然需要重建销售门店映射
+        _rebuild_sales_mapping(conn)
+        cur.close()
+        return
+
+    # 读取Excel
+    df = pd.read_excel(filepath, header=0, engine='xlrd')
+    print(f"  总行数: {len(df)}")
+
+    # 清理数据
+    raw_rows = []
+    numbered = []
+    for idx, row in df.iterrows():
+        row_no = idx + 2  # 1-indexed from row 2
+        seq = row.iloc[0]
+        seq_int = int(seq) if isinstance(seq, (int, float)) and not pd.isna(seq) else None
+        raw = {
+            "row": row_no, "seq": seq_int,
+            "name": clean(row.iloc[1]) if len(row) > 1 else None,
+            "company": clean(row.iloc[2]) if len(row) > 2 else None,
+            "brand": clean(row.iloc[3]) if len(row) > 3 else None,
+            "address": clean(row.iloc[4]) if len(row) > 4 else None,
+            "area": clean(row.iloc[5]) if len(row) > 5 else None,
+            "opened": clean(row.iloc[6]) if len(row) > 6 else None,
+            "lease": clean(row.iloc[7]) if len(row) > 7 else None,
+            "license": clean(row.iloc[8]) if len(row) > 8 else None,
+            "note": clean(row.iloc[9]) if len(row) > 9 else None,
+        }
+        raw_rows.append(raw)
+        if seq_int is not None:
+            numbered.append(raw)
+
+    print(f"  编号门店数: {len(numbered)}")
+
+    # 清理旧数据
+    cur.execute("TRUNCATE public.sales_store_location_mapping RESTART IDENTITY CASCADE")
+    cur.execute("DELETE FROM public.store_location_master WHERE source_file = %s OR source_file = 'sales_db_placeholder'", (source_file,))
+    cur.execute("DELETE FROM public.store_location_source_rows WHERE source_file = %s", (source_file,))
+
+    # 导入源行
+    source_values = [(
+        source_file, r["row"], r["seq"], r["name"], r["company"], r["brand"], r["address"],
+        r["area"], r["opened"], r["lease"], r["license"], r["note"]
+    ) for r in raw_rows]
+    execute_values(cur, """
+        INSERT INTO public.store_location_source_rows
+        (source_file, source_row, source_store_no, store_short_name, company_name, brand_name,
+         business_address, area_raw, opened_raw, lease_expiry_raw, license_raw, note)
+        VALUES %s
+    """, source_values, page_size=500)
+    print(f"  源行导入: {len(source_values)} 行")
+
+    # 构建主表数据
+    masters = []
+    for r in numbered:
+        area_sqm, area_method = parse_area(r["area"])
+        province, city, district = parse_admin(r["address"])
+        lat, lon, display, precision, confidence = fallback_geocode(province, city, district, r["address"])
+        gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
+        name = r["name"] or f"未命名门店{r['seq']}"
+        if "停业" in name:
+            status = "停业"
+        elif r["opened"] == "未开业":
+            status = "未开业"
+        else:
+            status = "在册"
+        masters.append({
+            **r, "area_sqm": area_sqm, "area_method": area_method,
+            "opened_date": parse_date(r["opened"]), "lease_date": parse_date(r["lease"]),
+            "license_date": parse_date(r["license"]), "status": status,
+            "province": province, "city": city, "district": district,
+            "lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon,
+            "display": display, "precision": precision, "confidence": confidence,
+        })
+
+    # 占位门店
+    placeholders = [
+        {"name": "甄选商城店", "address": None, "province": None, "city": None, "district": None,
+         "status": "线上虚拟门店", "precision": "not_applicable", "display": None, "confidence": 0.0},
+        {"name": "西安含光店", "address": "陕西省西安市含光路（具体门牌待补）",
+         "province": "陕西省", "city": "西安市", "district": None,
+         "status": "地址待补", "precision": "city_centroid", "display": "陕西省西安市", "confidence": 0.10},
+    ]
+    for i, p in enumerate(placeholders, start=1):
+        coord = ADMIN_CENTROIDS.get(f"{p['province']}{p['city']}") if p["province"] else None
+        lat, lon = coord if coord else (None, None)
+        gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
+        masters.append({
+            "row": None, "seq": 9000 + i, "name": p["name"], "company": None,
+            "brand": "西部马华牛肉面", "address": p["address"], "area": None,
+            "opened": None, "lease": None, "license": None, "note": "经营数据占位记录",
+            "area_sqm": None, "area_method": "missing", "opened_date": None,
+            "lease_date": None, "license_date": None, "status": p["status"],
+            "province": p["province"], "city": p["city"], "district": p["district"],
+            "lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon,
+            "display": p["display"], "precision": p["precision"], "confidence": p["confidence"],
+        })
+
+    # 导入主表
+    location_ids = {}
+    for m in masters:
+        provider = "offline_admin_centroid" if m["lat"] is not None else "none"
+        geocode_status = "fallback_low_precision" if m["lat"] is not None else m["precision"]
+        src_file = source_file if m["seq"] < 9000 else "sales_db_placeholder"
+        cur.execute("""
+            INSERT INTO public.store_location_master
+            (source_file, source_row, source_store_no, store_short_name, company_name, brand_name,
+             business_address, area_raw, area_sqm, area_parse_method, opened_raw, opened_date,
+             lease_expiry_raw, lease_expiry_date, license_raw, license_date, operating_status,
+             province, city, district, geocode_query, geocode_provider, geocode_status,
+             geocode_precision, geocode_confidence, geocode_display_name,
+             latitude_wgs84, longitude_wgs84, latitude_gcj02, longitude_gcj02, geocode_raw, geocoded_at)
+            VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,
+                    %s,%s,%s,%s,%s, now())
+            RETURNING location_id
+        """, (
+            src_file, m["row"], m["seq"], m["name"], m["company"], m["brand"], m["address"],
+            m["area"], m["area_sqm"], m["area_method"], m["opened"], m["opened_date"],
+            m["lease"], m["lease_date"], m["license"], m["license_date"], m["status"],
+            m["province"], m["city"], m["district"], m["address"],
+            provider, geocode_status, m["precision"], m["confidence"], m["display"],
+            m["lat"], m["lon"], m["gcj_lat"], m["gcj_lon"],
+            Json({"notice": "行政区中心点兜底，不是门店精确坐标"}) if m["lat"] is not None else None
+        ))
+        location_ids[m["name"]] = cur.fetchone()[0]
+
+    print(f"  主表导入: {len(masters)} 家门店")
+
+    # 从DB读取location_ids用于映射
+    cur.execute("SELECT location_id, store_short_name FROM public.store_location_master")
+    location_ids = {name: lid for lid, name in cur.fetchall()}
+
+    _rebuild_sales_mapping(conn, location_ids, masters)
+
+    conn.commit()
+    cur.close()
+    print(f"门店位置导入完成")
+
+
+def _rebuild_sales_mapping(conn, location_ids=None, masters=None):
+    """重建销售门店→位置映射表"""
+    cur = conn.cursor()
+
+    # 如果没有传入location_ids，从DB读取
+    if location_ids is None:
+        cur.execute("SELECT location_id, store_short_name FROM public.store_location_master")
+        location_ids = {name: lid for lid, name in cur.fetchall()}
+
+    if not location_ids:
+        print("  跳过销售门店映射: 无位置数据")
+        cur.close()
+        return
+
+    # 清理旧映射
+    cur.execute("TRUNCATE public.sales_store_location_mapping")
+
+    # 获取销售门店列表
+    try:
+        cur.execute("SELECT store_code, store_name FROM analytics.v_store_scorecard ORDER BY store_name")
+        sales_stores = cur.fetchall()
+    except Exception:
+        sales_stores = []
+        print("  警告: analytics.v_store_scorecard 不存在，跳过销售门店映射")
+        cur.close()
+        return
+
+    # 构建候选列表
+    if masters:
+        source_candidates = [(m["name"], normalize_name(m["name"])) for m in masters]
+    else:
+        source_candidates = [(name, normalize_name(name)) for name in location_ids.keys()]
+
+    mapped = []
+    for store_code, store_name in sales_stores:
+        target_name = MANUAL_ALIAS.get(store_name)
+        method = "manual_alias" if target_name else "normalized_name"
+        score = 1.0 if target_name else 0.0
+        if not target_name:
+            nn = normalize_name(store_name)
+            exact = [x for x in source_candidates if x[1] == nn and nn]
+            if exact:
+                target_name, _ = exact[0]
+                score = 1.0
+            else:
+                ranked = []
+                for candidate, cn in source_candidates:
+                    s = SequenceMatcher(None, nn, cn).ratio()
+                    if nn and cn and (nn in cn or cn in nn):
+                        s = max(s, 0.92)
+                    ranked.append((s, candidate))
+                if ranked:
+                    score, target_name = max(ranked)
+                    method = "fuzzy_name"
+        location_id = location_ids.get(target_name)
+        if not location_id or score < 0.60:
+            print(f"  警告: 经营门店无法映射: {store_code} {store_name} -> {target_name} score={score}")
+            continue
+        status = "confirmed" if method in {"manual_alias", "normalized_name"} or score >= 0.85 else "reviewed_low_confidence"
+        note = None if status == "confirmed" else "名称相似匹配，建议业务复核"
+        mapped.append((store_code, store_name, location_id, method, score, status, note))
+
+    if mapped:
+        execute_values(cur, """
+            INSERT INTO public.sales_store_location_mapping
+            (sales_store_code, sales_store_name, location_id, mapping_method, mapping_confidence, mapping_status, review_note)
+            VALUES %s
+        """, mapped, page_size=500)
+    conn.commit()
+    print(f"  销售门店映射: {len(mapped)} 家")
+    cur.close()
+
+
+def import_store_name_mapping(conn):
+    """导入薪资名称↔账单名称映射（静态8条）"""
+    cur = conn.cursor()
+    cur.execute("SELECT count(*) FROM public.store_name_mapping")
+    if cur.fetchone()[0] > 0:
+        print("store_name_mapping已有数据，跳过")
+        cur.close()
+        return
+
+    execute_values(
+        cur,
+        "INSERT INTO public.store_name_mapping (salary_name, bill_name) VALUES %s",
+        STORE_NAME_MAPPING,
+        page_size=100
+    )
+    conn.commit()
+    print(f"store_name_mapping导入完成: {len(STORE_NAME_MAPPING)} 条")
+    cur.close()
+
+
+def import_inventory_store_mapping(conn):
+    """从库存成本数据推导库存门店映射"""
+    cur = conn.cursor()
+    cur.execute("SELECT count(*) FROM public.inventory_store_mapping")
+    if cur.fetchone()[0] > 0:
+        print("inventory_store_mapping已有数据，跳过")
+        cur.close()
+        return
+
+    # 从库存成本记录中提取唯一的成本单位编码和名称
+    cur.execute("""
+        SELECT DISTINCT cost_unit_code, cost_unit_name
+        FROM public.inventory_cost_records
+        WHERE report_month = '2026-04-01'
+          AND cost_unit_code IS NOT NULL
+        ORDER BY cost_unit_code
+    """)
+    cost_units = cur.fetchall()
+
+    mappings = []
+    for code, name in cost_units:
+        # 成本单位编码与销售门店编码相同的直接映射
+        mappings.append((
+            code, name, code, name,
+            '经营门店', True, '门店编码精确匹配', 100.00, None
+        ))
+
+    if mappings:
+        execute_values(cur, """
+            INSERT INTO public.inventory_store_mapping
+            (cost_unit_code, cost_unit_name, sales_store_code, sales_store_name,
+             unit_type, include_in_operating_cost, mapping_method, mapping_confidence, review_note)
+            VALUES %s
+        """, mappings, page_size=500)
+        conn.commit()
+        print(f"inventory_store_mapping导入完成: {len(mappings)} 条")
+    else:
+        print("inventory_store_mapping: 无库存成本数据可推导")
+
+    cur.close()
+
+
+def import_operating_expense_store_mapping(conn):
+    """从营业费用数据推导费用门店映射"""
+    cur = conn.cursor()
+    cur.execute("SELECT count(*) FROM public.operating_expense_store_mapping")
+    if cur.fetchone()[0] > 0:
+        print("operating_expense_store_mapping已有数据，跳过")
+        cur.close()
+        return
+
+    # 从营业费用记录中提取唯一的成本单位名称
+    cur.execute("""
+        SELECT DISTINCT cost_unit_source_name
+        FROM public.operating_expense_records
+        WHERE report_month = '2026-04-01'
+          AND cost_unit_source_name IS NOT NULL
+        ORDER BY cost_unit_source_name
+    """)
+    expense_units = cur.fetchall()
+
+    # 尝试从销售数据获取门店列表做匹配
+    try:
+        cur.execute("SELECT store_code, store_name FROM analytics.dim_store ORDER BY store_code")
+        sales_stores = cur.fetchall()
+    except Exception:
+        sales_stores = []
+
+    mappings = []
+    for (source_name,) in expense_units:
+        # 标准化名称：去掉"金额"后缀
+        normalized = re.sub(r"金额$", "", source_name).strip()
+        # 尝试精确匹配
+        matched_code = None
+        matched_name = None
+        method = "unmapped"
+        confidence = 0.0
+
+        for sc, sn in sales_stores:
+            if normalized == sn or normalized in sn or sn in normalized:
+                matched_code = sc
+                matched_name = sn
+                method = "标准名称精确匹配"
+                confidence = 100.0
+                break
+
+        if not matched_code:
+            # 尝试编码前缀匹配
+            m = re.match(r"^(\d+)\s+", normalized)
+            if m:
+                prefix = m.group(1)
+                for sc, sn in sales_stores:
+                    if sc == prefix:
+                        matched_code = sc
+                        matched_name = sn
+                        method = "编码前缀匹配"
+                        confidence = 90.0
+                        break
+
+        mappings.append((
+            source_name, normalized,
+            matched_code, matched_name,
+            '经营门店', matched_code is not None,
+            method if matched_code else "待人工确认",
+            confidence,
+            None if matched_code else "需人工确认匹配关系"
+        ))
+
+    if mappings:
+        execute_values(cur, """
+            INSERT INTO public.operating_expense_store_mapping
+            (cost_unit_source_name, normalized_cost_unit_name, sales_store_code, sales_store_name,
+             unit_type, include_in_operating_analysis, mapping_method, mapping_confidence, review_note)
+            VALUES %s
+        """, mappings, page_size=500)
+        conn.commit()
+        print(f"operating_expense_store_mapping导入完成: {len(mappings)} 条")
+    else:
+        print("operating_expense_store_mapping: 无费用数据可推导")
+
+    cur.close()
+
+
+# ============================================================
+# Main
+# ============================================================
+
+def main():
+    parser = argparse.ArgumentParser(description='门店位置与映射表导入')
+    parser.add_argument('--file', required=True, help='各店信息Excel文件路径')
+    parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
+    parser.add_argument('--skip-location', action='store_true', help='跳过门店位置导入')
+    parser.add_argument('--skip-mappings', action='store_true', help='跳过映射表导入')
+
+    args = parser.parse_args()
+
+    conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
+    conn.autocommit = False
+
+    try:
+        if not args.skip_location:
+            import_store_location(conn, args.file)
+
+        if not args.skip_mappings:
+            print("\n=== 映射表导入 ===")
+            import_store_name_mapping(conn)
+            import_inventory_store_mapping(conn)
+            import_operating_expense_store_mapping(conn)
+
+        print("\n=== 导入完成 ===")
+
+    except Exception as e:
+        conn.rollback()
+        print(f"错误: {e}", file=sys.stderr)
+        raise
+    finally:
+        conn.close()
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/refresh_materialized_views.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+物化视图按依赖顺序刷新
+用法: python3 refresh_materialized_views.py --db bill_query_test
+"""
+import argparse
+import psycopg2
+import time
+import sys
+
+from config import DB_CONFIG, MATVIEW_LAYERS
+
+
+def refresh_all(dbname, verbose=True):
+    conn = psycopg2.connect(
+        host=DB_CONFIG['host'],
+        port=DB_CONFIG['port'],
+        dbname=dbname,
+        user=DB_CONFIG['user'],
+        password=DB_CONFIG['password'] or None,
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+
+    total = sum(len(layer) for layer in MATVIEW_LAYERS)
+    done = 0
+    failed = []
+
+    for layer_idx, layer in enumerate(MATVIEW_LAYERS):
+        if verbose:
+            print(f"\n--- L{layer_idx}: {len(layer)} 个物化视图 ---")
+        for mv in layer:
+            done += 1
+            t0 = time.time()
+            try:
+                cur.execute(f'REFRESH MATERIALIZED VIEW analytics."{mv}"')
+                elapsed = time.time() - t0
+                if verbose:
+                    print(f"  [{done}/{total}] OK  {mv} ({elapsed:.1f}s)")
+            except Exception as e:
+                elapsed = time.time() - t0
+                err_msg = str(e).strip().split('\n')[0]
+                if verbose:
+                    print(f"  [{done}/{total}] FAIL {mv} ({elapsed:.1f}s): {err_msg}")
+                failed.append((mv, err_msg))
+                conn.rollback()
+
+    cur.close()
+    conn.close()
+
+    if failed:
+        print(f"\n=== 完成: {total - len(failed)}/{total} 成功, {len(failed)} 失败 ===")
+        for mv, err in failed:
+            print(f"  FAIL {mv}: {err}")
+        # 重试失败项（可能因依赖顺序问题，第二轮能成功）
+        if len(failed) > 0:
+            print(f"\n--- 重试 {len(failed)} 个失败项 ---")
+            conn = psycopg2.connect(
+                host=DB_CONFIG['host'],
+                port=DB_CONFIG['port'],
+                dbname=dbname,
+                user=DB_CONFIG['user'],
+                password=DB_CONFIG['password'] or None,
+            )
+            conn.autocommit = True
+            cur = conn.cursor()
+            retry_failed = []
+            for mv, _ in failed:
+                t0 = time.time()
+                try:
+                    cur.execute(f'REFRESH MATERIALIZED VIEW analytics."{mv}"')
+                    elapsed = time.time() - t0
+                    print(f"  OK  {mv} ({elapsed:.1f}s)")
+                except Exception as e:
+                    err_msg = str(e).strip().split('\n')[0]
+                    print(f"  FAIL {mv}: {err_msg}")
+                    retry_failed.append(mv)
+                conn.rollback()
+            cur.close()
+            conn.close()
+            if retry_failed:
+                print(f"\n最终失败: {retry_failed}")
+                return 1
+            else:
+                print(f"\n=== 全部 {total} 个物化视图刷新成功 ===")
+                return 0
+    else:
+        print(f"\n=== 全部 {total} 个物化视图刷新成功 ===")
+        return 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description='物化视图按依赖顺序刷新')
+    parser.add_argument('--db', default='bill_query_test', help='目标数据库')
+    args = parser.parse_args()
+
+    print(f"\n=== 物化视图刷新 ===")
+    print(f"数据库: {args.db}")
+    rc = refresh_all(args.db)
+    sys.exit(rc)
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/run_all.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""
+一键全量导入 + 物化视图刷新
+用法: python3 run_all.py --month 2026-04-01 --db bill_query_test
+      python3 run_all.py --month 2026-04-01 --db bill_query_test --skip-import  # 仅刷新物化
+      python3 run_all.py --month 2026-04-01 --db bill_query_test --skip-views   # 仅导入
+      python3 run_all.py --month 2026-04-01 --db bill_query_test --only bill_records  # 仅导入某类
+"""
+import argparse
+import os
+import subprocess
+import sys
+import time
+
+from config import DB_CONFIG, get_data_sources, PIPELINE_DIR
+
+
+def run_script(name, cmd, cwd=None):
+    """运行子进程并实时输出"""
+    print(f"\n{'='*60}")
+    print(f"  执行: {name}")
+    print(f"  命令: {' '.join(cmd)}")
+    print(f"{'='*60}")
+    t0 = time.time()
+    result = subprocess.run(cmd, cwd=cwd or PIPELINE_DIR)
+    elapsed = time.time() - t0
+    status = "成功" if result.returncode == 0 else f"失败(rc={result.returncode})"
+    print(f"  → {name}: {status} ({elapsed:.1f}s)")
+    return result.returncode == 0
+
+
+def main():
+    parser = argparse.ArgumentParser(description='全量数据导入与物化刷新流水线')
+    parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
+    parser.add_argument('--db', default='bill_query_test', help='目标数据库')
+    parser.add_argument('--skip-import', action='store_true', help='跳过数据导入，仅刷新物化视图')
+    parser.add_argument('--skip-views', action='store_true', help='跳过物化视图刷新')
+    parser.add_argument('--only', default=None,
+                        help='仅导入指定数据集: bill_records, monthly, salary, store_location, derived')
+    parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)')
+    args = parser.parse_args()
+
+    sources = get_data_sources(args.month)
+    if not sources:
+        print(f"错误: 无法找到月份 {args.month} 的数据源")
+        sys.exit(1)
+
+    print(f"\n{'#'*60}")
+    print(f"  数据导入与物化刷新流水线")
+    print(f"  月份: {args.month}")
+    print(f"  数据库: {args.db}")
+    print(f"  数据源根: {sources.get('bill_records', 'N/A')}")
+    print(f"{'#'*60}")
+
+    py = sys.executable
+    results = []
+
+    # ============================================================
+    # Step 1: 数据导入
+    # ============================================================
+    if not args.skip_import:
+        target = args.only
+
+        # 1.1 薪资+考勤
+        if target in (None, 'salary'):
+            cmd = [py, 'import_salary_attendance.py', '--month', args.month, '--db', args.db]
+            sal_file = sources.get('salary')
+            att_file = sources.get('attendance')
+            if sal_file and os.path.isfile(sal_file):
+                cmd.extend(['--salary-file', sal_file])
+            if att_file and os.path.isfile(att_file):
+                cmd.extend(['--attendance-file', att_file])
+            ok = run_script("薪资+考勤导入", cmd)
+            results.append(('薪资+考勤', ok))
+
+        # 1.2 账单查询原始数据（bill_records + bill_columns）
+        if target in (None, 'bill_records'):
+            bill_dir = sources.get('bill_records')
+            if bill_dir and os.path.exists(bill_dir):
+                ok = run_script("账单查询导入(bill_records)", [
+                    py, 'import_bill_records.py',
+                    '--month', args.month,
+                    '--file', bill_dir,
+                    '--db', args.db,
+                ])
+                results.append(('bill_records', ok))
+            else:
+                print(f"\n  跳过 bill_records: 路径不存在 {bill_dir}")
+                results.append(('bill_records', False))
+
+        # 1.3 月度业务数据（库存/销售/成本/费用/中央厨房/配送）
+        if target in (None, 'monthly'):
+            monthly_datasets = [
+                ('inventory', 'inventory', None),
+                ('dish_sales', 'dish_sales', None),
+                ('dish_cost', 'dish_cost', None),
+                ('operating_expense', 'operating_expense', None),
+                ('central_kitchen_finished', 'central_kitchen', 'finished'),
+                ('central_kitchen_recipe', 'central_kitchen', 'recipe'),
+                ('central_kitchen_material', 'central_kitchen', 'material'),
+                ('central_kitchen_processing', 'central_kitchen', 'processing'),
+                ('distribution', 'distribution', None),
+                ('distribution_2', 'distribution', None),
+            ]
+
+            for src_key, dataset, ck_type in monthly_datasets:
+                filepath = sources.get(src_key)
+                if not filepath or not os.path.exists(filepath):
+                    print(f"\n  跳过 {dataset}/{ck_type or ''}: 文件不存在 {filepath}")
+                    results.append((f'{dataset}/{ck_type or src_key}', False))
+                    continue
+
+                cmd = [
+                    py, 'import_monthly_data.py',
+                    '--dataset', dataset,
+                    '--month', args.month,
+                    '--file', filepath,
+                    '--db', args.db,
+                ]
+                if ck_type:
+                    cmd.extend(['--ck-type', ck_type])
+
+                label = f"月度数据-{dataset}" + (f"-{ck_type}" if ck_type else f"({os.path.basename(filepath)})")
+                ok = run_script(label, cmd)
+                results.append((label, ok))
+
+        # 1.4 门店位置与映射表
+        if target in (None, 'store_location'):
+            store_file = sources.get('store_location')
+            if store_file and os.path.exists(store_file):
+                ok = run_script("门店位置与映射表导入", [
+                    py, 'import_store_location.py',
+                    '--file', store_file,
+                    '--db', args.db,
+                ])
+                results.append(('门店位置', ok))
+            else:
+                print(f"\n  跳过门店位置: 文件不存在 {store_file}")
+                results.append(('门店位置', False))
+
+        # 1.5 派生分析数据
+        if target in (None, 'derived'):
+            diag_date = args.diagnosis_date or ''
+            cmd = [
+                py, 'import_derived_data.py',
+                '--month', args.month,
+                '--db', args.db,
+            ]
+            if diag_date:
+                cmd.extend(['--diagnosis-date', diag_date])
+            ok = run_script("派生分析数据生成", cmd)
+            results.append(('派生数据', ok))
+
+    # ============================================================
+    # Step 2: 物化视图刷新
+    # ============================================================
+    if not args.skip_views:
+        ok = run_script("物化视图刷新", [
+            py, 'refresh_materialized_views.py',
+            '--db', args.db,
+        ], cwd=PIPELINE_DIR)
+        results.append(('物化视图', ok))
+
+    # ============================================================
+    # Step 3: 验证
+    # ============================================================
+    ok = run_script("数据一致性验证", [
+        py, 'verify.py',
+        '--db', args.db,
+    ], cwd=PIPELINE_DIR)
+    results.append(('验证', ok))
+
+    # ============================================================
+    # 汇总
+    # ============================================================
+    print(f"\n{'#'*60}")
+    print(f"  流水线执行汇总")
+    print(f"{'#'*60}")
+    all_ok = True
+    for name, ok in results:
+        status = "✓ 成功" if ok else "✗ 失败"
+        print(f"  {status}  {name}")
+        if not ok:
+            all_ok = False
+
+    if all_ok:
+        print(f"\n  全部完成!")
+        sys.exit(0)
+    else:
+        print(f"\n  部分失败，请检查上方日志")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/verify.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""
+测试库与正式库行数对比验证
+用法: python3 verify.py --db bill_query_test [--prod bill_query]
+"""
+import argparse
+import psycopg2
+import sys
+
+from config import DB_CONFIG
+
+
+def get_table_counts(dbname):
+    conn = psycopg2.connect(
+        host=DB_CONFIG['host'],
+        port=DB_CONFIG['port'],
+        dbname=dbname,
+        user=DB_CONFIG['user'],
+        password=DB_CONFIG['password'] or None,
+    )
+    cur = conn.cursor()
+    cur.execute("""
+        SELECT table_name
+        FROM information_schema.tables
+        WHERE table_schema = 'public'
+          AND table_type = 'BASE TABLE'
+        ORDER BY table_name
+    """)
+    tables = [r[0] for r in cur.fetchall()]
+
+    counts = {}
+    for tbl in tables:
+        cur.execute(f"SELECT count(*) FROM public.{tbl}")
+        counts[tbl] = cur.fetchone()[0]
+
+    # 物化视图行数
+    cur.execute("""
+        SELECT matviewname
+        FROM pg_matviews
+        WHERE schemaname = 'analytics'
+        ORDER BY matviewname
+    """)
+    mvs = [r[0] for r in cur.fetchall()]
+    mv_counts = {}
+    for mv in mvs:
+        try:
+            cur.execute(f'SELECT count(*) FROM analytics."{mv}"')
+            mv_counts[mv] = cur.fetchone()[0]
+        except Exception:
+            mv_counts[mv] = -1
+            conn.rollback()
+
+    cur.close()
+    conn.close()
+    return counts, mv_counts
+
+
+def main():
+    parser = argparse.ArgumentParser(description='测试库与正式库数据一致性验证')
+    parser.add_argument('--db', default='bill_query_test', help='测试库名')
+    parser.add_argument('--prod', default='bill_query', help='正式库名')
+    args = parser.parse_args()
+
+    print(f"\n=== 数据一致性验证 ===")
+    print(f"正式库: {args.prod}")
+    print(f"测试库: {args.db}")
+
+    prod_counts, prod_mv = get_table_counts(args.prod)
+    test_counts, test_mv = get_table_counts(args.db)
+
+    # 基础表对比
+    print(f"\n--- 基础表 ({len(prod_counts)} 张) ---")
+    ok = 0
+    diff = 0
+    for tbl in sorted(prod_counts.keys()):
+        p = prod_counts[tbl]
+        t = test_counts.get(tbl, 'MISSING')
+        flag = 'OK' if t == p else 'DIFF'
+        if flag == 'OK':
+            ok += 1
+        else:
+            diff += 1
+        if flag != 'OK' or True:  # 显示全部
+            print(f"  {flag:4s} {tbl:45s} prod={p:<12s} test={t}")
+
+    print(f"\n  基础表: {ok} OK, {diff} DIFF")
+
+    # 物化视图对比
+    print(f"\n--- 物化视图 ({len(prod_mv)} 个) ---")
+    mv_ok = 0
+    mv_diff = 0
+    for mv in sorted(prod_mv.keys()):
+        p = prod_mv[mv]
+        t = test_mv.get(mv, 'MISSING')
+        flag = 'OK' if t == p else 'DIFF'
+        if flag == 'OK':
+            mv_ok += 1
+        else:
+            mv_diff += 1
+        print(f"  {flag:4s} {mv:45s} prod={p:<12s} test={t}")
+
+    print(f"\n  物化视图: {mv_ok} OK, {mv_diff} DIFF")
+    print(f"\n=== 总计: {ok + mv_ok} OK, {diff + mv_diff} DIFF ===")
+
+    if diff + mv_diff > 0:
+        sys.exit(1)
+    sys.exit(0)
+
+
+if __name__ == '__main__':
+    main()

db/pipeline/verify_import.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+对比验证脚本：比较测试库与正式库的4月数据
+用法: python3 verify_import.py
+"""
+import psycopg2
+
+def connect(dbname):
+    return psycopg2.connect(host='localhost', port=5432, dbname=dbname, user='freedak')
+
+def compare_table(test_cur, prod_cur, table_name, where_clause='1=1', key_cols=None, sum_cols=None):
+    """对比两个库中同一表的数据"""
+    print(f"\n--- {table_name} ---")
+    
+    # Row count
+    test_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}")
+    test_count = test_cur.fetchone()[0]
+    prod_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}")
+    prod_count = prod_cur.fetchone()[0]
+    
+    match = "✅" if test_count == prod_count else "❌"
+    print(f"  行数: 测试库={test_count}, 正式库={prod_count} {match}")
+    
+    # Sum comparison
+    if sum_cols:
+        for col in sum_cols:
+            test_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}")
+            test_sum = test_cur.fetchone()[0]
+            prod_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}")
+            prod_sum = prod_cur.fetchone()[0]
+            match = "✅" if abs(float(test_sum or 0) - float(prod_sum or 0)) < 0.01 else "❌"
+            print(f"  {col}合计: 测试库={test_sum}, 正式库={prod_sum} {match}")
+    
+    return test_count == prod_count
+
+def main():
+    test_conn = connect('bill_query_test')
+    prod_conn = connect('bill_query')
+    test_cur = test_conn.cursor()
+    prod_cur = prod_conn.cursor()
+    
+    print("=" * 60)
+    print("  数据对比验证: 测试库 vs 正式库 (4月数据)")
+    print("=" * 60)
+    
+    results = []
+    
+    # 1. 库存成本
+    results.append(compare_table(
+        test_cur, prod_cur, 'inventory_cost_records',
+        where_clause="report_month = '2026-04-01'",
+        sum_cols=['consumption_amount', 'opening_amount', 'ending_amount', 'purchase_amount']
+    ))
+    
+    # 2. 菜品销售明细 (测试库只导入了1个文件，按文件对比)
+    test_cur.execute("SELECT DISTINCT source_file FROM public.dish_sales_details")
+    test_files = [r[0] for r in test_cur.fetchall()]
+    for sf in test_files:
+        results.append(compare_table(
+            test_cur, prod_cur, 'dish_sales_details',
+            where_clause=f"source_file = '{sf}'",
+            sum_cols=['gross_amount', 'received_amount']
+        ))
+    
+    # 3. 菜品成本
+    results.append(compare_table(
+        test_cur, prod_cur, 'dish_cost_analysis_summary',
+        where_clause="1=1",
+        sum_cols=['theoretical_cost', 'actual_cost', 'sales_amount']
+    ))
+    # 3b. 菜品成本原料明细
+    results.append(compare_table(
+        test_cur, prod_cur, 'dish_cost_analysis_material_detail',
+        where_clause="1=1",
+        sum_cols=['theoretical_amount', 'actual_amount']
+    ))
+    
+    # 4. 营业费用
+    results.append(compare_table(
+        test_cur, prod_cur, 'operating_expense_records',
+        where_clause="report_month = '2026-04-01'",
+        sum_cols=['amount']
+    ))
+    
+    # 5. 中央厨房
+    ck_tables = {
+        'central_kitchen_finished_receipt': ('receipt_date', ['amount', 'quantity']),
+        'central_kitchen_recipe_consumption': ('business_date', ['theoretical_quantity', 'theoretical_amount', 'issue_quantity', 'issue_amount']),
+        'central_kitchen_material_daily': ('business_date', ['amount']),
+        'central_kitchen_processing_cost': ('report_month', ['theoretical_cost', 'actual_cost']),
+    }
+    for tbl, (date_col, sums) in ck_tables.items():
+        # Check which sum cols actually exist
+        test_cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{tbl}'")
+        actual_cols = [r[0] for r in test_cur.fetchall()]
+        valid_sums = [c for c in sums if c in actual_cols]
+        if date_col == 'report_month':
+            where = f"{date_col} = '2026-04-01'"
+        else:
+            where = f"{date_col} >= '2026-04-01' AND {date_col} < '2026-05-01'"
+        results.append(compare_table(
+            test_cur, prod_cur, tbl,
+            where_clause=where,
+            sum_cols=valid_sums if valid_sums else None
+        ))
+    
+    # 6. 配送明细
+    results.append(compare_table(
+        test_cur, prod_cur, 'distribution_detail_records',
+        where_clause="report_month = '2026-04-01'",
+        sum_cols=['cost_total_amount', 'outbound_total_amount', 'total_quantity']
+    ))
+    
+    # 7. 薪资
+    results.append(compare_table(
+        test_cur, prod_cur, 'salary_detail_records',
+        where_clause="1=1",
+        sum_cols=['net_pay', 'gross_pay']
+    ))
+    
+    # 8. 考勤
+    results.append(compare_table(
+        test_cur, prod_cur, 'attendance_records',
+        where_clause="1=1",
+        sum_cols=None
+    ))
+    
+    # Summary
+    print("\n" + "=" * 60)
+    passed = sum(results)
+    total = len(results)
+    print(f"  通过: {passed}/{total}")
+    if passed == total:
+        print("  ✅ 所有对比通过")
+    else:
+        print(f"  ❌ {total - passed} 项对比失败")
+    print("=" * 60)
+    
+    test_cur.close()
+    prod_cur.close()
+    test_conn.close()
+    prod_conn.close()
+
+if __name__ == '__main__':
+    main()