feat: 新增db/pipeline数据导入管线，适配5月数据格式
- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
This commit is contained in:
freedakgmail
@@ -18,3 +18,4 @@ backups/
|
||||
.playwright-cli/
|
||||
.tmp/
|
||||
数据/
|
||||
__pycache__/
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
# 数据导入与物化刷新流水线
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
db/pipeline/
|
||||
├── README.md ← 本文件
|
||||
├── config.py ← 数据源路径与数据库配置
|
||||
├── run_all.py ← 一键全量导入+物化刷新
|
||||
├── refresh_materialized_views.py ← 物化视图按依赖顺序刷新
|
||||
├── verify.py ← 测试库与正式库行数对比验证
|
||||
├── verify_import.py ← 详细数据对比验证
|
||||
├── import_bill_records.py ← 账单查询原始数据导入（4月格式，196列宽表）
|
||||
├── import_bill_records_may.py ← 5月专用账单导入（51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details）
|
||||
├── import_monthly_data.py ← 月度业务数据导入（库存/销售/成本/费用/中央厨房/配送）
|
||||
├── import_salary_attendance.py ← 薪资考勤数据导入
|
||||
├── import_store_location.py ← 门店位置与映射表导入
|
||||
└── import_derived_data.py ← 派生分析数据生成
|
||||
```
|
||||
|
||||
## 导入脚本
|
||||
|
||||
| 脚本 | 数据表 | 数据源 |
|
||||
|------|--------|--------|
|
||||
| `import_bill_records.py` | bill_records, bill_columns | 账单查询 Excel 目录（17个文件） |
|
||||
| `import_monthly_data.py` | inventory_cost_records, dish_sales_details, dish_cost_analysis_*, operating_expense_records, central_kitchen_*, distribution_detail_records | 各业务 Excel 文件 |
|
||||
| `import_salary_attendance.py` | salary_detail_records, attendance_records | 薪资/考勤 Excel |
|
||||
| `import_store_location.py` | store_location_master, store_location_source_rows, sales_store_location_mapping, store_name_mapping, inventory_store_mapping, operating_expense_store_mapping | 各店信息 Excel |
|
||||
| `import_derived_data.py` | dish_diagnosis_snapshot, central_kitchen_manufacturing_cost_pool | 从已有表派生 |
|
||||
|
||||
## 物化视图刷新顺序（按依赖分层）
|
||||
|
||||
| 层级 | 物化视图 | 依赖 |
|
||||
|------|----------|------|
|
||||
| L0 | bill_fact | 基础表 |
|
||||
| L1 | mv_daily_revenue, mv_dish_basket_monthly, mv_dish_pair_summary_monthly, mv_dish_sku_summary_monthly, mv_dish_store_summary_monthly, mv_district_site_benchmark_monthly, mv_inventory_cost_classified_monthly, mv_site_segment_benchmark_monthly, mv_store_*_monthly (12个), mv_store_scorecard | bill_fact |
|
||||
| L1 | dish_sales_april | bill_fact |
|
||||
| L2 | dish_basket_april, dish_category_summary_april, dish_member_sku_april, dish_sku_summary_april, dish_store_sku_april | dish_sales_april |
|
||||
| L2 | mv_dish_sku_abc_monthly | mv_dish_sku_summary_monthly |
|
||||
| L2 | v_store_benchmark, v_store_category_mix, v_store_platform_economics | bill_fact |
|
||||
| L3 | dish_store_summary_april | dish_basket_april |
|
||||
| L3 | v_store_action_list | v_store_benchmark, v_store_category_mix, v_store_platform_economics |
|
||||
| L3 | mv_store_action_priority_deep_april, mv_store_theoretical_actual_cost_april | bill_fact / dish_store_summary_april |
|
||||
| L4 | v_store_execution_priority | v_store_action_list |
|
||||
|
||||
## 使用方法
|
||||
|
||||
### 一键全量导入
|
||||
```bash
|
||||
python3 db/pipeline/run_all.py --month 2026-04-01 --db bill_query_test
|
||||
```
|
||||
|
||||
### 仅刷新物化视图
|
||||
```bash
|
||||
python3 db/pipeline/refresh_materialized_views.py --db bill_query_test
|
||||
```
|
||||
|
||||
### 验证数据一致性
|
||||
```bash
|
||||
python3 db/pipeline/verify.py --db bill_query_test
|
||||
```
|
||||
|
||||
### 单独导入某类数据
|
||||
参见各脚本 `--help`，例如：
|
||||
```bash
|
||||
python3 db/pipeline/import_monthly_data.py --dataset inventory --month 2026-04-01 --file <path> --db bill_query_test
|
||||
```
|
||||
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据导入流水线配置
|
||||
统一管理数据源路径、数据库连接参数、导入脚本路径
|
||||
"""
|
||||
import os
|
||||
import glob
|
||||
import sys
|
||||
|
||||
# ============================================================
|
||||
# 路径常量
|
||||
# ============================================================
|
||||
PIPELINE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
DB_DIR = os.path.dirname(PIPELINE_DIR)
|
||||
BASE_DIR = os.path.dirname(DB_DIR)
|
||||
|
||||
# 数据源根目录（西部马华数据分析 下有完整的业务数据）
|
||||
DATA_ROOT = os.path.join(os.path.dirname(BASE_DIR), '西部马华数据分析')
|
||||
# SBrainCO3 数据目录（部分文件只在这里有）
|
||||
DATA_SBRAIN = os.path.join(BASE_DIR, '数据')
|
||||
|
||||
# ============================================================
|
||||
# 数据库配置
|
||||
# ============================================================
|
||||
DB_CONFIG = {
|
||||
'host': 'localhost',
|
||||
'port': 5432,
|
||||
'user': 'freedak',
|
||||
'password': '',
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# 月份配置
|
||||
# ============================================================
|
||||
MONTH_CONFIG = {
|
||||
'2026-04-01': {
|
||||
'month_dir': '4月',
|
||||
'date_range': '2026-04-01--2026-04-30',
|
||||
'dist_half1': '2026年4月1-15日',
|
||||
'dist_half2': '2026年4月16-30日',
|
||||
'inventory_file': '4.13盘点倒挤成本报表2026年4月明细.xlsx',
|
||||
'dish_cost_file': '菜品成本分析报表.xlsx',
|
||||
'expense_file': '2026年4月营业费用分析.xls',
|
||||
'ck_subdir': '中央厨房4月',
|
||||
'salary_file': '薪资拆分明细表_脱敏.xlsx',
|
||||
'attendance_file': '考勤数据表-脱敏.xlsx',
|
||||
'store_info_file': '各店信息新（20260508）.xls',
|
||||
'bill_records_subdir': '账单查询',
|
||||
'dish_sales_subdir': '账单查询-菜品销售明细表',
|
||||
'data_subdirs': {}, # 4月文件平铺在月份目录下
|
||||
},
|
||||
'2026-05-01': {
|
||||
'month_dir': '5月',
|
||||
'date_range': '2026-05-01--2026-05-31',
|
||||
'dist_half1': '2026年5月1-15日',
|
||||
'dist_half2': '2026年5月16-31日',
|
||||
'inventory_file': '4.13盘点倒挤成本报表2026年5月.xlsx',
|
||||
'dish_cost_file': '2026年5月菜品成本分析报表.xlsx',
|
||||
'expense_file': '2026年5月营业费用分析.xls',
|
||||
'ck_subdir': '中央厨房',
|
||||
'salary_file': '2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx',
|
||||
'attendance_file': '2026年5月考勤数据表.xlsx',
|
||||
'store_info_file': None, # 5月无新各店信息文件，复用4月
|
||||
'bill_records_subdir': None, # 5月无独立账单查询目录
|
||||
'dish_sales_subdir': '账单查询-菜品销售明细表',
|
||||
'data_subdirs': { # 5月文件在子目录中
|
||||
'inventory': '供应链数据',
|
||||
'dish_cost': '营业数据',
|
||||
'operating_expense': '',
|
||||
'distribution': '供应链数据',
|
||||
'salary': '营业数据',
|
||||
'attendance': '营业数据',
|
||||
'central_kitchen': '中央厨房',
|
||||
'dish_sales': '营业数据',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _find_file(base_dirs, pattern):
|
||||
"""在多个候选目录中查找文件"""
|
||||
for d in base_dirs:
|
||||
if not d:
|
||||
continue
|
||||
path = os.path.join(d, pattern)
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
matches = glob.glob(os.path.join(d, pattern))
|
||||
if matches:
|
||||
return matches[0]
|
||||
return None
|
||||
|
||||
|
||||
def get_data_sources(month):
|
||||
"""
|
||||
根据月份返回数据源路径字典
|
||||
month: 'YYYY-MM-01' 格式
|
||||
"""
|
||||
cfg = MONTH_CONFIG.get(month)
|
||||
if not cfg:
|
||||
print(f"警告: 未知月份 {month}，请在 config.py MONTH_CONFIG 中添加配置")
|
||||
return {}
|
||||
|
||||
month_dir = cfg['month_dir']
|
||||
data_dir = os.path.join(DATA_ROOT, month_dir)
|
||||
sbrain_data = os.path.join(DATA_SBRAIN, month_dir)
|
||||
subdirs = cfg.get('data_subdirs', {})
|
||||
|
||||
sources = {}
|
||||
|
||||
def resolve(subdir_key, filename):
|
||||
"""在 data_dir 和 sbrain_data 中按子目录结构查找文件"""
|
||||
sub = subdirs.get(subdir_key, '')
|
||||
candidates = []
|
||||
if sub:
|
||||
candidates.append(os.path.join(sbrain_data, sub, filename))
|
||||
candidates.append(os.path.join(data_dir, sub, filename))
|
||||
candidates.append(os.path.join(sbrain_data, filename))
|
||||
candidates.append(os.path.join(data_dir, filename))
|
||||
for c in candidates:
|
||||
if os.path.isfile(c):
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
def resolve_dir(subdir_key, dirname):
|
||||
"""在 data_dir 和 sbrain_data 中按子目录结构查找目录"""
|
||||
sub = subdirs.get(subdir_key, '')
|
||||
candidates = []
|
||||
if sub:
|
||||
candidates.append(os.path.join(sbrain_data, sub, dirname))
|
||||
candidates.append(os.path.join(data_dir, sub, dirname))
|
||||
candidates.append(os.path.join(sbrain_data, dirname))
|
||||
candidates.append(os.path.join(data_dir, dirname))
|
||||
for c in candidates:
|
||||
if os.path.isdir(c):
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
# 1. 账单查询（bill_records）- 目录
|
||||
bill_subdir = cfg.get('bill_records_subdir')
|
||||
if bill_subdir:
|
||||
sources['bill_records'] = resolve_dir('bill_records', bill_subdir)
|
||||
else:
|
||||
sources['bill_records'] = None
|
||||
|
||||
# 2. 菜品销售明细 - 目录
|
||||
sources['dish_sales'] = resolve_dir('dish_sales', cfg['dish_sales_subdir'])
|
||||
|
||||
# 3. 库存成本
|
||||
sources['inventory'] = resolve('inventory', cfg['inventory_file'])
|
||||
|
||||
# 4. 菜品成本BOM
|
||||
sources['dish_cost'] = resolve('dish_cost', cfg['dish_cost_file'])
|
||||
|
||||
# 5. 营业费用
|
||||
sources['operating_expense'] = resolve('operating_expense', cfg['expense_file'])
|
||||
|
||||
# 6. 中央厨房（4个子文件）
|
||||
ck_subdir = cfg['ck_subdir']
|
||||
ck_dir_sbrain = os.path.join(sbrain_data, ck_subdir)
|
||||
ck_dir_data = os.path.join(data_dir, ck_subdir)
|
||||
ck_dir = ck_dir_sbrain if os.path.isdir(ck_dir_sbrain) else ck_dir_data
|
||||
date_range = cfg['date_range']
|
||||
sources['central_kitchen_finished'] = _find_file(
|
||||
[ck_dir], f'{date_range} 23 59 59完工入库统计分析报表.xlsx'
|
||||
) or os.path.join(ck_dir, f'{date_range} 23 59 59完工入库统计分析报表.xlsx')
|
||||
sources['central_kitchen_recipe'] = _find_file(
|
||||
[ck_dir], f'按配方导出-货品实际与理论耗用({date_range}).xlsx'
|
||||
) or os.path.join(ck_dir, f'按配方导出-货品实际与理论耗用({date_range}).xlsx')
|
||||
sources['central_kitchen_material'] = _find_file(
|
||||
[ck_dir], f'按货品导出-货品实际与理论耗用({date_range}).xlsx'
|
||||
) or os.path.join(ck_dir, f'按货品导出-货品实际与理论耗用({date_range}).xlsx')
|
||||
sources['central_kitchen_processing'] = _find_file(
|
||||
[ck_dir], '加工单价分析报表.xlsx'
|
||||
) or os.path.join(ck_dir, '加工单价分析报表.xlsx')
|
||||
|
||||
# 7. 配送明细（2个文件）
|
||||
sources['distribution'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half1"]}.xlsx')
|
||||
sources['distribution_2'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half2"]}.xlsx')
|
||||
|
||||
# 8. 薪资+考勤
|
||||
sources['salary'] = resolve('salary', cfg['salary_file'])
|
||||
sources['attendance'] = resolve('attendance', cfg['attendance_file'])
|
||||
|
||||
# 9. 门店位置
|
||||
store_info_file = cfg.get('store_info_file')
|
||||
if store_info_file:
|
||||
sources['store_location'] = resolve('', store_info_file)
|
||||
else:
|
||||
# 5月无新文件，复用4月
|
||||
for p in [
|
||||
os.path.join(DATA_SBRAIN, '4月', '各店信息新（20260508）.xls'),
|
||||
os.path.join(DATA_ROOT, '4月', '各店信息新（20260508）.xls'),
|
||||
]:
|
||||
if os.path.isfile(p):
|
||||
sources['store_location'] = p
|
||||
break
|
||||
else:
|
||||
sources['store_location'] = None
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 物化视图刷新顺序（按依赖分层，同层可并行）
|
||||
# ============================================================
|
||||
MATVIEW_LAYERS = [
|
||||
# L0: 基础物化视图，依赖基础表
|
||||
['bill_fact'],
|
||||
|
||||
# L1: 依赖 bill_fact 或基础表
|
||||
[
|
||||
'mv_daily_revenue',
|
||||
'mv_dish_basket_monthly',
|
||||
'mv_dish_pair_summary_monthly',
|
||||
'mv_dish_sku_summary_monthly',
|
||||
'mv_dish_store_summary_monthly',
|
||||
'mv_district_site_benchmark_monthly',
|
||||
'mv_inventory_cost_classified_monthly',
|
||||
'mv_site_segment_benchmark_monthly',
|
||||
'mv_store_action_priority_deep_monthly',
|
||||
'mv_store_area_efficiency_monthly',
|
||||
'mv_store_benchmark_composite_monthly',
|
||||
'mv_store_category_mix_monthly',
|
||||
'mv_store_deep_diagnosis_monthly',
|
||||
'mv_store_member_opportunity_monthly',
|
||||
'mv_store_overlap_risk_monthly',
|
||||
'mv_store_platform_economics_monthly',
|
||||
'mv_store_repeat_summary_monthly',
|
||||
'mv_store_risk_rating_monthly',
|
||||
'mv_store_scorecard',
|
||||
'mv_store_site_profile_monthly',
|
||||
'mv_store_site_replication_monthly',
|
||||
'mv_store_theoretical_actual_cost_monthly',
|
||||
'dish_sales_april',
|
||||
'v_store_benchmark',
|
||||
'v_store_category_mix',
|
||||
'v_store_platform_economics',
|
||||
],
|
||||
|
||||
# L2: 依赖 L1 物化视图
|
||||
[
|
||||
'dish_basket_april',
|
||||
'dish_category_summary_april',
|
||||
'dish_member_sku_april',
|
||||
'dish_sku_summary_april',
|
||||
'dish_store_sku_april',
|
||||
'mv_dish_sku_abc_monthly',
|
||||
'v_store_action_list',
|
||||
],
|
||||
|
||||
# L3: 依赖 L2
|
||||
[
|
||||
'dish_store_summary_april',
|
||||
'mv_store_action_priority_deep_april',
|
||||
'mv_store_theoretical_actual_cost_april',
|
||||
'v_store_execution_priority',
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
def get_dbname():
|
||||
"""获取默认数据库名"""
|
||||
return 'bill_query_test'
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
账单查询原始数据导入脚本
|
||||
从Excel文件导入bill_records和bill_columns表。
|
||||
|
||||
用法:
|
||||
python3 import_bill_records.py --month 2026-04-01 --file <文件或目录> --db bill_query_test
|
||||
python3 import_bill_records.py --month 2026-04-01 --file <目录> --db bill_query_test --skip-columns
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import psycopg2
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
|
||||
def file_sha256(filepath):
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(8192), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def to_text(val):
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if s.lower() in ('nan', 'none', ''):
|
||||
return None
|
||||
return s
|
||||
|
||||
|
||||
def import_bill_columns(conn, filepaths):
|
||||
"""从第一个Excel文件提取列定义，导入bill_columns表"""
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT count(*) FROM public.bill_columns")
|
||||
existing = cur.fetchone()[0]
|
||||
if existing > 0:
|
||||
print(f"bill_columns已有{existing}行，跳过列定义导入")
|
||||
return
|
||||
|
||||
if not filepaths:
|
||||
print("无文件可用，跳过列定义导入")
|
||||
return
|
||||
|
||||
fpath = filepaths[0]
|
||||
df = pd.read_excel(fpath, header=None, nrows=2, engine='openpyxl')
|
||||
main_headers = df.iloc[0].tolist()
|
||||
sub_headers = df.iloc[1].tolist()
|
||||
|
||||
columns = []
|
||||
for i, (mh, sh) in enumerate(zip(main_headers, sub_headers)):
|
||||
col_name = f"c{i+1:03d}"
|
||||
group_h = to_text(mh) or ''
|
||||
sub_h = to_text(sh) or ''
|
||||
# If group and sub are the same, only keep group
|
||||
if group_h and sub_h and group_h == sub_h:
|
||||
sub_h = ''
|
||||
columns.append((col_name, i + 1, group_h, sub_h))
|
||||
|
||||
execute_values(
|
||||
cur,
|
||||
"INSERT INTO public.bill_columns (column_name, excel_position, group_header, sub_header) VALUES %s",
|
||||
columns,
|
||||
page_size=500
|
||||
)
|
||||
conn.commit()
|
||||
print(f"bill_columns导入完成: {len(columns)} 列")
|
||||
cur.close()
|
||||
|
||||
|
||||
def import_bill_records(conn, filepath, report_month):
|
||||
"""导入单个Excel文件的账单数据到bill_records"""
|
||||
fname = os.path.basename(filepath)
|
||||
sha = file_sha256(filepath)
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# 幂等检查
|
||||
cur.execute(
|
||||
"SELECT 1 FROM public.bill_records WHERE source_file = %s LIMIT 1",
|
||||
(fname,)
|
||||
)
|
||||
if cur.fetchone():
|
||||
print(f" {fname} 已导入过，跳过")
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
# 读取Excel：row0=主表头, row1=子表头, 数据从row2开始
|
||||
df = pd.read_excel(filepath, header=None, engine='openpyxl')
|
||||
# 跳过前2行（表头）
|
||||
data_df = df.iloc[2:]
|
||||
# 过滤空行（第一列为空）
|
||||
data_df = data_df.dropna(subset=[data_df.columns[0]])
|
||||
data_rows = data_df.values.tolist()
|
||||
|
||||
print(f" 数据行数: {len(data_rows)}, 列数: {data_df.shape[1]}")
|
||||
|
||||
if not data_rows:
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
# 获取DB列名 (c001 ~ c196)
|
||||
num_cols = data_df.shape[1]
|
||||
col_names = [f"c{i+1:03d}" for i in range(num_cols)]
|
||||
all_cols = ['source_file', 'source_row'] + col_names
|
||||
col_str = ','.join(all_cols)
|
||||
|
||||
batch = []
|
||||
for idx, row in enumerate(data_rows):
|
||||
source_row = idx + 3 # 1-indexed, data starts at Excel row 3
|
||||
vals = [fname, source_row]
|
||||
for i in range(num_cols):
|
||||
v = row[i] if i < len(row) else None
|
||||
vals.append(to_text(v))
|
||||
batch.append(tuple(vals))
|
||||
if len(batch) >= 500:
|
||||
execute_values(
|
||||
cur,
|
||||
f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
|
||||
batch,
|
||||
page_size=500
|
||||
)
|
||||
conn.commit()
|
||||
if (idx + 1) % 10000 == 0:
|
||||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
execute_values(
|
||||
cur,
|
||||
f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
|
||||
batch,
|
||||
page_size=500
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||||
cur.close()
|
||||
return len(data_rows)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='账单查询原始数据导入')
|
||||
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
||||
parser.add_argument('--file', required=True, help='Excel文件路径或目录')
|
||||
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
||||
parser.add_argument('--skip-columns', action='store_true', help='跳过bill_columns导入')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
|
||||
conn.autocommit = False
|
||||
|
||||
# 收集文件
|
||||
files = []
|
||||
if os.path.isdir(args.file):
|
||||
for f in sorted(os.listdir(args.file)):
|
||||
if f.endswith('.xlsx') and not f.startswith('.'):
|
||||
files.append(os.path.join(args.file, f))
|
||||
else:
|
||||
files.append(args.file)
|
||||
|
||||
print(f"\n=== 账单查询原始数据导入 ===")
|
||||
print(f"文件数: {len(files)}")
|
||||
print(f"数据库: {args.db}")
|
||||
|
||||
try:
|
||||
# 导入列定义
|
||||
if not args.skip_columns:
|
||||
import_bill_columns(conn, files)
|
||||
|
||||
# 导入数据
|
||||
total = 0
|
||||
for idx, fpath in enumerate(files):
|
||||
fname = os.path.basename(fpath)
|
||||
print(f"\n[{idx+1}/{len(files)}] {fname}")
|
||||
total += import_bill_records(conn, fpath, args.month)
|
||||
|
||||
print(f"\n账单数据导入完成: 共 {total} 行")
|
||||
print("=== 导入完成 ===")
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,549 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
5月账单数据导入脚本
|
||||
将5月格式的"正餐事业部_全渠道订单明细"(51列)导入bill_records(196列结构)，
|
||||
将"正餐事业部_品项销售明细"(50列)导入dish_sales_details。
|
||||
|
||||
5月数据格式与4月不同：
|
||||
- 4月: 账单查询196列宽表 + 菜品销售明细表(多文件)
|
||||
- 5月: 全渠道订单明细51列 + 品项销售明细50列 (各1个文件)
|
||||
|
||||
用法:
|
||||
python3 import_bill_records_may.py --month 2026-05-01 \
|
||||
--orders <全渠道订单明细.xlsx> \
|
||||
--items <品项销售明细.xlsx> \
|
||||
--db bill_query_test
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import psycopg2
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
|
||||
DB_CONFIG = {
|
||||
'host': 'localhost',
|
||||
'port': 5432,
|
||||
'user': 'freedak',
|
||||
'password': '',
|
||||
}
|
||||
|
||||
|
||||
def file_sha256(filepath):
|
||||
h = hashlib.sha256()
|
||||
with open(filepath, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(8192), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def to_text(val):
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if s.lower() in ('nan', 'none', '', '--'):
|
||||
return None
|
||||
return s
|
||||
|
||||
|
||||
def to_num(val):
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
return None
|
||||
s = str(val).strip().replace(',', '').replace('¥', '').replace('￥', '')
|
||||
if s.lower() in ('nan', 'none', '', '--'):
|
||||
return None
|
||||
try:
|
||||
return str(float(s))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def to_timestamp_str(val):
|
||||
"""将日期/时间值转为字符串格式 YYYY/MM/DD HH:MM:SS"""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
return None
|
||||
if isinstance(val, pd.Timestamp):
|
||||
return val.strftime('%Y/%m/%d %H:%M:%S')
|
||||
s = str(val).strip()
|
||||
if s.lower() in ('nan', 'none', '', '--'):
|
||||
return None
|
||||
# 尝试解析各种日期格式
|
||||
for fmt in ['%Y/%m/%d %H:%M:%S', '%Y-%m-%d %H:%M:%S', '%Y/%m/%d', '%Y-%m-%d']:
|
||||
try:
|
||||
ts = pd.to_datetime(s, format=fmt)
|
||||
return ts.strftime('%Y/%m/%d %H:%M:%S')
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
# pandas通用解析
|
||||
try:
|
||||
ts = pd.to_datetime(s)
|
||||
return ts.strftime('%Y/%m/%d %H:%M:%S')
|
||||
except (ValueError, TypeError):
|
||||
return s
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 5月全渠道订单明细 → bill_records 列映射
|
||||
# 5月col索引 → bill_records cXXX
|
||||
# ============================================================
|
||||
ORDERS_COL_MAP = {
|
||||
# 5月col: (bill_records col_name, converter)
|
||||
0: None, # 省份 → 无对应
|
||||
1: None, # 城市 → 无对应
|
||||
2: 'c003', # 门店名称 → c003 store_name
|
||||
3: None, # 营业日期 → 无直接对应(bill_fact用closed_at)
|
||||
4: 'c004', # 餐段 → c004 meal_period
|
||||
5: 'c005', # 订单号 → c005 bill_no
|
||||
6: None, # 原单号
|
||||
7: None, # 外卖订单号
|
||||
8: None, # 全渠道流水号
|
||||
9: 'c008', # 取餐号 → c008 table_no
|
||||
10: None, # 桌牌号
|
||||
11: 'c006', # 桌台区域 → c006 business_area
|
||||
12: 'c009', # 订单金额 → c009 consumption
|
||||
13: None, # 顾客实付
|
||||
14: 'c114', # 订单收入 → c114 received_total
|
||||
15: 'c068', # 订单优惠 → c068 discount_total
|
||||
16: None, # 订单来源
|
||||
17: None, # 订单子来源
|
||||
18: None, # 经营模式
|
||||
19: None, # 用餐方式
|
||||
20: None, # 宴会类型
|
||||
21: 'c194', # 订单状态 → c194 bill_status
|
||||
22: None, # 退单标识
|
||||
23: 'c190', # 下单员 → c190 waiter
|
||||
24: 'c191', # 收银员 → c191 cashier
|
||||
25: None, # 桌台提成人
|
||||
26: None, # 销售员 → c192 (不在bill_fact中，但可填)
|
||||
27: 'c175', # 创建时间 → c175 opened_at
|
||||
28: 'c176', # 完成时间 → c176 closed_at
|
||||
29: None, # 接单时间
|
||||
30: None, # 下单制作时间
|
||||
31: None, # 预定用餐时间
|
||||
32: None, # 订单备注
|
||||
33: None, # 桌台备注
|
||||
34: 'c178', # 用餐人数 → c178 guest_count
|
||||
35: 'c179', # 席数 → c179 table_count
|
||||
36: None, # 是否会员
|
||||
37: None, # 会员姓名
|
||||
38: 'c185', # 会员卡号 → c185 member_id
|
||||
39: None, # 会员手机号
|
||||
40: None, # 下单人手机号
|
||||
41: None, # 发票
|
||||
42: None, # 菜品金额
|
||||
43: None, # 菜品收入
|
||||
44: None, # 退单时间
|
||||
45: None, # 取消时间
|
||||
46: None, # 外卖订单已退金额
|
||||
47: None, # 敏感操作
|
||||
48: None, # 结账方式 → 需要解析支付方式
|
||||
49: None, # 企业版订单
|
||||
50: None, # 企业版订单是否开发票
|
||||
}
|
||||
|
||||
# 结账方式解析 → bill_records支付方式列
|
||||
PAYMENT_PATTERNS = {
|
||||
'c140': [r'现金'], # 现金
|
||||
'c143': [r'支付宝'], # 支付宝支付实收
|
||||
'c144': [r'微信'], # 微信支付实收
|
||||
'c145': [r'美团.*实收', r'大众点评.*实收', r'新美大实收', r'一键买单'], # 新美大实收
|
||||
'c146': [r'云闪付'], # 云闪付实收
|
||||
'c147': [r'抖音'], # 抖音券实收
|
||||
'c151': [r'美团外卖'], # 美团外卖实收
|
||||
'c152': [r'淘宝闪购', r'闪购'], # 淘宝闪购实收
|
||||
'c150': [r'京东外卖'], # 京东外卖实收
|
||||
'c149': [r'挂账'], # 挂账消费
|
||||
}
|
||||
|
||||
# 折扣/佣金解析
|
||||
DISCOUNT_PATTERNS = {
|
||||
'c087': [r'会员积分'], # 会员积分抵现
|
||||
'c088': [r'会员代金券'], # 会员代金券折扣
|
||||
'c089': [r'会员预存'], # 会员预存折扣
|
||||
'c095': [r'抖音.*折扣', r'抖音券折扣'], # 抖音券折扣
|
||||
'c097': [r'美团.*佣金'], # 美团外卖佣金
|
||||
'c098': [r'淘宝.*佣金', r'闪购.*佣金'], # 淘宝闪购佣金
|
||||
'c100': [r'京东.*佣金'], # 京东外卖佣金
|
||||
}
|
||||
|
||||
|
||||
def parse_payment(payment_str):
|
||||
"""解析结账方式字段，返回 {cXXX: amount_str}"""
|
||||
result = {}
|
||||
if not payment_str or payment_str == '--':
|
||||
return result
|
||||
|
||||
# 格式: "方式1 金额1,方式2 金额2" 或 "方式1 金额1"
|
||||
parts = payment_str.split(',')
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
# 提取金额（最后一个数字）
|
||||
amount_match = re.search(r'([\d.]+)\s*$', part)
|
||||
amount = amount_match.group(1) if amount_match else None
|
||||
|
||||
if not amount:
|
||||
continue
|
||||
|
||||
# 匹配支付方式
|
||||
for col, patterns in PAYMENT_PATTERNS.items():
|
||||
if col in result:
|
||||
continue
|
||||
for pat in patterns:
|
||||
if re.search(pat, part):
|
||||
if col in result:
|
||||
result[col] = str(float(result[col]) + float(amount))
|
||||
else:
|
||||
result[col] = amount
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def parse_discounts(discount_str):
|
||||
"""解析折扣字段"""
|
||||
return parse_payment(discount_str) # 同样的解析逻辑
|
||||
|
||||
|
||||
def build_store_code_map(items_filepath):
|
||||
"""从品项销售明细中构建 门店名称→机构编码 映射"""
|
||||
df = pd.read_excel(items_filepath, header=2, engine='openpyxl')
|
||||
df = df.dropna(subset=[df.columns[0]])
|
||||
|
||||
mapping = {}
|
||||
if '门店名称' in df.columns and '机构编码' in df.columns:
|
||||
for _, row in df[['门店名称', '机构编码']].drop_duplicates().iterrows():
|
||||
name = str(row['门店名称']).strip()
|
||||
code = str(row['机构编码']).strip()
|
||||
if name and code and name != 'nan' and code != 'nan' and code != '--':
|
||||
mapping[name] = code
|
||||
|
||||
print(f" 门店编码映射: {len(mapping)} 个门店")
|
||||
for name, code in sorted(mapping.items()):
|
||||
print(f" {code} → {name}")
|
||||
|
||||
return mapping
|
||||
|
||||
|
||||
def import_bill_records_may(conn, orders_filepath, store_code_map, report_month):
|
||||
"""将5月全渠道订单明细导入bill_records"""
|
||||
fname = os.path.basename(orders_filepath)
|
||||
sha = file_sha256(orders_filepath)
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# 幂等检查
|
||||
cur.execute(
|
||||
"SELECT 1 FROM public.bill_records WHERE source_file = %s LIMIT 1",
|
||||
(fname,)
|
||||
)
|
||||
if cur.fetchone():
|
||||
print(f" {fname} 已导入过，跳过")
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
# 读取Excel：row0=标题, row1=日期范围, row2=列头, row3+=数据
|
||||
df = pd.read_excel(orders_filepath, header=2, engine='openpyxl')
|
||||
df = df.dropna(subset=[df.columns[0]])
|
||||
data_rows = df.values.tolist()
|
||||
|
||||
print(f" 数据行数: {len(data_rows)}, 列数: {df.shape[1]}")
|
||||
if not data_rows:
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
# 构建196列的列名
|
||||
all_col_names = [f"c{i+1:03d}" for i in range(196)]
|
||||
insert_cols = ['source_file', 'source_row'] + all_col_names
|
||||
col_str = ','.join(insert_cols)
|
||||
|
||||
# 反转映射: 5月col_index → cXXX
|
||||
col_to_c = {}
|
||||
for may_col, c_name in ORDERS_COL_MAP.items():
|
||||
if c_name:
|
||||
col_to_c[may_col] = c_name
|
||||
|
||||
batch = []
|
||||
for idx, row in enumerate(data_rows):
|
||||
source_row = idx + 4 # 数据从Excel第4行开始(1-indexed)
|
||||
# 初始化196列全为None
|
||||
c_vals = {f"c{i+1:03d}": None for i in range(196)}
|
||||
|
||||
# 映射已知字段
|
||||
for may_idx, c_name in col_to_c.items():
|
||||
if may_idx < len(row):
|
||||
val = row[may_idx]
|
||||
# 时间字段特殊处理
|
||||
if c_name in ('c175', 'c176'):
|
||||
c_vals[c_name] = to_timestamp_str(val)
|
||||
elif c_name in ('c009', 'c068', 'c114', 'c178', 'c179'):
|
||||
c_vals[c_name] = to_num(val)
|
||||
else:
|
||||
c_vals[c_name] = to_text(val)
|
||||
|
||||
# 从门店名称查找门店编码 → c002
|
||||
store_name = to_text(row[2]) if len(row) > 2 else None
|
||||
if store_name and store_name in store_code_map:
|
||||
c_vals['c002'] = store_code_map[store_name]
|
||||
|
||||
# 序号 → c001
|
||||
c_vals['c001'] = str(idx + 1)
|
||||
|
||||
# 解析结账方式 → 支付方式列
|
||||
payment_str = to_text(row[48]) if len(row) > 48 else None
|
||||
if payment_str:
|
||||
payments = parse_payment(payment_str)
|
||||
for c_name, amount in payments.items():
|
||||
c_vals[c_name] = amount
|
||||
|
||||
# 计算时长(分) → c177
|
||||
opened = c_vals.get('c175')
|
||||
closed = c_vals.get('c176')
|
||||
if opened and closed:
|
||||
try:
|
||||
t1 = pd.to_datetime(opened, format='%Y/%m/%d %H:%M:%S')
|
||||
t2 = pd.to_datetime(closed, format='%Y/%m/%d %H:%M:%S')
|
||||
duration = (t2 - t1).total_seconds() / 60
|
||||
if duration >= 0:
|
||||
c_vals['c177'] = str(int(duration))
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
|
||||
# 构建行数据
|
||||
vals = [fname, source_row] + [c_vals[f"c{i+1:03d}"] for i in range(196)]
|
||||
batch.append(tuple(vals))
|
||||
|
||||
if len(batch) >= 500:
|
||||
execute_values(
|
||||
cur,
|
||||
f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
|
||||
batch,
|
||||
page_size=500
|
||||
)
|
||||
conn.commit()
|
||||
if (idx + 1) % 5000 == 0:
|
||||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
execute_values(
|
||||
cur,
|
||||
f"INSERT INTO public.bill_records ({col_str}) VALUES %s",
|
||||
batch,
|
||||
page_size=500
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||||
cur.close()
|
||||
return len(data_rows)
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 5月品项销售明细 → dish_sales_details 字段映射
|
||||
# ============================================================
|
||||
ITEMS_FIELD_MAP = {
|
||||
'城市': None, # col0
|
||||
'机构编码': 'store_code', # col1
|
||||
'门店名称': 'store_name', # col2
|
||||
'营业日期': None, # col3
|
||||
'下单时间所属餐段': 'business_type', # col4
|
||||
'出品部门': 'production_department', # col5
|
||||
'菜品大类': 'category_level1', # col6
|
||||
'菜品小类': 'category_level2', # col7
|
||||
'菜品编码': None, # col8
|
||||
'品项名称': 'dish_name', # col9
|
||||
'关联菜品名称': None, # col10
|
||||
'商品别名': None, # col11
|
||||
'品项类型': 'item_type', # col12
|
||||
'菜品类型': None, # col13
|
||||
'菜品标签': None, # col14
|
||||
'规格': None, # col15
|
||||
'单位': 'unit', # col16
|
||||
'关联做法': 'preparation_method', # col17
|
||||
'关联加料': None, # col18
|
||||
'关联餐盒': None, # col19
|
||||
'销售方式': None, # col20
|
||||
'订单号': 'bill_no', # col21
|
||||
'销售数量': 'sales_quantity', # col22
|
||||
'赠送数量': None, # col23
|
||||
'销售金额(元)': 'gross_amount', # col24
|
||||
'赠送金额(元)': None, # col25
|
||||
'优惠金额(元)': None, # col26
|
||||
'品项收入(元)': 'received_amount', # col27
|
||||
'点菜时间': None, # col28
|
||||
'下单时间': 'ordered_at', # col29
|
||||
'接单/结账/退菜时间': None, # col30
|
||||
'收银员': None, # col31
|
||||
'点菜员': None, # col32
|
||||
'下单人': None, # col33
|
||||
'订单分类': None, # col34
|
||||
'订单来源': None, # col35
|
||||
'新订单来源': None, # col36
|
||||
'订单子来源': None, # col37
|
||||
'桌台区域': None, # col38
|
||||
'取餐号': 'table_or_pickup_no', # col39
|
||||
'桌牌号': None, # col40
|
||||
'订单金额(元)': None, # col41
|
||||
'营业额(元)': None, # col42
|
||||
'订单优惠(元)': None, # col43
|
||||
'订单收入(元)': None, # col44
|
||||
'标记': None, # col45
|
||||
'退菜数量': None, # col46
|
||||
'退菜金额(元)': None, # col47
|
||||
'敏感操作类型': None, # col48
|
||||
'单品备注': None, # col49
|
||||
}
|
||||
|
||||
NUMERIC_FIELDS = {'sales_quantity', 'gross_amount', 'received_amount', 'unit_price', 'guest_count'}
|
||||
TIMESTAMP_FIELDS = {'opened_at', 'closed_at', 'ordered_at'}
|
||||
|
||||
|
||||
def import_dish_sales_may(conn, items_filepath, report_month):
|
||||
"""将5月品项销售明细导入dish_sales_details"""
|
||||
fname = os.path.basename(items_filepath)
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# 幂等检查
|
||||
cur.execute(
|
||||
"SELECT 1 FROM public.dish_sales_import_log WHERE source_file = %s",
|
||||
(fname,)
|
||||
)
|
||||
if cur.fetchone():
|
||||
print(f" {fname} 已导入过，跳过")
|
||||
return 0
|
||||
|
||||
file_size = os.path.getsize(items_filepath)
|
||||
|
||||
# 读取Excel：row0=标题, row1=日期范围, row2=列头, row3+=数据
|
||||
df = pd.read_excel(items_filepath, header=2, engine='openpyxl')
|
||||
df = df.dropna(subset=[df.columns[0]])
|
||||
data_rows = df.values.tolist()
|
||||
headers = list(df.columns)
|
||||
|
||||
print(f" 数据行数: {len(data_rows)}, 列数: {df.shape[1]}")
|
||||
if not data_rows:
|
||||
return 0
|
||||
|
||||
# 构建列名→索引映射
|
||||
col_map = {}
|
||||
for i, h in enumerate(headers):
|
||||
h_str = str(h).strip() if h else ''
|
||||
if h_str:
|
||||
col_map[h_str] = i
|
||||
|
||||
# 确定要插入的列
|
||||
cols_to_insert = {}
|
||||
for header_name, db_col in ITEMS_FIELD_MAP.items():
|
||||
if db_col and header_name in col_map:
|
||||
cols_to_insert[db_col] = col_map[header_name]
|
||||
|
||||
# 插入import log
|
||||
cur.execute(
|
||||
"""INSERT INTO public.dish_sales_import_log (source_file, file_size_bytes, data_rows, status)
|
||||
VALUES (%s, %s, %s, 'importing')""",
|
||||
(fname, file_size, len(data_rows))
|
||||
)
|
||||
|
||||
all_cols = ['source_file', 'source_row'] + list(cols_to_insert.keys())
|
||||
col_names = ','.join(all_cols)
|
||||
|
||||
batch = []
|
||||
for idx, r in enumerate(data_rows):
|
||||
source_row = idx + 4
|
||||
vals = [fname, source_row]
|
||||
for db_col, excel_idx in cols_to_insert.items():
|
||||
v = r[excel_idx] if excel_idx < len(r) else None
|
||||
if db_col in NUMERIC_FIELDS:
|
||||
vals.append(to_num(v))
|
||||
elif db_col in TIMESTAMP_FIELDS:
|
||||
vals.append(to_timestamp_str(v))
|
||||
else:
|
||||
vals.append(to_text(v))
|
||||
batch.append(tuple(vals))
|
||||
|
||||
if len(batch) >= 500:
|
||||
execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
|
||||
conn.commit()
|
||||
if (idx + 1) % 10000 == 0:
|
||||
print(f" 已导入 {idx + 1}/{len(data_rows)} 行")
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
execute_values(cur, f"INSERT INTO public.dish_sales_details ({col_names}) VALUES %s", batch, page_size=500)
|
||||
conn.commit()
|
||||
|
||||
print(f" 已导入 {len(data_rows)}/{len(data_rows)} 行")
|
||||
|
||||
# 更新log状态
|
||||
cur.execute(
|
||||
"UPDATE public.dish_sales_import_log SET status = 'completed' WHERE source_file = %s",
|
||||
(fname,)
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
cur.close()
|
||||
return len(data_rows)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='5月账单数据导入(全渠道订单明细+品项销售明细)')
|
||||
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
||||
parser.add_argument('--orders', required=True, help='全渠道订单明细Excel文件路径')
|
||||
parser.add_argument('--items', required=True, help='品项销售明细Excel文件路径')
|
||||
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
||||
parser.add_argument('--skip-orders', action='store_true', help='跳过订单明细导入')
|
||||
parser.add_argument('--skip-items', action='store_true', help='跳过品项销售明细导入')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"\n=== 5月账单数据导入 ===")
|
||||
print(f"月份: {args.month}")
|
||||
print(f"数据库: {args.db}")
|
||||
print(f"订单明细: {args.orders}")
|
||||
print(f"品项明细: {args.items}")
|
||||
|
||||
cfg = DB_CONFIG.copy()
|
||||
cfg['dbname'] = args.db
|
||||
conn = psycopg2.connect(**cfg)
|
||||
conn.autocommit = False
|
||||
|
||||
try:
|
||||
# 先从品项明细构建门店编码映射
|
||||
store_code_map = {}
|
||||
if not args.skip_items and os.path.isfile(args.items):
|
||||
print("\n--- 构建门店编码映射 ---")
|
||||
store_code_map = build_store_code_map(args.items)
|
||||
|
||||
# 导入bill_records
|
||||
if not args.skip_orders and os.path.isfile(args.orders):
|
||||
print("\n--- 导入全渠道订单明细 → bill_records ---")
|
||||
import_bill_records_may(conn, args.orders, store_code_map, args.month)
|
||||
|
||||
# 导入dish_sales_details
|
||||
if not args.skip_items and os.path.isfile(args.items):
|
||||
print("\n--- 导入品项销售明细 → dish_sales_details ---")
|
||||
import_dish_sales_may(conn, args.items, args.month)
|
||||
|
||||
print("\n=== 5月账单数据导入完成 ===")
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f'错误: {e}', file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
派生分析数据导入脚本
|
||||
生成以下派生表：
|
||||
- dish_diagnosis_snapshot (菜品诊断快照，从dish_cost_analysis_summary + BOM生成)
|
||||
- central_kitchen_manufacturing_cost_pool (中央厨房制造成本池，从薪资+营业费用+完工报表派生)
|
||||
|
||||
用法:
|
||||
python3 import_derived_data.py --month 2026-04-01 --db bill_query_test
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
|
||||
def generate_dish_diagnosis(conn, diag_date):
|
||||
"""生成菜品诊断快照"""
|
||||
cur = conn.cursor()
|
||||
|
||||
# 检查是否已有数据
|
||||
cur.execute("SELECT count(*) FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
|
||||
existing = cur.fetchone()[0]
|
||||
if existing > 0:
|
||||
print(f" dish_diagnosis_snapshot 已有 {existing} 行 (date={diag_date})，先删除再生成")
|
||||
|
||||
cur.execute("DELETE FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
|
||||
|
||||
# 检查依赖表是否存在且有数据
|
||||
cur.execute("SELECT count(*) FROM public.dish_cost_analysis_summary")
|
||||
summary_count = cur.fetchone()[0]
|
||||
if summary_count == 0:
|
||||
print(" 跳过: dish_cost_analysis_summary 无数据")
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
# 检查 analytics.fact_recipe_bom 是否存在
|
||||
cur.execute("""
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'analytics' AND table_name = 'fact_recipe_bom'
|
||||
)
|
||||
""")
|
||||
has_bom = cur.fetchone()[0]
|
||||
|
||||
bom_join = ""
|
||||
bom_select = "0 AS cnt, 0 AS unique_cnt, 0 AS avg_waste"
|
||||
if has_bom:
|
||||
bom_select = "COALESCE(bom.cnt, 0), COALESCE(bom.unique_cnt, 0), COALESCE(round(bom.avg_waste::numeric, 2), 0)"
|
||||
bom_join = """
|
||||
LEFT JOIN (
|
||||
SELECT b.sku_code,
|
||||
count(*) AS cnt,
|
||||
count(*) FILTER (WHERE r.ref_count = 1) AS unique_cnt,
|
||||
sum(b.waste_rate * b.standard_gross_quantity) / nullif(sum(b.standard_gross_quantity), 0) AS avg_waste
|
||||
FROM analytics.fact_recipe_bom b
|
||||
LEFT JOIN (
|
||||
SELECT material_code, count(DISTINCT sku_code) AS ref_count
|
||||
FROM analytics.fact_recipe_bom GROUP BY material_code
|
||||
) r ON r.material_code = b.material_code
|
||||
GROUP BY b.sku_code
|
||||
) bom ON bom.sku_code = s.dish_code
|
||||
"""
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO public.dish_diagnosis_snapshot (
|
||||
diagnosis_date, dish_code, dish_name, category_l1,
|
||||
sales_amount, sales_quantity,
|
||||
theoretical_margin_pct, actual_margin_pct, cost_variance_amount,
|
||||
cost_tier, bom_complexity_score, unique_material_count, waste_rate_avg,
|
||||
diagnosis_type, diagnosis_detail, suggested_action, priority
|
||||
)
|
||||
SELECT
|
||||
'{diag_date}'::date,
|
||||
s.dish_code,
|
||||
s.dish_name,
|
||||
s.category_level1,
|
||||
round(s.sales_amount::numeric, 2),
|
||||
round(s.sales_quantity::numeric, 2),
|
||||
round(s.theoretical_margin_rate_pct::numeric, 2),
|
||||
round(s.actual_margin_rate_pct::numeric, 2),
|
||||
round(s.cost_variance_amount::numeric, 2),
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
|
||||
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
||||
AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN '紧急'
|
||||
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
||||
AND (s.cost_variance_amount / s.theoretical_cost) > 0.1 THEN '整改'
|
||||
WHEN s.cost_variance_amount > 0 THEN '关注'
|
||||
ELSE '正常'
|
||||
END,
|
||||
{bom_select},
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN '负毛利'
|
||||
WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN '低毛利-定价偏低'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '低毛利-成本超耗'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '高超耗-份量超标'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 100 THEN '高超耗-分摊异常'
|
||||
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN '配方复杂-低销量'
|
||||
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN '独有原料风险'
|
||||
WHEN s.cost_variance_amount > 0 THEN '成本差异'
|
||||
ELSE '正常'
|
||||
END,
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN '实际毛利率为负，需先核查BOM/单位/分摊'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN '理论毛利率为负，定价低于标准成本'
|
||||
WHEN s.theoretical_margin_rate_pct < 30 THEN '理论毛利率低于30%，定价偏低'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '实际成本超理论，存在超耗'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '物料损耗率超过20%'
|
||||
ELSE '成本基本正常'
|
||||
END,
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN 'fix_data'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN 'price_up'
|
||||
WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN 'price_up'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN 'recipe_optimize'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN 'portion_reduce'
|
||||
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'delist'
|
||||
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'evaluate_delist'
|
||||
WHEN s.cost_variance_amount > 0 THEN 'monitor'
|
||||
ELSE 'keep'
|
||||
END,
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN 'P0'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN 'P0'
|
||||
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
||||
AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN 'P0'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 THEN 'P1'
|
||||
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'P2'
|
||||
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'P2'
|
||||
ELSE 'P3'
|
||||
END
|
||||
FROM public.dish_cost_analysis_summary s
|
||||
{bom_join}
|
||||
WHERE s.dish_code IS NOT NULL
|
||||
"""
|
||||
|
||||
cur.execute(sql)
|
||||
inserted = cur.rowcount
|
||||
conn.commit()
|
||||
print(f" dish_diagnosis_snapshot 生成完成: {inserted} 行 (date={diag_date})")
|
||||
cur.close()
|
||||
return inserted
|
||||
|
||||
|
||||
def generate_manufacturing_cost_pool(conn, report_month):
|
||||
"""生成中央厨房制造成本池"""
|
||||
cur = conn.cursor()
|
||||
|
||||
# 检查是否已有数据
|
||||
cur.execute("SELECT count(*) FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
|
||||
existing = cur.fetchone()[0]
|
||||
if existing > 0:
|
||||
print(f" central_kitchen_manufacturing_cost_pool 已有 {existing} 行，先删除再生成")
|
||||
|
||||
cur.execute("DELETE FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
|
||||
|
||||
# 检查依赖表是否有数据
|
||||
cur.execute("SELECT count(*) FROM public.salary_detail_records")
|
||||
salary_count = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM public.operating_expense_records WHERE report_month = %s", (report_month,))
|
||||
expense_count = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM public.central_kitchen_finished_receipt WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date", (report_month, report_month))
|
||||
ck_count = cur.fetchone()[0]
|
||||
|
||||
if salary_count == 0 and expense_count == 0 and ck_count == 0:
|
||||
print(" 跳过: 依赖表无数据")
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
rows = []
|
||||
|
||||
# 1. 直接人工：中央厨房生产人员工资
|
||||
if salary_count > 0:
|
||||
cur.execute("""
|
||||
SELECT round(coalesce(sum(net_pay), 0)::numeric, 2)
|
||||
FROM public.salary_detail_records
|
||||
WHERE org_level1 LIKE '%加工配送中心%' OR org_level1 LIKE '%中央厨房%'
|
||||
OR org_level2 LIKE '%加工配送中心%' OR org_level2 LIKE '%中央厨房%'
|
||||
OR org_level3 LIKE '%加工配送中心%' OR org_level3 LIKE '%中央厨房%'
|
||||
OR org_level4 LIKE '%加工配送中心%' OR org_level4 LIKE '%中央厨房%'
|
||||
OR org_level5 LIKE '%加工配送中心%' OR org_level5 LIKE '%中央厨房%'
|
||||
OR org_level6 LIKE '%加工配送中心%' OR org_level6 LIKE '%中央厨房%'
|
||||
OR org_level7 LIKE '%加工配送中心%' OR org_level7 LIKE '%中央厨房%'
|
||||
OR org_level8 LIKE '%加工配送中心%' OR org_level8 LIKE '%中央厨房%'
|
||||
""")
|
||||
salary_total = cur.fetchone()[0]
|
||||
if salary_total and float(salary_total) > 0:
|
||||
rows.append((
|
||||
report_month, '直接人工', '中央厨房生产人员工资', '薪资明细',
|
||||
'salary_detail_records:加工配送中心/中央厨房',
|
||||
float(salary_total), 1.0, float(salary_total),
|
||||
'中央厨房组织直接归集', False, True,
|
||||
None
|
||||
))
|
||||
|
||||
# 2. 制造费用：从营业费用中供应链部分按比例分摊
|
||||
if expense_count > 0:
|
||||
# 中央厨房人数/加工配送中心人数比例 = 7/11 ≈ 0.6364
|
||||
ck_ratio = 0.63636364
|
||||
expense_items = [
|
||||
('50302', '餐厅房租', 'operating_expense_records:供应链/50302'),
|
||||
('50307', '水费', 'operating_expense_records:供应链/50307'),
|
||||
('50308', '电费', 'operating_expense_records:供应链/50308'),
|
||||
('50310', '员工宿舍费用', 'operating_expense_records:供应链/50310'),
|
||||
('50315', '维修费', 'operating_expense_records:供应链/50315'),
|
||||
]
|
||||
for acct_code, acct_name, source_ref in expense_items:
|
||||
cur.execute("""
|
||||
SELECT round(coalesce(sum(amount), 0)::numeric, 2)
|
||||
FROM public.operating_expense_records
|
||||
WHERE report_month = %s AND account_code = %s
|
||||
AND cost_unit_source_name LIKE '供应链%%'
|
||||
""", (report_month, acct_code))
|
||||
amount = cur.fetchone()[0]
|
||||
if amount and float(amount) > 0:
|
||||
allocated = round(float(amount) * ck_ratio, 6)
|
||||
rows.append((
|
||||
report_month, '制造费用', acct_name, '营业费用',
|
||||
source_ref, float(amount), ck_ratio, allocated,
|
||||
'中央厨房人数/加工配送中心人数（临时）', True, True,
|
||||
'供应链共享费用，待电表、面积或工时数据后替换'
|
||||
))
|
||||
|
||||
# 3. 源报表费用：完工报表费用成本
|
||||
if ck_count > 0:
|
||||
cur.execute("""
|
||||
SELECT round(coalesce(sum(source_fee_cost), 0)::numeric, 4)
|
||||
FROM public.central_kitchen_finished_receipt
|
||||
WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date
|
||||
""", (report_month, report_month))
|
||||
fee_total = cur.fetchone()[0]
|
||||
if fee_total and float(fee_total) > 0:
|
||||
rows.append((
|
||||
report_month, '源报表费用', '完工报表费用成本', '完工入库报表',
|
||||
'central_kitchen_finished_receipt.source_fee_cost',
|
||||
float(fee_total), 1.0, float(fee_total),
|
||||
'源报表原值', True, False,
|
||||
'无水电人工等明细来源，为避免与工资和营业费用重复，暂不计入重建成本'
|
||||
))
|
||||
|
||||
if rows:
|
||||
from psycopg2.extras import execute_values
|
||||
execute_values(cur, """
|
||||
INSERT INTO public.central_kitchen_manufacturing_cost_pool
|
||||
(report_month, cost_type, cost_subtype, source_type, source_reference,
|
||||
source_amount, central_kitchen_share_pct, allocated_amount,
|
||||
allocation_method, is_provisional, include_in_rebuilt_cost, note)
|
||||
VALUES %s
|
||||
""", rows, page_size=100)
|
||||
|
||||
conn.commit()
|
||||
print(f" central_kitchen_manufacturing_cost_pool 生成完成: {len(rows)} 行")
|
||||
cur.close()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='派生分析数据导入')
|
||||
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
||||
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
||||
parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
|
||||
conn.autocommit = False
|
||||
|
||||
diag_date = args.diagnosis_date or datetime.date.today().isoformat()
|
||||
|
||||
try:
|
||||
print(f"\n=== 派生分析数据生成 ===")
|
||||
print(f"数据库: {args.db}, 月份: {args.month}")
|
||||
|
||||
print("\n--- 菜品诊断快照 ---")
|
||||
generate_dish_diagnosis(conn, diag_date)
|
||||
|
||||
print("\n--- 中央厨房制造成本池 ---")
|
||||
generate_manufacturing_cost_pool(conn, args.month)
|
||||
|
||||
print("\n=== 生成完成 ===")
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+170
@@ -0,0 +1,170 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# 5月数据导入脚本
|
||||
# 用法: bash import_may_data.sh [数据库名]
|
||||
# 默认数据库: bill_query_test
|
||||
# ============================================================
|
||||
set -e
|
||||
|
||||
DB="${1:-bill_query_test}"
|
||||
MONTH="2026-05-01"
|
||||
DIAG_DATE="2026-08-18"
|
||||
|
||||
# 数据根目录
|
||||
DATA="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/5月"
|
||||
DATA4="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/4月"
|
||||
PIPELINE="/Users/freedak/Documents/AIDashboard/SBrainCO3/db/pipeline"
|
||||
PY=python3
|
||||
|
||||
echo "============================================================"
|
||||
echo " 5月数据导入 → 数据库: ${DB}"
|
||||
echo " 月份: ${MONTH}"
|
||||
echo "============================================================"
|
||||
|
||||
# ============================================================
|
||||
# Step 1: 薪资 + 考勤
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [1/9] 薪资 + 考勤 ==="
|
||||
${PY} "${PIPELINE}/import_salary_attendance.py" \
|
||||
--month "${MONTH}" \
|
||||
--db "${DB}" \
|
||||
--salary-file "${DATA}/营业数据/2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx" \
|
||||
--attendance-file "${DATA}/营业数据/2026年5月考勤数据表.xlsx"
|
||||
|
||||
# ============================================================
|
||||
# Step 2: bill_records + dish_sales_details（5月专用脚本）
|
||||
# 5月格式: 全渠道订单明细(51列) + 品项销售明细(50列)
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [2/9] bill_records + 菜品销售明细 ==="
|
||||
${PY} "${PIPELINE}/import_bill_records_may.py" \
|
||||
--month "${MONTH}" \
|
||||
--orders "${DATA}/营业数据/正餐事业部_全渠道订单明细_20260730_1112_1785381233074.xlsx" \
|
||||
--items "${DATA}/营业数据/正餐事业部_品项销售明细_20260730_1128_1785382191936.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 4: 库存成本（盘点倒挤成本）
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [4/9] 库存成本 ==="
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset inventory \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/供应链数据/4.13盘点倒挤成本报表2026年5月.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 5: 菜品成本BOM
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [5/9] 菜品成本BOM ==="
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset dish_cost \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/营业数据/2026年5月菜品成本分析报表.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 6: 营业费用
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [6/9] 营业费用 ==="
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset operating_expense \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/2026年5月营业费用分析.xls" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 7: 中央厨房（4个子文件）
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [7/9] 中央厨房 ==="
|
||||
echo " [7a] 完工入库"
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset central_kitchen --ck-type finished \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/中央厨房/2026-05-01--2026-05-31 23 59 59完工入库统计分析报表.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
echo " [7b] 配方耗用"
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset central_kitchen --ck-type recipe \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/中央厨房/按配方导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
echo " [7c] 货品耗用"
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset central_kitchen --ck-type material \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/中央厨房/按货品导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
echo " [7d] 加工单价"
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset central_kitchen --ck-type processing \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/中央厨房/加工单价分析报表.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 8: 配送明细（2个文件）
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [8/9] 配送明细 ==="
|
||||
echo " [8a] 1-15日"
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset distribution \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月1-15日.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
echo " [8b] 16-31日"
|
||||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||||
--dataset distribution \
|
||||
--month "${MONTH}" \
|
||||
--file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月16-31日.xlsx" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 9: 门店位置与映射表（复用4月各店信息文件）
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [9/9] 门店位置与映射表 ==="
|
||||
${PY} "${PIPELINE}/import_store_location.py" \
|
||||
--file "${DATA4}/各店信息新（20260508）.xls" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 10: 派生分析数据
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [10] 派生分析数据 ==="
|
||||
${PY} "${PIPELINE}/import_derived_data.py" \
|
||||
--month "${MONTH}" \
|
||||
--db "${DB}" \
|
||||
--diagnosis-date "${DIAG_DATE}"
|
||||
|
||||
# ============================================================
|
||||
# Step 11: 物化视图刷新
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [11] 物化视图刷新 ==="
|
||||
${PY} "${PIPELINE}/refresh_materialized_views.py" \
|
||||
--db "${DB}"
|
||||
|
||||
# ============================================================
|
||||
# Step 12: 验证
|
||||
# ============================================================
|
||||
echo ""
|
||||
echo "=== [12] 数据一致性验证 ==="
|
||||
${PY} "${PIPELINE}/verify.py" \
|
||||
--db "${DB}" || true
|
||||
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo " 5月数据导入完成"
|
||||
echo "============================================================"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,12 +3,13 @@
|
||||
薪资拆分明细表 + 考勤数据表 导入脚本
|
||||
遵循现有 import_log + records 模式
|
||||
"""
|
||||
import openpyxl
|
||||
import hashlib
|
||||
import psycopg2
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
import argparse
|
||||
import pandas as pd
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
DB_CONFIG = {
|
||||
'host': 'localhost',
|
||||
@@ -18,9 +19,9 @@ DB_CONFIG = {
|
||||
'password': '',
|
||||
}
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
SALARY_FILE = os.path.join(BASE_DIR, '马兰拉面数据分析', '薪资拆分明细表_脱敏.xlsx')
|
||||
ATTENDANCE_FILE = os.path.join(BASE_DIR, '马兰拉面数据分析', '考勤数据表-脱敏.xlsx')
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
SALARY_FILE = os.path.join(BASE_DIR, '数据', '4月', '薪资拆分明细表_脱敏.xlsx')
|
||||
ATTENDANCE_FILE = os.path.join(BASE_DIR, '数据', '4月', '考勤数据表-脱敏.xlsx')
|
||||
|
||||
def file_sha256(filepath):
|
||||
h = hashlib.sha256()
|
||||
@@ -31,34 +32,42 @@ def file_sha256(filepath):
|
||||
|
||||
def to_num(val):
|
||||
if val is None or val == '':
|
||||
return 0
|
||||
return None
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
return None
|
||||
try:
|
||||
return float(val)
|
||||
except (ValueError, TypeError):
|
||||
return 0
|
||||
return None
|
||||
|
||||
def to_text(val):
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, float) and pd.isna(val):
|
||||
return None
|
||||
s = str(val).strip()
|
||||
if s.lower() == 'nan' or s.lower() == 'none':
|
||||
return None
|
||||
return s if s else None
|
||||
|
||||
def import_salary(conn, report_month='2026-04-01'):
|
||||
print(f'导入薪资拆分明细表: {SALARY_FILE}')
|
||||
sha = file_sha256(SALARY_FILE)
|
||||
wb = openpyxl.load_workbook(SALARY_FILE, read_only=True, data_only=True)
|
||||
ws = wb[wb.sheetnames[0]]
|
||||
def import_salary(conn, report_month='2026-04-01', salary_file=None):
|
||||
filepath = salary_file or SALARY_FILE
|
||||
print(f'导入薪资拆分明细表: {filepath}')
|
||||
sha = file_sha256(filepath)
|
||||
|
||||
# Row 1: title, Row 2: merged title, Row 3: headers, Row 4+: data
|
||||
rows = list(ws.iter_rows(min_row=4, values_only=True))
|
||||
data_rows = [r for r in rows if r[0] is not None and r[9] is not None and str(r[9]).strip()]
|
||||
# Read with pandas (row 3 = headers, row 4+ = data)
|
||||
raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
|
||||
print(f' 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}')
|
||||
|
||||
# Data starts from row 3 (0-indexed), filter rows with employee_code (col 9)
|
||||
df = raw_df.iloc[3:].copy()
|
||||
df = df[df[9].notna() & (df[9].astype(str).str.strip() != '') & (df[9].astype(str).str.strip() != 'nan')]
|
||||
data_rows = df.values.tolist()
|
||||
print(f' 数据行数: {len(data_rows)}')
|
||||
|
||||
report_month = report_month
|
||||
source_file = os.path.basename(SALARY_FILE)
|
||||
source_file = os.path.basename(filepath)
|
||||
|
||||
cur = conn.cursor()
|
||||
# Check if already imported
|
||||
cur.execute(
|
||||
'SELECT import_id FROM public.salary_import_log WHERE report_month = %s AND source_file = %s AND file_sha256 = %s',
|
||||
(report_month, source_file, sha)
|
||||
@@ -66,10 +75,8 @@ def import_salary(conn, report_month='2026-04-01'):
|
||||
existing = cur.fetchone()
|
||||
if existing:
|
||||
print(f' 已导入过, import_id={existing[0]}, 跳过')
|
||||
wb.close()
|
||||
return
|
||||
|
||||
# Insert import log
|
||||
cur.execute(
|
||||
'''INSERT INTO public.salary_import_log (report_month, source_file, file_sha256, workbook_rows, imported_rows)
|
||||
VALUES (%s, %s, %s, %s, %s) RETURNING import_id''',
|
||||
@@ -78,9 +85,8 @@ def import_salary(conn, report_month='2026-04-01'):
|
||||
import_id = cur.fetchone()[0]
|
||||
print(f' import_id={import_id}')
|
||||
|
||||
# Batch insert
|
||||
insert_sql = '''INSERT INTO public.salary_detail_records (
|
||||
import_id, source_row,
|
||||
# Column mapping: use execute_values for batch insert
|
||||
col_names = '''import_id, source_row,
|
||||
org_level1, org_level2, org_level3, org_level4, org_level5, org_level6, org_level7, org_level8,
|
||||
employee_code, salary_period, position, work_type, employment_type, hire_date, leave_date,
|
||||
salary_standard, base_wage, overtime_subsidy, social_subsidy, position_wage, tenure_wage,
|
||||
@@ -100,13 +106,11 @@ def import_salary(conn, report_month='2026-04-01'):
|
||||
external_subsidy, external_other_deduction, external_gross,
|
||||
pension_deduction, medical_deduction, unemployment_deduction,
|
||||
external_social_total, external_tax, external_net, internal_tax, internal_net,
|
||||
external_unit, attendance_remark, employment_type_orig, salary_category
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'''
|
||||
external_unit, attendance_remark, employment_type_orig, salary_category'''
|
||||
|
||||
batch = []
|
||||
batch_size = 500
|
||||
for idx, r in enumerate(data_rows):
|
||||
source_row = idx + 4 # Excel row number
|
||||
source_row = idx + 4
|
||||
vals = [
|
||||
import_id, source_row,
|
||||
to_text(r[1]), to_text(r[2]), to_text(r[3]), to_text(r[4]), to_text(r[5]), to_text(r[6]), to_text(r[7]), to_text(r[8]),
|
||||
@@ -130,34 +134,36 @@ def import_salary(conn, report_month='2026-04-01'):
|
||||
to_num(r[90]), to_num(r[91]), to_num(r[92]), to_num(r[93]), to_num(r[94]),
|
||||
to_text(r[95]), to_text(r[96]), to_text(r[97]), to_text(r[98]),
|
||||
]
|
||||
batch.append(vals)
|
||||
if len(batch) >= batch_size:
|
||||
cur.executemany(insert_sql, batch)
|
||||
batch.append(tuple(vals))
|
||||
if len(batch) >= 500:
|
||||
execute_values(cur, f'INSERT INTO public.salary_detail_records ({col_names}) VALUES %s', batch, page_size=500)
|
||||
conn.commit()
|
||||
print(f' 已导入 {idx + 1}/{len(data_rows)} 行')
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
cur.executemany(insert_sql, batch)
|
||||
execute_values(cur, f'INSERT INTO public.salary_detail_records ({col_names}) VALUES %s', batch, page_size=500)
|
||||
conn.commit()
|
||||
print(f' 已导入 {len(data_rows)}/{len(data_rows)} 行')
|
||||
|
||||
wb.close()
|
||||
print(f' 薪资明细导入完成: {len(data_rows)} 行')
|
||||
|
||||
def import_attendance(conn, report_month='2026-04-01'):
|
||||
print(f'\n导入考勤数据表: {ATTENDANCE_FILE}')
|
||||
sha = file_sha256(ATTENDANCE_FILE)
|
||||
wb = openpyxl.load_workbook(ATTENDANCE_FILE, read_only=True, data_only=True)
|
||||
ws = wb[wb.sheetnames[0]]
|
||||
def import_attendance(conn, report_month='2026-04-01', attendance_file=None):
|
||||
filepath = attendance_file or ATTENDANCE_FILE
|
||||
print(f'\n导入考勤数据表: {filepath}')
|
||||
sha = file_sha256(filepath)
|
||||
|
||||
# Row 1: headers, Row 2+: data
|
||||
rows = list(ws.iter_rows(min_row=2, values_only=True))
|
||||
data_rows = [r for r in rows if r[0] is not None]
|
||||
# Read with pandas (row 0 = headers, row 1+ = data)
|
||||
raw_df = pd.read_excel(filepath, header=None, engine='openpyxl')
|
||||
print(f' 总行数: {len(raw_df)}, 列数: {len(raw_df.columns)}')
|
||||
|
||||
# Data starts from row 1
|
||||
df = raw_df.iloc[1:].copy()
|
||||
df = df[df[0].notna() & (df[0].astype(str).str.strip() != '') & (df[0].astype(str).str.strip() != 'nan')]
|
||||
data_rows = df.values.tolist()
|
||||
print(f' 数据行数: {len(data_rows)}')
|
||||
|
||||
report_month = report_month
|
||||
source_file = os.path.basename(ATTENDANCE_FILE)
|
||||
source_file = os.path.basename(filepath)
|
||||
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
@@ -167,7 +173,6 @@ def import_attendance(conn, report_month='2026-04-01'):
|
||||
existing = cur.fetchone()
|
||||
if existing:
|
||||
print(f' 已导入过, import_id={existing[0]}, 跳过')
|
||||
wb.close()
|
||||
return
|
||||
|
||||
cur.execute(
|
||||
@@ -178,44 +183,52 @@ def import_attendance(conn, report_month='2026-04-01'):
|
||||
import_id = cur.fetchone()[0]
|
||||
print(f' import_id={import_id}')
|
||||
|
||||
# 33 columns: 工号, 岗位, 所在部门, day_01..day_30
|
||||
# 34 columns: import_id, source_row, employee_code, position, department, day_01..day_31
|
||||
cols = ['import_id', 'source_row', 'employee_code', 'position', 'department']
|
||||
cols += [f'day_{str(i).zfill(2)}' for i in range(1, 31)]
|
||||
placeholders = ', '.join(['%s'] * len(cols))
|
||||
col_names = ', '.join(cols)
|
||||
insert_sql = f'INSERT INTO public.attendance_records ({col_names}) VALUES ({placeholders})'
|
||||
cols += [f'day_{str(i).zfill(2)}' for i in range(1, 32)]
|
||||
col_names = ','.join(cols)
|
||||
|
||||
batch = []
|
||||
batch_size = 500
|
||||
for idx, r in enumerate(data_rows):
|
||||
source_row = idx + 2
|
||||
vals = [import_id, source_row, to_text(r[0]), to_text(r[1]), to_text(r[2])]
|
||||
# Days 1-30 (columns 3-32)
|
||||
for d in range(30):
|
||||
# Days 1-31 (columns 3-33)
|
||||
for d in range(31):
|
||||
vals.append(to_text(r[3 + d]) if 3 + d < len(r) else None)
|
||||
batch.append(vals)
|
||||
if len(batch) >= batch_size:
|
||||
cur.executemany(insert_sql, batch)
|
||||
batch.append(tuple(vals))
|
||||
if len(batch) >= 500:
|
||||
execute_values(cur, f'INSERT INTO public.attendance_records ({col_names}) VALUES %s', batch, page_size=500)
|
||||
conn.commit()
|
||||
print(f' 已导入 {idx + 1}/{len(data_rows)} 行')
|
||||
batch = []
|
||||
|
||||
if batch:
|
||||
cur.executemany(insert_sql, batch)
|
||||
execute_values(cur, f'INSERT INTO public.attendance_records ({col_names}) VALUES %s', batch, page_size=500)
|
||||
conn.commit()
|
||||
print(f' 已导入 {len(data_rows)}/{len(data_rows)} 行')
|
||||
|
||||
wb.close()
|
||||
print(f' 考勤数据导入完成: {len(data_rows)} 行')
|
||||
|
||||
def main():
|
||||
report_month = sys.argv[1] if len(sys.argv) > 1 else '2026-04-01'
|
||||
print(f'导入月份: {report_month}')
|
||||
conn = psycopg2.connect(**DB_CONFIG)
|
||||
parser = argparse.ArgumentParser(description='薪资考勤数据导入')
|
||||
parser.add_argument('--month', default='2026-04-01', help='报告月份')
|
||||
parser.add_argument('--db', default='bill_query', help='数据库名')
|
||||
parser.add_argument('--skip-salary', action='store_true', help='跳过薪资导入')
|
||||
parser.add_argument('--skip-attendance', action='store_true', help='跳过考勤导入')
|
||||
parser.add_argument('--salary-file', default=None, help='薪资Excel文件路径 (默认使用内置路径)')
|
||||
parser.add_argument('--attendance-file', default=None, help='考勤Excel文件路径 (默认使用内置路径)')
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f'导入月份: {args.month}, 数据库: {args.db}')
|
||||
cfg = DB_CONFIG.copy()
|
||||
cfg['dbname'] = args.db
|
||||
conn = psycopg2.connect(**cfg)
|
||||
conn.autocommit = False
|
||||
try:
|
||||
import_salary(conn, report_month)
|
||||
import_attendance(conn, report_month)
|
||||
if not args.skip_salary:
|
||||
import_salary(conn, args.month, args.salary_file)
|
||||
if not args.skip_attendance:
|
||||
import_attendance(conn, args.month, args.attendance_file)
|
||||
print('\n=== 导入完成 ===')
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
@@ -0,0 +1,619 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
门店位置与映射表导入脚本
|
||||
从 各店信息新（20260508）.xls 导入以下表：
|
||||
- store_location_source_rows (原始行)
|
||||
- store_location_master (门店主表，含地理编码)
|
||||
- sales_store_location_mapping (销售门店→位置映射)
|
||||
|
||||
同时导入静态映射表：
|
||||
- store_name_mapping (薪资名称↔账单名称映射，8条手工数据)
|
||||
- inventory_store_mapping (库存成本单位→销售门店映射)
|
||||
- operating_expense_store_mapping (费用单位→销售门店映射)
|
||||
|
||||
用法:
|
||||
python3 import_store_location.py --file <各店信息.xls> --db bill_query_test
|
||||
python3 import_store_location.py --file <各店信息.xls> --db bill_query_test --skip-mappings
|
||||
"""
|
||||
import argparse
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import date, datetime
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
import pandas as pd
|
||||
import psycopg2
|
||||
from psycopg2.extras import Json, execute_values
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 静态数据
|
||||
# ============================================================
|
||||
|
||||
# WGS84行政区中心点（低精度兜底）
|
||||
ADMIN_CENTROIDS = {
|
||||
"北京市东城区": (39.92855, 116.41637),
|
||||
"北京市西城区": (39.91231, 116.36679),
|
||||
"北京市朝阳区": (39.92149, 116.44355),
|
||||
"北京市海淀区": (39.95933, 116.29845),
|
||||
"北京市丰台区": (39.85856, 116.28625),
|
||||
"北京市石景山区": (39.90569, 116.22298),
|
||||
"北京市昌平区": (40.22077, 116.23128),
|
||||
"北京市大兴区": (39.72684, 116.34159),
|
||||
"北京市通州区": (39.90249, 116.65643),
|
||||
"北京市房山区": (39.74788, 116.14327),
|
||||
"北京市怀柔区": (40.31600, 116.63170),
|
||||
"北京市北京经济技术开发区": (39.79500, 116.50600),
|
||||
"上海市浦东新区": (31.22114, 121.54409),
|
||||
"上海市青浦区": (31.15074, 121.12417),
|
||||
"浙江省杭州市余杭区": (30.41875, 120.29940),
|
||||
"陕西省西安市": (34.34157, 108.93977),
|
||||
}
|
||||
|
||||
MANUAL_ALIAS = {
|
||||
"火锅北三环店": "火锅",
|
||||
"双安总店": "双安",
|
||||
"温泉店": "温泉西部马华",
|
||||
"生命园路店": "北大生命园",
|
||||
"永丰悦界店": "永丰路",
|
||||
"哈马尔罕总部基地店": "哈马尔罕",
|
||||
}
|
||||
|
||||
# 薪资名称 → 账单名称 手工映射
|
||||
STORE_NAME_MAPPING = [
|
||||
("双安店", "双安总店"),
|
||||
("安宁庄快手店", "安宁庄快手"),
|
||||
("海淀大街店", "海淀大街"),
|
||||
("百子湾店", "百子湾路店"),
|
||||
("哈马尔罕大钟寺店", "大钟寺店"),
|
||||
("大钟寺店", "大钟寺店"),
|
||||
("阿里疆（温泉路店）", "温泉店"),
|
||||
("温泉店", "温泉店"),
|
||||
]
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 工具函数
|
||||
# ============================================================
|
||||
|
||||
def clean(v):
|
||||
if v is None:
|
||||
return None
|
||||
s = str(v).strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def normalize_name(v):
|
||||
s = clean(v) or ""
|
||||
s = re.sub(r"[（(].*?[）)]", "", s)
|
||||
for token in ("西部马华", "牛肉面", "餐饮店", "餐厅", "总店"):
|
||||
s = s.replace(token, "")
|
||||
s = re.sub(r"店$", "", s)
|
||||
return re.sub(r"\s+", "", s)
|
||||
|
||||
|
||||
def parse_date(v):
|
||||
if isinstance(v, datetime):
|
||||
return v.date()
|
||||
if isinstance(v, date):
|
||||
return v
|
||||
if isinstance(v, (int, float)):
|
||||
try:
|
||||
return (datetime(1899, 12, 30) + datetime.timedelta(days=float(v))).date()
|
||||
except Exception:
|
||||
return None
|
||||
s = clean(v)
|
||||
if not s or s in {"长期", "未开业"}:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(s).date()
|
||||
except ValueError:
|
||||
pass
|
||||
for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d"):
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def parse_area(v):
|
||||
if isinstance(v, (int, float)):
|
||||
return float(v), "numeric"
|
||||
s = clean(v)
|
||||
if not s:
|
||||
return None, "missing"
|
||||
nums = [float(x) for x in re.findall(r"\d+(?:\.\d+)?", s)]
|
||||
if len(nums) >= 2 and ("-" in s or "至" in s or "~" in s):
|
||||
return sum(nums[:2]) / 2, "range_midpoint"
|
||||
if nums:
|
||||
return nums[0], "text_numeric"
|
||||
return None, "unparsed"
|
||||
|
||||
|
||||
def parse_admin(address):
|
||||
a = clean(address) or ""
|
||||
if "上海市" in a:
|
||||
province, city = "上海市", "上海市"
|
||||
elif "浙江省" in a or "杭州市" in a:
|
||||
province, city = "浙江省", "杭州市"
|
||||
elif "陕西省" in a or "西安市" in a:
|
||||
province, city = "陕西省", "西安市"
|
||||
else:
|
||||
province, city = "北京市", "北京市"
|
||||
|
||||
district = None
|
||||
candidates = ["东城区", "西城区", "朝阳区", "海淀区", "丰台区", "石景山区",
|
||||
"昌平区", "大兴区", "通州区", "房山区", "怀柔区",
|
||||
"浦东新区", "青浦区", "余杭区"]
|
||||
for item in candidates:
|
||||
if item in a:
|
||||
district = item
|
||||
break
|
||||
if "北京经济技术开发区" in a:
|
||||
district = "北京经济技术开发区"
|
||||
return province, city, district
|
||||
|
||||
|
||||
def wgs84_to_gcj02(lat, lon):
|
||||
if lat is None or lon is None:
|
||||
return None, None
|
||||
if not (72.004 <= lon <= 137.8347 and 0.8293 <= lat <= 55.8271):
|
||||
return lat, lon
|
||||
a, ee = 6378245.0, 0.00669342162296594323
|
||||
dlat = _transform_lat(lon - 105.0, lat - 35.0)
|
||||
dlon = _transform_lon(lon - 105.0, lat - 35.0)
|
||||
radlat = lat / 180.0 * math.pi
|
||||
magic = math.sin(radlat)
|
||||
magic = 1 - ee * magic * magic
|
||||
sqrtmagic = math.sqrt(magic)
|
||||
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * math.pi)
|
||||
dlon = (dlon * 180.0) / (a / sqrtmagic * math.cos(radlat) * math.pi)
|
||||
return lat + dlat, lon + dlon
|
||||
|
||||
|
||||
def _transform_lat(x, y):
|
||||
ret = -100.0 + 2.0*x + 3.0*y + 0.2*y*y + 0.1*x*y + 0.2*math.sqrt(abs(x))
|
||||
ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0
|
||||
ret += (20.0*math.sin(y*math.pi) + 40.0*math.sin(y/3.0*math.pi))*2.0/3.0
|
||||
ret += (160.0*math.sin(y/12.0*math.pi) + 320*math.sin(y*math.pi/30.0))*2.0/3.0
|
||||
return ret
|
||||
|
||||
|
||||
def _transform_lon(x, y):
|
||||
ret = 300.0 + x + 2.0*y + 0.1*x*x + 0.1*x*y + 0.1*math.sqrt(abs(x))
|
||||
ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0
|
||||
ret += (20.0*math.sin(x*math.pi) + 40.0*math.sin(x/3.0*math.pi))*2.0/3.0
|
||||
ret += (150.0*math.sin(x/12.0*math.pi) + 300.0*math.sin(x/30.0*math.pi))*2.0/3.0
|
||||
return ret
|
||||
|
||||
|
||||
def fallback_geocode(province, city, district, address):
|
||||
if not address:
|
||||
return None, None, None, "pending_no_address", 0.0
|
||||
key = f"{city}{district}" if district else city
|
||||
if province not in {"北京市", "上海市"} and city:
|
||||
key = f"{province}{city}{district or ''}"
|
||||
coord = ADMIN_CENTROIDS.get(key)
|
||||
if coord:
|
||||
precision = "district_centroid" if district else "city_centroid"
|
||||
confidence = 0.25 if district else 0.10
|
||||
return coord[0], coord[1], key, precision, confidence
|
||||
city_key = f"{province}{city}" if province != city else city
|
||||
coord = ADMIN_CENTROIDS.get(city_key)
|
||||
if coord:
|
||||
return coord[0], coord[1], city_key, "city_centroid", 0.10
|
||||
return None, None, None, "pending_exact_geocode", 0.0
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 导入函数
|
||||
# ============================================================
|
||||
|
||||
def import_store_location(conn, filepath):
|
||||
"""导入门店位置主表和源行表"""
|
||||
source_file = os.path.basename(filepath)
|
||||
print(f"\n=== 门店位置导入 ===")
|
||||
print(f"文件: {source_file}")
|
||||
|
||||
cur = conn.cursor()
|
||||
|
||||
# 幂等检查
|
||||
cur.execute("SELECT 1 FROM public.store_location_master WHERE source_file = %s LIMIT 1", (source_file,))
|
||||
if cur.fetchone():
|
||||
print(" 位置数据已导入过，跳过位置导入")
|
||||
# 仍然需要重建销售门店映射
|
||||
_rebuild_sales_mapping(conn)
|
||||
cur.close()
|
||||
return
|
||||
|
||||
# 读取Excel
|
||||
df = pd.read_excel(filepath, header=0, engine='xlrd')
|
||||
print(f" 总行数: {len(df)}")
|
||||
|
||||
# 清理数据
|
||||
raw_rows = []
|
||||
numbered = []
|
||||
for idx, row in df.iterrows():
|
||||
row_no = idx + 2 # 1-indexed from row 2
|
||||
seq = row.iloc[0]
|
||||
seq_int = int(seq) if isinstance(seq, (int, float)) and not pd.isna(seq) else None
|
||||
raw = {
|
||||
"row": row_no, "seq": seq_int,
|
||||
"name": clean(row.iloc[1]) if len(row) > 1 else None,
|
||||
"company": clean(row.iloc[2]) if len(row) > 2 else None,
|
||||
"brand": clean(row.iloc[3]) if len(row) > 3 else None,
|
||||
"address": clean(row.iloc[4]) if len(row) > 4 else None,
|
||||
"area": clean(row.iloc[5]) if len(row) > 5 else None,
|
||||
"opened": clean(row.iloc[6]) if len(row) > 6 else None,
|
||||
"lease": clean(row.iloc[7]) if len(row) > 7 else None,
|
||||
"license": clean(row.iloc[8]) if len(row) > 8 else None,
|
||||
"note": clean(row.iloc[9]) if len(row) > 9 else None,
|
||||
}
|
||||
raw_rows.append(raw)
|
||||
if seq_int is not None:
|
||||
numbered.append(raw)
|
||||
|
||||
print(f" 编号门店数: {len(numbered)}")
|
||||
|
||||
# 清理旧数据
|
||||
cur.execute("TRUNCATE public.sales_store_location_mapping RESTART IDENTITY CASCADE")
|
||||
cur.execute("DELETE FROM public.store_location_master WHERE source_file = %s OR source_file = 'sales_db_placeholder'", (source_file,))
|
||||
cur.execute("DELETE FROM public.store_location_source_rows WHERE source_file = %s", (source_file,))
|
||||
|
||||
# 导入源行
|
||||
source_values = [(
|
||||
source_file, r["row"], r["seq"], r["name"], r["company"], r["brand"], r["address"],
|
||||
r["area"], r["opened"], r["lease"], r["license"], r["note"]
|
||||
) for r in raw_rows]
|
||||
execute_values(cur, """
|
||||
INSERT INTO public.store_location_source_rows
|
||||
(source_file, source_row, source_store_no, store_short_name, company_name, brand_name,
|
||||
business_address, area_raw, opened_raw, lease_expiry_raw, license_raw, note)
|
||||
VALUES %s
|
||||
""", source_values, page_size=500)
|
||||
print(f" 源行导入: {len(source_values)} 行")
|
||||
|
||||
# 构建主表数据
|
||||
masters = []
|
||||
for r in numbered:
|
||||
area_sqm, area_method = parse_area(r["area"])
|
||||
province, city, district = parse_admin(r["address"])
|
||||
lat, lon, display, precision, confidence = fallback_geocode(province, city, district, r["address"])
|
||||
gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
|
||||
name = r["name"] or f"未命名门店{r['seq']}"
|
||||
if "停业" in name:
|
||||
status = "停业"
|
||||
elif r["opened"] == "未开业":
|
||||
status = "未开业"
|
||||
else:
|
||||
status = "在册"
|
||||
masters.append({
|
||||
**r, "area_sqm": area_sqm, "area_method": area_method,
|
||||
"opened_date": parse_date(r["opened"]), "lease_date": parse_date(r["lease"]),
|
||||
"license_date": parse_date(r["license"]), "status": status,
|
||||
"province": province, "city": city, "district": district,
|
||||
"lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon,
|
||||
"display": display, "precision": precision, "confidence": confidence,
|
||||
})
|
||||
|
||||
# 占位门店
|
||||
placeholders = [
|
||||
{"name": "甄选商城店", "address": None, "province": None, "city": None, "district": None,
|
||||
"status": "线上虚拟门店", "precision": "not_applicable", "display": None, "confidence": 0.0},
|
||||
{"name": "西安含光店", "address": "陕西省西安市含光路（具体门牌待补）",
|
||||
"province": "陕西省", "city": "西安市", "district": None,
|
||||
"status": "地址待补", "precision": "city_centroid", "display": "陕西省西安市", "confidence": 0.10},
|
||||
]
|
||||
for i, p in enumerate(placeholders, start=1):
|
||||
coord = ADMIN_CENTROIDS.get(f"{p['province']}{p['city']}") if p["province"] else None
|
||||
lat, lon = coord if coord else (None, None)
|
||||
gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
|
||||
masters.append({
|
||||
"row": None, "seq": 9000 + i, "name": p["name"], "company": None,
|
||||
"brand": "西部马华牛肉面", "address": p["address"], "area": None,
|
||||
"opened": None, "lease": None, "license": None, "note": "经营数据占位记录",
|
||||
"area_sqm": None, "area_method": "missing", "opened_date": None,
|
||||
"lease_date": None, "license_date": None, "status": p["status"],
|
||||
"province": p["province"], "city": p["city"], "district": p["district"],
|
||||
"lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon,
|
||||
"display": p["display"], "precision": p["precision"], "confidence": p["confidence"],
|
||||
})
|
||||
|
||||
# 导入主表
|
||||
location_ids = {}
|
||||
for m in masters:
|
||||
provider = "offline_admin_centroid" if m["lat"] is not None else "none"
|
||||
geocode_status = "fallback_low_precision" if m["lat"] is not None else m["precision"]
|
||||
src_file = source_file if m["seq"] < 9000 else "sales_db_placeholder"
|
||||
cur.execute("""
|
||||
INSERT INTO public.store_location_master
|
||||
(source_file, source_row, source_store_no, store_short_name, company_name, brand_name,
|
||||
business_address, area_raw, area_sqm, area_parse_method, opened_raw, opened_date,
|
||||
lease_expiry_raw, lease_expiry_date, license_raw, license_date, operating_status,
|
||||
province, city, district, geocode_query, geocode_provider, geocode_status,
|
||||
geocode_precision, geocode_confidence, geocode_display_name,
|
||||
latitude_wgs84, longitude_wgs84, latitude_gcj02, longitude_gcj02, geocode_raw, geocoded_at)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,
|
||||
%s,%s,%s,%s,%s, now())
|
||||
RETURNING location_id
|
||||
""", (
|
||||
src_file, m["row"], m["seq"], m["name"], m["company"], m["brand"], m["address"],
|
||||
m["area"], m["area_sqm"], m["area_method"], m["opened"], m["opened_date"],
|
||||
m["lease"], m["lease_date"], m["license"], m["license_date"], m["status"],
|
||||
m["province"], m["city"], m["district"], m["address"],
|
||||
provider, geocode_status, m["precision"], m["confidence"], m["display"],
|
||||
m["lat"], m["lon"], m["gcj_lat"], m["gcj_lon"],
|
||||
Json({"notice": "行政区中心点兜底，不是门店精确坐标"}) if m["lat"] is not None else None
|
||||
))
|
||||
location_ids[m["name"]] = cur.fetchone()[0]
|
||||
|
||||
print(f" 主表导入: {len(masters)} 家门店")
|
||||
|
||||
# 从DB读取location_ids用于映射
|
||||
cur.execute("SELECT location_id, store_short_name FROM public.store_location_master")
|
||||
location_ids = {name: lid for lid, name in cur.fetchall()}
|
||||
|
||||
_rebuild_sales_mapping(conn, location_ids, masters)
|
||||
|
||||
conn.commit()
|
||||
cur.close()
|
||||
print(f"门店位置导入完成")
|
||||
|
||||
|
||||
def _rebuild_sales_mapping(conn, location_ids=None, masters=None):
|
||||
"""重建销售门店→位置映射表"""
|
||||
cur = conn.cursor()
|
||||
|
||||
# 如果没有传入location_ids，从DB读取
|
||||
if location_ids is None:
|
||||
cur.execute("SELECT location_id, store_short_name FROM public.store_location_master")
|
||||
location_ids = {name: lid for lid, name in cur.fetchall()}
|
||||
|
||||
if not location_ids:
|
||||
print(" 跳过销售门店映射: 无位置数据")
|
||||
cur.close()
|
||||
return
|
||||
|
||||
# 清理旧映射
|
||||
cur.execute("TRUNCATE public.sales_store_location_mapping")
|
||||
|
||||
# 获取销售门店列表
|
||||
try:
|
||||
cur.execute("SELECT store_code, store_name FROM analytics.v_store_scorecard ORDER BY store_name")
|
||||
sales_stores = cur.fetchall()
|
||||
except Exception:
|
||||
sales_stores = []
|
||||
print(" 警告: analytics.v_store_scorecard 不存在，跳过销售门店映射")
|
||||
cur.close()
|
||||
return
|
||||
|
||||
# 构建候选列表
|
||||
if masters:
|
||||
source_candidates = [(m["name"], normalize_name(m["name"])) for m in masters]
|
||||
else:
|
||||
source_candidates = [(name, normalize_name(name)) for name in location_ids.keys()]
|
||||
|
||||
mapped = []
|
||||
for store_code, store_name in sales_stores:
|
||||
target_name = MANUAL_ALIAS.get(store_name)
|
||||
method = "manual_alias" if target_name else "normalized_name"
|
||||
score = 1.0 if target_name else 0.0
|
||||
if not target_name:
|
||||
nn = normalize_name(store_name)
|
||||
exact = [x for x in source_candidates if x[1] == nn and nn]
|
||||
if exact:
|
||||
target_name, _ = exact[0]
|
||||
score = 1.0
|
||||
else:
|
||||
ranked = []
|
||||
for candidate, cn in source_candidates:
|
||||
s = SequenceMatcher(None, nn, cn).ratio()
|
||||
if nn and cn and (nn in cn or cn in nn):
|
||||
s = max(s, 0.92)
|
||||
ranked.append((s, candidate))
|
||||
if ranked:
|
||||
score, target_name = max(ranked)
|
||||
method = "fuzzy_name"
|
||||
location_id = location_ids.get(target_name)
|
||||
if not location_id or score < 0.60:
|
||||
print(f" 警告: 经营门店无法映射: {store_code} {store_name} -> {target_name} score={score}")
|
||||
continue
|
||||
status = "confirmed" if method in {"manual_alias", "normalized_name"} or score >= 0.85 else "reviewed_low_confidence"
|
||||
note = None if status == "confirmed" else "名称相似匹配，建议业务复核"
|
||||
mapped.append((store_code, store_name, location_id, method, score, status, note))
|
||||
|
||||
if mapped:
|
||||
execute_values(cur, """
|
||||
INSERT INTO public.sales_store_location_mapping
|
||||
(sales_store_code, sales_store_name, location_id, mapping_method, mapping_confidence, mapping_status, review_note)
|
||||
VALUES %s
|
||||
""", mapped, page_size=500)
|
||||
conn.commit()
|
||||
print(f" 销售门店映射: {len(mapped)} 家")
|
||||
cur.close()
|
||||
|
||||
|
||||
def import_store_name_mapping(conn):
|
||||
"""导入薪资名称↔账单名称映射（静态8条）"""
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT count(*) FROM public.store_name_mapping")
|
||||
if cur.fetchone()[0] > 0:
|
||||
print("store_name_mapping已有数据，跳过")
|
||||
cur.close()
|
||||
return
|
||||
|
||||
execute_values(
|
||||
cur,
|
||||
"INSERT INTO public.store_name_mapping (salary_name, bill_name) VALUES %s",
|
||||
STORE_NAME_MAPPING,
|
||||
page_size=100
|
||||
)
|
||||
conn.commit()
|
||||
print(f"store_name_mapping导入完成: {len(STORE_NAME_MAPPING)} 条")
|
||||
cur.close()
|
||||
|
||||
|
||||
def import_inventory_store_mapping(conn):
|
||||
"""从库存成本数据推导库存门店映射"""
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT count(*) FROM public.inventory_store_mapping")
|
||||
if cur.fetchone()[0] > 0:
|
||||
print("inventory_store_mapping已有数据，跳过")
|
||||
cur.close()
|
||||
return
|
||||
|
||||
# 从库存成本记录中提取唯一的成本单位编码和名称
|
||||
cur.execute("""
|
||||
SELECT DISTINCT cost_unit_code, cost_unit_name
|
||||
FROM public.inventory_cost_records
|
||||
WHERE report_month = '2026-04-01'
|
||||
AND cost_unit_code IS NOT NULL
|
||||
ORDER BY cost_unit_code
|
||||
""")
|
||||
cost_units = cur.fetchall()
|
||||
|
||||
mappings = []
|
||||
for code, name in cost_units:
|
||||
# 成本单位编码与销售门店编码相同的直接映射
|
||||
mappings.append((
|
||||
code, name, code, name,
|
||||
'经营门店', True, '门店编码精确匹配', 100.00, None
|
||||
))
|
||||
|
||||
if mappings:
|
||||
execute_values(cur, """
|
||||
INSERT INTO public.inventory_store_mapping
|
||||
(cost_unit_code, cost_unit_name, sales_store_code, sales_store_name,
|
||||
unit_type, include_in_operating_cost, mapping_method, mapping_confidence, review_note)
|
||||
VALUES %s
|
||||
""", mappings, page_size=500)
|
||||
conn.commit()
|
||||
print(f"inventory_store_mapping导入完成: {len(mappings)} 条")
|
||||
else:
|
||||
print("inventory_store_mapping: 无库存成本数据可推导")
|
||||
|
||||
cur.close()
|
||||
|
||||
|
||||
def import_operating_expense_store_mapping(conn):
|
||||
"""从营业费用数据推导费用门店映射"""
|
||||
cur = conn.cursor()
|
||||
cur.execute("SELECT count(*) FROM public.operating_expense_store_mapping")
|
||||
if cur.fetchone()[0] > 0:
|
||||
print("operating_expense_store_mapping已有数据，跳过")
|
||||
cur.close()
|
||||
return
|
||||
|
||||
# 从营业费用记录中提取唯一的成本单位名称
|
||||
cur.execute("""
|
||||
SELECT DISTINCT cost_unit_source_name
|
||||
FROM public.operating_expense_records
|
||||
WHERE report_month = '2026-04-01'
|
||||
AND cost_unit_source_name IS NOT NULL
|
||||
ORDER BY cost_unit_source_name
|
||||
""")
|
||||
expense_units = cur.fetchall()
|
||||
|
||||
# 尝试从销售数据获取门店列表做匹配
|
||||
try:
|
||||
cur.execute("SELECT store_code, store_name FROM analytics.dim_store ORDER BY store_code")
|
||||
sales_stores = cur.fetchall()
|
||||
except Exception:
|
||||
sales_stores = []
|
||||
|
||||
mappings = []
|
||||
for (source_name,) in expense_units:
|
||||
# 标准化名称：去掉"金额"后缀
|
||||
normalized = re.sub(r"金额$", "", source_name).strip()
|
||||
# 尝试精确匹配
|
||||
matched_code = None
|
||||
matched_name = None
|
||||
method = "unmapped"
|
||||
confidence = 0.0
|
||||
|
||||
for sc, sn in sales_stores:
|
||||
if normalized == sn or normalized in sn or sn in normalized:
|
||||
matched_code = sc
|
||||
matched_name = sn
|
||||
method = "标准名称精确匹配"
|
||||
confidence = 100.0
|
||||
break
|
||||
|
||||
if not matched_code:
|
||||
# 尝试编码前缀匹配
|
||||
m = re.match(r"^(\d+)\s+", normalized)
|
||||
if m:
|
||||
prefix = m.group(1)
|
||||
for sc, sn in sales_stores:
|
||||
if sc == prefix:
|
||||
matched_code = sc
|
||||
matched_name = sn
|
||||
method = "编码前缀匹配"
|
||||
confidence = 90.0
|
||||
break
|
||||
|
||||
mappings.append((
|
||||
source_name, normalized,
|
||||
matched_code, matched_name,
|
||||
'经营门店', matched_code is not None,
|
||||
method if matched_code else "待人工确认",
|
||||
confidence,
|
||||
None if matched_code else "需人工确认匹配关系"
|
||||
))
|
||||
|
||||
if mappings:
|
||||
execute_values(cur, """
|
||||
INSERT INTO public.operating_expense_store_mapping
|
||||
(cost_unit_source_name, normalized_cost_unit_name, sales_store_code, sales_store_name,
|
||||
unit_type, include_in_operating_analysis, mapping_method, mapping_confidence, review_note)
|
||||
VALUES %s
|
||||
""", mappings, page_size=500)
|
||||
conn.commit()
|
||||
print(f"operating_expense_store_mapping导入完成: {len(mappings)} 条")
|
||||
else:
|
||||
print("operating_expense_store_mapping: 无费用数据可推导")
|
||||
|
||||
cur.close()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# Main
|
||||
# ============================================================
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='门店位置与映射表导入')
|
||||
parser.add_argument('--file', required=True, help='各店信息Excel文件路径')
|
||||
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
||||
parser.add_argument('--skip-location', action='store_true', help='跳过门店位置导入')
|
||||
parser.add_argument('--skip-mappings', action='store_true', help='跳过映射表导入')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
|
||||
conn.autocommit = False
|
||||
|
||||
try:
|
||||
if not args.skip_location:
|
||||
import_store_location(conn, args.file)
|
||||
|
||||
if not args.skip_mappings:
|
||||
print("\n=== 映射表导入 ===")
|
||||
import_store_name_mapping(conn)
|
||||
import_inventory_store_mapping(conn)
|
||||
import_operating_expense_store_mapping(conn)
|
||||
|
||||
print("\n=== 导入完成 ===")
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
物化视图按依赖顺序刷新
|
||||
用法: python3 refresh_materialized_views.py --db bill_query_test
|
||||
"""
|
||||
import argparse
|
||||
import psycopg2
|
||||
import time
|
||||
import sys
|
||||
|
||||
from config import DB_CONFIG, MATVIEW_LAYERS
|
||||
|
||||
|
||||
def refresh_all(dbname, verbose=True):
|
||||
conn = psycopg2.connect(
|
||||
host=DB_CONFIG['host'],
|
||||
port=DB_CONFIG['port'],
|
||||
dbname=dbname,
|
||||
user=DB_CONFIG['user'],
|
||||
password=DB_CONFIG['password'] or None,
|
||||
)
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
|
||||
total = sum(len(layer) for layer in MATVIEW_LAYERS)
|
||||
done = 0
|
||||
failed = []
|
||||
|
||||
for layer_idx, layer in enumerate(MATVIEW_LAYERS):
|
||||
if verbose:
|
||||
print(f"\n--- L{layer_idx}: {len(layer)} 个物化视图 ---")
|
||||
for mv in layer:
|
||||
done += 1
|
||||
t0 = time.time()
|
||||
try:
|
||||
cur.execute(f'REFRESH MATERIALIZED VIEW analytics."{mv}"')
|
||||
elapsed = time.time() - t0
|
||||
if verbose:
|
||||
print(f" [{done}/{total}] OK {mv} ({elapsed:.1f}s)")
|
||||
except Exception as e:
|
||||
elapsed = time.time() - t0
|
||||
err_msg = str(e).strip().split('\n')[0]
|
||||
if verbose:
|
||||
print(f" [{done}/{total}] FAIL {mv} ({elapsed:.1f}s): {err_msg}")
|
||||
failed.append((mv, err_msg))
|
||||
conn.rollback()
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
|
||||
if failed:
|
||||
print(f"\n=== 完成: {total - len(failed)}/{total} 成功, {len(failed)} 失败 ===")
|
||||
for mv, err in failed:
|
||||
print(f" FAIL {mv}: {err}")
|
||||
# 重试失败项（可能因依赖顺序问题，第二轮能成功）
|
||||
if len(failed) > 0:
|
||||
print(f"\n--- 重试 {len(failed)} 个失败项 ---")
|
||||
conn = psycopg2.connect(
|
||||
host=DB_CONFIG['host'],
|
||||
port=DB_CONFIG['port'],
|
||||
dbname=dbname,
|
||||
user=DB_CONFIG['user'],
|
||||
password=DB_CONFIG['password'] or None,
|
||||
)
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
retry_failed = []
|
||||
for mv, _ in failed:
|
||||
t0 = time.time()
|
||||
try:
|
||||
cur.execute(f'REFRESH MATERIALIZED VIEW analytics."{mv}"')
|
||||
elapsed = time.time() - t0
|
||||
print(f" OK {mv} ({elapsed:.1f}s)")
|
||||
except Exception as e:
|
||||
err_msg = str(e).strip().split('\n')[0]
|
||||
print(f" FAIL {mv}: {err_msg}")
|
||||
retry_failed.append(mv)
|
||||
conn.rollback()
|
||||
cur.close()
|
||||
conn.close()
|
||||
if retry_failed:
|
||||
print(f"\n最终失败: {retry_failed}")
|
||||
return 1
|
||||
else:
|
||||
print(f"\n=== 全部 {total} 个物化视图刷新成功 ===")
|
||||
return 0
|
||||
else:
|
||||
print(f"\n=== 全部 {total} 个物化视图刷新成功 ===")
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='物化视图按依赖顺序刷新')
|
||||
parser.add_argument('--db', default='bill_query_test', help='目标数据库')
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"\n=== 物化视图刷新 ===")
|
||||
print(f"数据库: {args.db}")
|
||||
rc = refresh_all(args.db)
|
||||
sys.exit(rc)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
一键全量导入 + 物化视图刷新
|
||||
用法: python3 run_all.py --month 2026-04-01 --db bill_query_test
|
||||
python3 run_all.py --month 2026-04-01 --db bill_query_test --skip-import # 仅刷新物化
|
||||
python3 run_all.py --month 2026-04-01 --db bill_query_test --skip-views # 仅导入
|
||||
python3 run_all.py --month 2026-04-01 --db bill_query_test --only bill_records # 仅导入某类
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from config import DB_CONFIG, get_data_sources, PIPELINE_DIR
|
||||
|
||||
|
||||
def run_script(name, cmd, cwd=None):
|
||||
"""运行子进程并实时输出"""
|
||||
print(f"\n{'='*60}")
|
||||
print(f" 执行: {name}")
|
||||
print(f" 命令: {' '.join(cmd)}")
|
||||
print(f"{'='*60}")
|
||||
t0 = time.time()
|
||||
result = subprocess.run(cmd, cwd=cwd or PIPELINE_DIR)
|
||||
elapsed = time.time() - t0
|
||||
status = "成功" if result.returncode == 0 else f"失败(rc={result.returncode})"
|
||||
print(f" → {name}: {status} ({elapsed:.1f}s)")
|
||||
return result.returncode == 0
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='全量数据导入与物化刷新流水线')
|
||||
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
||||
parser.add_argument('--db', default='bill_query_test', help='目标数据库')
|
||||
parser.add_argument('--skip-import', action='store_true', help='跳过数据导入，仅刷新物化视图')
|
||||
parser.add_argument('--skip-views', action='store_true', help='跳过物化视图刷新')
|
||||
parser.add_argument('--only', default=None,
|
||||
help='仅导入指定数据集: bill_records, monthly, salary, store_location, derived')
|
||||
parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)')
|
||||
args = parser.parse_args()
|
||||
|
||||
sources = get_data_sources(args.month)
|
||||
if not sources:
|
||||
print(f"错误: 无法找到月份 {args.month} 的数据源")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"\n{'#'*60}")
|
||||
print(f" 数据导入与物化刷新流水线")
|
||||
print(f" 月份: {args.month}")
|
||||
print(f" 数据库: {args.db}")
|
||||
print(f" 数据源根: {sources.get('bill_records', 'N/A')}")
|
||||
print(f"{'#'*60}")
|
||||
|
||||
py = sys.executable
|
||||
results = []
|
||||
|
||||
# ============================================================
|
||||
# Step 1: 数据导入
|
||||
# ============================================================
|
||||
if not args.skip_import:
|
||||
target = args.only
|
||||
|
||||
# 1.1 薪资+考勤
|
||||
if target in (None, 'salary'):
|
||||
cmd = [py, 'import_salary_attendance.py', '--month', args.month, '--db', args.db]
|
||||
sal_file = sources.get('salary')
|
||||
att_file = sources.get('attendance')
|
||||
if sal_file and os.path.isfile(sal_file):
|
||||
cmd.extend(['--salary-file', sal_file])
|
||||
if att_file and os.path.isfile(att_file):
|
||||
cmd.extend(['--attendance-file', att_file])
|
||||
ok = run_script("薪资+考勤导入", cmd)
|
||||
results.append(('薪资+考勤', ok))
|
||||
|
||||
# 1.2 账单查询原始数据（bill_records + bill_columns）
|
||||
if target in (None, 'bill_records'):
|
||||
bill_dir = sources.get('bill_records')
|
||||
if bill_dir and os.path.exists(bill_dir):
|
||||
ok = run_script("账单查询导入(bill_records)", [
|
||||
py, 'import_bill_records.py',
|
||||
'--month', args.month,
|
||||
'--file', bill_dir,
|
||||
'--db', args.db,
|
||||
])
|
||||
results.append(('bill_records', ok))
|
||||
else:
|
||||
print(f"\n 跳过 bill_records: 路径不存在 {bill_dir}")
|
||||
results.append(('bill_records', False))
|
||||
|
||||
# 1.3 月度业务数据（库存/销售/成本/费用/中央厨房/配送）
|
||||
if target in (None, 'monthly'):
|
||||
monthly_datasets = [
|
||||
('inventory', 'inventory', None),
|
||||
('dish_sales', 'dish_sales', None),
|
||||
('dish_cost', 'dish_cost', None),
|
||||
('operating_expense', 'operating_expense', None),
|
||||
('central_kitchen_finished', 'central_kitchen', 'finished'),
|
||||
('central_kitchen_recipe', 'central_kitchen', 'recipe'),
|
||||
('central_kitchen_material', 'central_kitchen', 'material'),
|
||||
('central_kitchen_processing', 'central_kitchen', 'processing'),
|
||||
('distribution', 'distribution', None),
|
||||
('distribution_2', 'distribution', None),
|
||||
]
|
||||
|
||||
for src_key, dataset, ck_type in monthly_datasets:
|
||||
filepath = sources.get(src_key)
|
||||
if not filepath or not os.path.exists(filepath):
|
||||
print(f"\n 跳过 {dataset}/{ck_type or ''}: 文件不存在 {filepath}")
|
||||
results.append((f'{dataset}/{ck_type or src_key}', False))
|
||||
continue
|
||||
|
||||
cmd = [
|
||||
py, 'import_monthly_data.py',
|
||||
'--dataset', dataset,
|
||||
'--month', args.month,
|
||||
'--file', filepath,
|
||||
'--db', args.db,
|
||||
]
|
||||
if ck_type:
|
||||
cmd.extend(['--ck-type', ck_type])
|
||||
|
||||
label = f"月度数据-{dataset}" + (f"-{ck_type}" if ck_type else f"({os.path.basename(filepath)})")
|
||||
ok = run_script(label, cmd)
|
||||
results.append((label, ok))
|
||||
|
||||
# 1.4 门店位置与映射表
|
||||
if target in (None, 'store_location'):
|
||||
store_file = sources.get('store_location')
|
||||
if store_file and os.path.exists(store_file):
|
||||
ok = run_script("门店位置与映射表导入", [
|
||||
py, 'import_store_location.py',
|
||||
'--file', store_file,
|
||||
'--db', args.db,
|
||||
])
|
||||
results.append(('门店位置', ok))
|
||||
else:
|
||||
print(f"\n 跳过门店位置: 文件不存在 {store_file}")
|
||||
results.append(('门店位置', False))
|
||||
|
||||
# 1.5 派生分析数据
|
||||
if target in (None, 'derived'):
|
||||
diag_date = args.diagnosis_date or ''
|
||||
cmd = [
|
||||
py, 'import_derived_data.py',
|
||||
'--month', args.month,
|
||||
'--db', args.db,
|
||||
]
|
||||
if diag_date:
|
||||
cmd.extend(['--diagnosis-date', diag_date])
|
||||
ok = run_script("派生分析数据生成", cmd)
|
||||
results.append(('派生数据', ok))
|
||||
|
||||
# ============================================================
|
||||
# Step 2: 物化视图刷新
|
||||
# ============================================================
|
||||
if not args.skip_views:
|
||||
ok = run_script("物化视图刷新", [
|
||||
py, 'refresh_materialized_views.py',
|
||||
'--db', args.db,
|
||||
], cwd=PIPELINE_DIR)
|
||||
results.append(('物化视图', ok))
|
||||
|
||||
# ============================================================
|
||||
# Step 3: 验证
|
||||
# ============================================================
|
||||
ok = run_script("数据一致性验证", [
|
||||
py, 'verify.py',
|
||||
'--db', args.db,
|
||||
], cwd=PIPELINE_DIR)
|
||||
results.append(('验证', ok))
|
||||
|
||||
# ============================================================
|
||||
# 汇总
|
||||
# ============================================================
|
||||
print(f"\n{'#'*60}")
|
||||
print(f" 流水线执行汇总")
|
||||
print(f"{'#'*60}")
|
||||
all_ok = True
|
||||
for name, ok in results:
|
||||
status = "✓ 成功" if ok else "✗ 失败"
|
||||
print(f" {status} {name}")
|
||||
if not ok:
|
||||
all_ok = False
|
||||
|
||||
if all_ok:
|
||||
print(f"\n 全部完成!")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(f"\n 部分失败，请检查上方日志")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
测试库与正式库行数对比验证
|
||||
用法: python3 verify.py --db bill_query_test [--prod bill_query]
|
||||
"""
|
||||
import argparse
|
||||
import psycopg2
|
||||
import sys
|
||||
|
||||
from config import DB_CONFIG
|
||||
|
||||
|
||||
def get_table_counts(dbname):
|
||||
conn = psycopg2.connect(
|
||||
host=DB_CONFIG['host'],
|
||||
port=DB_CONFIG['port'],
|
||||
dbname=dbname,
|
||||
user=DB_CONFIG['user'],
|
||||
password=DB_CONFIG['password'] or None,
|
||||
)
|
||||
cur = conn.cursor()
|
||||
cur.execute("""
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_type = 'BASE TABLE'
|
||||
ORDER BY table_name
|
||||
""")
|
||||
tables = [r[0] for r in cur.fetchall()]
|
||||
|
||||
counts = {}
|
||||
for tbl in tables:
|
||||
cur.execute(f"SELECT count(*) FROM public.{tbl}")
|
||||
counts[tbl] = cur.fetchone()[0]
|
||||
|
||||
# 物化视图行数
|
||||
cur.execute("""
|
||||
SELECT matviewname
|
||||
FROM pg_matviews
|
||||
WHERE schemaname = 'analytics'
|
||||
ORDER BY matviewname
|
||||
""")
|
||||
mvs = [r[0] for r in cur.fetchall()]
|
||||
mv_counts = {}
|
||||
for mv in mvs:
|
||||
try:
|
||||
cur.execute(f'SELECT count(*) FROM analytics."{mv}"')
|
||||
mv_counts[mv] = cur.fetchone()[0]
|
||||
except Exception:
|
||||
mv_counts[mv] = -1
|
||||
conn.rollback()
|
||||
|
||||
cur.close()
|
||||
conn.close()
|
||||
return counts, mv_counts
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='测试库与正式库数据一致性验证')
|
||||
parser.add_argument('--db', default='bill_query_test', help='测试库名')
|
||||
parser.add_argument('--prod', default='bill_query', help='正式库名')
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"\n=== 数据一致性验证 ===")
|
||||
print(f"正式库: {args.prod}")
|
||||
print(f"测试库: {args.db}")
|
||||
|
||||
prod_counts, prod_mv = get_table_counts(args.prod)
|
||||
test_counts, test_mv = get_table_counts(args.db)
|
||||
|
||||
# 基础表对比
|
||||
print(f"\n--- 基础表 ({len(prod_counts)} 张) ---")
|
||||
ok = 0
|
||||
diff = 0
|
||||
for tbl in sorted(prod_counts.keys()):
|
||||
p = prod_counts[tbl]
|
||||
t = test_counts.get(tbl, 'MISSING')
|
||||
flag = 'OK' if t == p else 'DIFF'
|
||||
if flag == 'OK':
|
||||
ok += 1
|
||||
else:
|
||||
diff += 1
|
||||
if flag != 'OK' or True: # 显示全部
|
||||
print(f" {flag:4s} {tbl:45s} prod={p:<12s} test={t}")
|
||||
|
||||
print(f"\n 基础表: {ok} OK, {diff} DIFF")
|
||||
|
||||
# 物化视图对比
|
||||
print(f"\n--- 物化视图 ({len(prod_mv)} 个) ---")
|
||||
mv_ok = 0
|
||||
mv_diff = 0
|
||||
for mv in sorted(prod_mv.keys()):
|
||||
p = prod_mv[mv]
|
||||
t = test_mv.get(mv, 'MISSING')
|
||||
flag = 'OK' if t == p else 'DIFF'
|
||||
if flag == 'OK':
|
||||
mv_ok += 1
|
||||
else:
|
||||
mv_diff += 1
|
||||
print(f" {flag:4s} {mv:45s} prod={p:<12s} test={t}")
|
||||
|
||||
print(f"\n 物化视图: {mv_ok} OK, {mv_diff} DIFF")
|
||||
print(f"\n=== 总计: {ok + mv_ok} OK, {diff + mv_diff} DIFF ===")
|
||||
|
||||
if diff + mv_diff > 0:
|
||||
sys.exit(1)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
对比验证脚本：比较测试库与正式库的4月数据
|
||||
用法: python3 verify_import.py
|
||||
"""
|
||||
import psycopg2
|
||||
|
||||
def connect(dbname):
|
||||
return psycopg2.connect(host='localhost', port=5432, dbname=dbname, user='freedak')
|
||||
|
||||
def compare_table(test_cur, prod_cur, table_name, where_clause='1=1', key_cols=None, sum_cols=None):
|
||||
"""对比两个库中同一表的数据"""
|
||||
print(f"\n--- {table_name} ---")
|
||||
|
||||
# Row count
|
||||
test_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}")
|
||||
test_count = test_cur.fetchone()[0]
|
||||
prod_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}")
|
||||
prod_count = prod_cur.fetchone()[0]
|
||||
|
||||
match = "✅" if test_count == prod_count else "❌"
|
||||
print(f" 行数: 测试库={test_count}, 正式库={prod_count} {match}")
|
||||
|
||||
# Sum comparison
|
||||
if sum_cols:
|
||||
for col in sum_cols:
|
||||
test_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}")
|
||||
test_sum = test_cur.fetchone()[0]
|
||||
prod_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}")
|
||||
prod_sum = prod_cur.fetchone()[0]
|
||||
match = "✅" if abs(float(test_sum or 0) - float(prod_sum or 0)) < 0.01 else "❌"
|
||||
print(f" {col}合计: 测试库={test_sum}, 正式库={prod_sum} {match}")
|
||||
|
||||
return test_count == prod_count
|
||||
|
||||
def main():
|
||||
test_conn = connect('bill_query_test')
|
||||
prod_conn = connect('bill_query')
|
||||
test_cur = test_conn.cursor()
|
||||
prod_cur = prod_conn.cursor()
|
||||
|
||||
print("=" * 60)
|
||||
print(" 数据对比验证: 测试库 vs 正式库 (4月数据)")
|
||||
print("=" * 60)
|
||||
|
||||
results = []
|
||||
|
||||
# 1. 库存成本
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'inventory_cost_records',
|
||||
where_clause="report_month = '2026-04-01'",
|
||||
sum_cols=['consumption_amount', 'opening_amount', 'ending_amount', 'purchase_amount']
|
||||
))
|
||||
|
||||
# 2. 菜品销售明细 (测试库只导入了1个文件，按文件对比)
|
||||
test_cur.execute("SELECT DISTINCT source_file FROM public.dish_sales_details")
|
||||
test_files = [r[0] for r in test_cur.fetchall()]
|
||||
for sf in test_files:
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'dish_sales_details',
|
||||
where_clause=f"source_file = '{sf}'",
|
||||
sum_cols=['gross_amount', 'received_amount']
|
||||
))
|
||||
|
||||
# 3. 菜品成本
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'dish_cost_analysis_summary',
|
||||
where_clause="1=1",
|
||||
sum_cols=['theoretical_cost', 'actual_cost', 'sales_amount']
|
||||
))
|
||||
# 3b. 菜品成本原料明细
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'dish_cost_analysis_material_detail',
|
||||
where_clause="1=1",
|
||||
sum_cols=['theoretical_amount', 'actual_amount']
|
||||
))
|
||||
|
||||
# 4. 营业费用
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'operating_expense_records',
|
||||
where_clause="report_month = '2026-04-01'",
|
||||
sum_cols=['amount']
|
||||
))
|
||||
|
||||
# 5. 中央厨房
|
||||
ck_tables = {
|
||||
'central_kitchen_finished_receipt': ('receipt_date', ['amount', 'quantity']),
|
||||
'central_kitchen_recipe_consumption': ('business_date', ['theoretical_quantity', 'theoretical_amount', 'issue_quantity', 'issue_amount']),
|
||||
'central_kitchen_material_daily': ('business_date', ['amount']),
|
||||
'central_kitchen_processing_cost': ('report_month', ['theoretical_cost', 'actual_cost']),
|
||||
}
|
||||
for tbl, (date_col, sums) in ck_tables.items():
|
||||
# Check which sum cols actually exist
|
||||
test_cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{tbl}'")
|
||||
actual_cols = [r[0] for r in test_cur.fetchall()]
|
||||
valid_sums = [c for c in sums if c in actual_cols]
|
||||
if date_col == 'report_month':
|
||||
where = f"{date_col} = '2026-04-01'"
|
||||
else:
|
||||
where = f"{date_col} >= '2026-04-01' AND {date_col} < '2026-05-01'"
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, tbl,
|
||||
where_clause=where,
|
||||
sum_cols=valid_sums if valid_sums else None
|
||||
))
|
||||
|
||||
# 6. 配送明细
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'distribution_detail_records',
|
||||
where_clause="report_month = '2026-04-01'",
|
||||
sum_cols=['cost_total_amount', 'outbound_total_amount', 'total_quantity']
|
||||
))
|
||||
|
||||
# 7. 薪资
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'salary_detail_records',
|
||||
where_clause="1=1",
|
||||
sum_cols=['net_pay', 'gross_pay']
|
||||
))
|
||||
|
||||
# 8. 考勤
|
||||
results.append(compare_table(
|
||||
test_cur, prod_cur, 'attendance_records',
|
||||
where_clause="1=1",
|
||||
sum_cols=None
|
||||
))
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
passed = sum(results)
|
||||
total = len(results)
|
||||
print(f" 通过: {passed}/{total}")
|
||||
if passed == total:
|
||||
print(" ✅ 所有对比通过")
|
||||
else:
|
||||
print(f" ❌ {total - passed} 项对比失败")
|
||||
print("=" * 60)
|
||||
|
||||
test_cur.close()
|
||||
prod_cur.close()
|
||||
test_conn.close()
|
||||
prod_conn.close()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user