feat: 新增db/pipeline数据导入管线，适配5月数据格式
- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
This commit is contained in:
freedakgmail
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据导入流水线配置
|
||||
统一管理数据源路径、数据库连接参数、导入脚本路径
|
||||
"""
|
||||
import os
|
||||
import glob
|
||||
import sys
|
||||
|
||||
# ============================================================
|
||||
# 路径常量
|
||||
# ============================================================
|
||||
PIPELINE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
DB_DIR = os.path.dirname(PIPELINE_DIR)
|
||||
BASE_DIR = os.path.dirname(DB_DIR)
|
||||
|
||||
# 数据源根目录（西部马华数据分析 下有完整的业务数据）
|
||||
DATA_ROOT = os.path.join(os.path.dirname(BASE_DIR), '西部马华数据分析')
|
||||
# SBrainCO3 数据目录（部分文件只在这里有）
|
||||
DATA_SBRAIN = os.path.join(BASE_DIR, '数据')
|
||||
|
||||
# ============================================================
|
||||
# 数据库配置
|
||||
# ============================================================
|
||||
DB_CONFIG = {
|
||||
'host': 'localhost',
|
||||
'port': 5432,
|
||||
'user': 'freedak',
|
||||
'password': '',
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# 月份配置
|
||||
# ============================================================
|
||||
MONTH_CONFIG = {
|
||||
'2026-04-01': {
|
||||
'month_dir': '4月',
|
||||
'date_range': '2026-04-01--2026-04-30',
|
||||
'dist_half1': '2026年4月1-15日',
|
||||
'dist_half2': '2026年4月16-30日',
|
||||
'inventory_file': '4.13盘点倒挤成本报表2026年4月明细.xlsx',
|
||||
'dish_cost_file': '菜品成本分析报表.xlsx',
|
||||
'expense_file': '2026年4月营业费用分析.xls',
|
||||
'ck_subdir': '中央厨房4月',
|
||||
'salary_file': '薪资拆分明细表_脱敏.xlsx',
|
||||
'attendance_file': '考勤数据表-脱敏.xlsx',
|
||||
'store_info_file': '各店信息新（20260508）.xls',
|
||||
'bill_records_subdir': '账单查询',
|
||||
'dish_sales_subdir': '账单查询-菜品销售明细表',
|
||||
'data_subdirs': {}, # 4月文件平铺在月份目录下
|
||||
},
|
||||
'2026-05-01': {
|
||||
'month_dir': '5月',
|
||||
'date_range': '2026-05-01--2026-05-31',
|
||||
'dist_half1': '2026年5月1-15日',
|
||||
'dist_half2': '2026年5月16-31日',
|
||||
'inventory_file': '4.13盘点倒挤成本报表2026年5月.xlsx',
|
||||
'dish_cost_file': '2026年5月菜品成本分析报表.xlsx',
|
||||
'expense_file': '2026年5月营业费用分析.xls',
|
||||
'ck_subdir': '中央厨房',
|
||||
'salary_file': '2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx',
|
||||
'attendance_file': '2026年5月考勤数据表.xlsx',
|
||||
'store_info_file': None, # 5月无新各店信息文件，复用4月
|
||||
'bill_records_subdir': None, # 5月无独立账单查询目录
|
||||
'dish_sales_subdir': '账单查询-菜品销售明细表',
|
||||
'data_subdirs': { # 5月文件在子目录中
|
||||
'inventory': '供应链数据',
|
||||
'dish_cost': '营业数据',
|
||||
'operating_expense': '',
|
||||
'distribution': '供应链数据',
|
||||
'salary': '营业数据',
|
||||
'attendance': '营业数据',
|
||||
'central_kitchen': '中央厨房',
|
||||
'dish_sales': '营业数据',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _find_file(base_dirs, pattern):
|
||||
"""在多个候选目录中查找文件"""
|
||||
for d in base_dirs:
|
||||
if not d:
|
||||
continue
|
||||
path = os.path.join(d, pattern)
|
||||
if os.path.isfile(path):
|
||||
return path
|
||||
matches = glob.glob(os.path.join(d, pattern))
|
||||
if matches:
|
||||
return matches[0]
|
||||
return None
|
||||
|
||||
|
||||
def get_data_sources(month):
|
||||
"""
|
||||
根据月份返回数据源路径字典
|
||||
month: 'YYYY-MM-01' 格式
|
||||
"""
|
||||
cfg = MONTH_CONFIG.get(month)
|
||||
if not cfg:
|
||||
print(f"警告: 未知月份 {month}，请在 config.py MONTH_CONFIG 中添加配置")
|
||||
return {}
|
||||
|
||||
month_dir = cfg['month_dir']
|
||||
data_dir = os.path.join(DATA_ROOT, month_dir)
|
||||
sbrain_data = os.path.join(DATA_SBRAIN, month_dir)
|
||||
subdirs = cfg.get('data_subdirs', {})
|
||||
|
||||
sources = {}
|
||||
|
||||
def resolve(subdir_key, filename):
|
||||
"""在 data_dir 和 sbrain_data 中按子目录结构查找文件"""
|
||||
sub = subdirs.get(subdir_key, '')
|
||||
candidates = []
|
||||
if sub:
|
||||
candidates.append(os.path.join(sbrain_data, sub, filename))
|
||||
candidates.append(os.path.join(data_dir, sub, filename))
|
||||
candidates.append(os.path.join(sbrain_data, filename))
|
||||
candidates.append(os.path.join(data_dir, filename))
|
||||
for c in candidates:
|
||||
if os.path.isfile(c):
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
def resolve_dir(subdir_key, dirname):
|
||||
"""在 data_dir 和 sbrain_data 中按子目录结构查找目录"""
|
||||
sub = subdirs.get(subdir_key, '')
|
||||
candidates = []
|
||||
if sub:
|
||||
candidates.append(os.path.join(sbrain_data, sub, dirname))
|
||||
candidates.append(os.path.join(data_dir, sub, dirname))
|
||||
candidates.append(os.path.join(sbrain_data, dirname))
|
||||
candidates.append(os.path.join(data_dir, dirname))
|
||||
for c in candidates:
|
||||
if os.path.isdir(c):
|
||||
return c
|
||||
return candidates[0]
|
||||
|
||||
# 1. 账单查询（bill_records）- 目录
|
||||
bill_subdir = cfg.get('bill_records_subdir')
|
||||
if bill_subdir:
|
||||
sources['bill_records'] = resolve_dir('bill_records', bill_subdir)
|
||||
else:
|
||||
sources['bill_records'] = None
|
||||
|
||||
# 2. 菜品销售明细 - 目录
|
||||
sources['dish_sales'] = resolve_dir('dish_sales', cfg['dish_sales_subdir'])
|
||||
|
||||
# 3. 库存成本
|
||||
sources['inventory'] = resolve('inventory', cfg['inventory_file'])
|
||||
|
||||
# 4. 菜品成本BOM
|
||||
sources['dish_cost'] = resolve('dish_cost', cfg['dish_cost_file'])
|
||||
|
||||
# 5. 营业费用
|
||||
sources['operating_expense'] = resolve('operating_expense', cfg['expense_file'])
|
||||
|
||||
# 6. 中央厨房（4个子文件）
|
||||
ck_subdir = cfg['ck_subdir']
|
||||
ck_dir_sbrain = os.path.join(sbrain_data, ck_subdir)
|
||||
ck_dir_data = os.path.join(data_dir, ck_subdir)
|
||||
ck_dir = ck_dir_sbrain if os.path.isdir(ck_dir_sbrain) else ck_dir_data
|
||||
date_range = cfg['date_range']
|
||||
sources['central_kitchen_finished'] = _find_file(
|
||||
[ck_dir], f'{date_range} 23 59 59完工入库统计分析报表.xlsx'
|
||||
) or os.path.join(ck_dir, f'{date_range} 23 59 59完工入库统计分析报表.xlsx')
|
||||
sources['central_kitchen_recipe'] = _find_file(
|
||||
[ck_dir], f'按配方导出-货品实际与理论耗用({date_range}).xlsx'
|
||||
) or os.path.join(ck_dir, f'按配方导出-货品实际与理论耗用({date_range}).xlsx')
|
||||
sources['central_kitchen_material'] = _find_file(
|
||||
[ck_dir], f'按货品导出-货品实际与理论耗用({date_range}).xlsx'
|
||||
) or os.path.join(ck_dir, f'按货品导出-货品实际与理论耗用({date_range}).xlsx')
|
||||
sources['central_kitchen_processing'] = _find_file(
|
||||
[ck_dir], '加工单价分析报表.xlsx'
|
||||
) or os.path.join(ck_dir, '加工单价分析报表.xlsx')
|
||||
|
||||
# 7. 配送明细（2个文件）
|
||||
sources['distribution'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half1"]}.xlsx')
|
||||
sources['distribution_2'] = resolve('distribution', f'全部门店-全部仓库-统计货品明细报表{cfg["dist_half2"]}.xlsx')
|
||||
|
||||
# 8. 薪资+考勤
|
||||
sources['salary'] = resolve('salary', cfg['salary_file'])
|
||||
sources['attendance'] = resolve('attendance', cfg['attendance_file'])
|
||||
|
||||
# 9. 门店位置
|
||||
store_info_file = cfg.get('store_info_file')
|
||||
if store_info_file:
|
||||
sources['store_location'] = resolve('', store_info_file)
|
||||
else:
|
||||
# 5月无新文件，复用4月
|
||||
for p in [
|
||||
os.path.join(DATA_SBRAIN, '4月', '各店信息新（20260508）.xls'),
|
||||
os.path.join(DATA_ROOT, '4月', '各店信息新（20260508）.xls'),
|
||||
]:
|
||||
if os.path.isfile(p):
|
||||
sources['store_location'] = p
|
||||
break
|
||||
else:
|
||||
sources['store_location'] = None
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 物化视图刷新顺序（按依赖分层，同层可并行）
|
||||
# ============================================================
|
||||
MATVIEW_LAYERS = [
|
||||
# L0: 基础物化视图，依赖基础表
|
||||
['bill_fact'],
|
||||
|
||||
# L1: 依赖 bill_fact 或基础表
|
||||
[
|
||||
'mv_daily_revenue',
|
||||
'mv_dish_basket_monthly',
|
||||
'mv_dish_pair_summary_monthly',
|
||||
'mv_dish_sku_summary_monthly',
|
||||
'mv_dish_store_summary_monthly',
|
||||
'mv_district_site_benchmark_monthly',
|
||||
'mv_inventory_cost_classified_monthly',
|
||||
'mv_site_segment_benchmark_monthly',
|
||||
'mv_store_action_priority_deep_monthly',
|
||||
'mv_store_area_efficiency_monthly',
|
||||
'mv_store_benchmark_composite_monthly',
|
||||
'mv_store_category_mix_monthly',
|
||||
'mv_store_deep_diagnosis_monthly',
|
||||
'mv_store_member_opportunity_monthly',
|
||||
'mv_store_overlap_risk_monthly',
|
||||
'mv_store_platform_economics_monthly',
|
||||
'mv_store_repeat_summary_monthly',
|
||||
'mv_store_risk_rating_monthly',
|
||||
'mv_store_scorecard',
|
||||
'mv_store_site_profile_monthly',
|
||||
'mv_store_site_replication_monthly',
|
||||
'mv_store_theoretical_actual_cost_monthly',
|
||||
'dish_sales_april',
|
||||
'v_store_benchmark',
|
||||
'v_store_category_mix',
|
||||
'v_store_platform_economics',
|
||||
],
|
||||
|
||||
# L2: 依赖 L1 物化视图
|
||||
[
|
||||
'dish_basket_april',
|
||||
'dish_category_summary_april',
|
||||
'dish_member_sku_april',
|
||||
'dish_sku_summary_april',
|
||||
'dish_store_sku_april',
|
||||
'mv_dish_sku_abc_monthly',
|
||||
'v_store_action_list',
|
||||
],
|
||||
|
||||
# L3: 依赖 L2
|
||||
[
|
||||
'dish_store_summary_april',
|
||||
'mv_store_action_priority_deep_april',
|
||||
'mv_store_theoretical_actual_cost_april',
|
||||
'v_store_execution_priority',
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
def get_dbname():
|
||||
"""获取默认数据库名"""
|
||||
return 'bill_query_test'
|
||||
Reference in New Issue
Block a user