freedakgmail
2fc1b96f44
- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
171 lines
6.2 KiB
Bash
Executable File
171 lines
6.2 KiB
Bash
Executable File
#!/bin/bash
|
||
# ============================================================
|
||
# 5月数据导入脚本
|
||
# 用法: bash import_may_data.sh [数据库名]
|
||
# 默认数据库: bill_query_test
|
||
# ============================================================
|
||
set -e
|
||
|
||
DB="${1:-bill_query_test}"
|
||
MONTH="2026-05-01"
|
||
DIAG_DATE="2026-08-18"
|
||
|
||
# 数据根目录
|
||
DATA="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/5月"
|
||
DATA4="/Users/freedak/Documents/AIDashboard/SBrainCO3/数据/4月"
|
||
PIPELINE="/Users/freedak/Documents/AIDashboard/SBrainCO3/db/pipeline"
|
||
PY=python3
|
||
|
||
echo "============================================================"
|
||
echo " 5月数据导入 → 数据库: ${DB}"
|
||
echo " 月份: ${MONTH}"
|
||
echo "============================================================"
|
||
|
||
# ============================================================
|
||
# Step 1: 薪资 + 考勤
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [1/9] 薪资 + 考勤 ==="
|
||
${PY} "${PIPELINE}/import_salary_attendance.py" \
|
||
--month "${MONTH}" \
|
||
--db "${DB}" \
|
||
--salary-file "${DATA}/营业数据/2026年5月薪资拆分明细表_北京西部马华餐饮有限公司.xlsx" \
|
||
--attendance-file "${DATA}/营业数据/2026年5月考勤数据表.xlsx"
|
||
|
||
# ============================================================
|
||
# Step 2: bill_records + dish_sales_details（5月专用脚本）
|
||
# 5月格式: 全渠道订单明细(51列) + 品项销售明细(50列)
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [2/9] bill_records + 菜品销售明细 ==="
|
||
${PY} "${PIPELINE}/import_bill_records_may.py" \
|
||
--month "${MONTH}" \
|
||
--orders "${DATA}/营业数据/正餐事业部_全渠道订单明细_20260730_1112_1785381233074.xlsx" \
|
||
--items "${DATA}/营业数据/正餐事业部_品项销售明细_20260730_1128_1785382191936.xlsx" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 4: 库存成本（盘点倒挤成本）
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [4/9] 库存成本 ==="
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset inventory \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/供应链数据/4.13盘点倒挤成本报表2026年5月.xlsx" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 5: 菜品成本BOM
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [5/9] 菜品成本BOM ==="
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset dish_cost \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/营业数据/2026年5月菜品成本分析报表.xlsx" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 6: 营业费用
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [6/9] 营业费用 ==="
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset operating_expense \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/2026年5月营业费用分析.xls" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 7: 中央厨房（4个子文件）
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [7/9] 中央厨房 ==="
|
||
echo " [7a] 完工入库"
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset central_kitchen --ck-type finished \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/中央厨房/2026-05-01--2026-05-31 23 59 59完工入库统计分析报表.xlsx" \
|
||
--db "${DB}"
|
||
|
||
echo " [7b] 配方耗用"
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset central_kitchen --ck-type recipe \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/中央厨房/按配方导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \
|
||
--db "${DB}"
|
||
|
||
echo " [7c] 货品耗用"
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset central_kitchen --ck-type material \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/中央厨房/按货品导出-货品实际与理论耗用(2026-05-01--2026-05-31).xlsx" \
|
||
--db "${DB}"
|
||
|
||
echo " [7d] 加工单价"
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset central_kitchen --ck-type processing \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/中央厨房/加工单价分析报表.xlsx" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 8: 配送明细（2个文件）
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [8/9] 配送明细 ==="
|
||
echo " [8a] 1-15日"
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset distribution \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月1-15日.xlsx" \
|
||
--db "${DB}"
|
||
|
||
echo " [8b] 16-31日"
|
||
${PY} "${PIPELINE}/import_monthly_data.py" \
|
||
--dataset distribution \
|
||
--month "${MONTH}" \
|
||
--file "${DATA}/供应链数据/全部门店-全部仓库-统计货品明细报表2026年5月16-31日.xlsx" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 9: 门店位置与映射表（复用4月各店信息文件）
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [9/9] 门店位置与映射表 ==="
|
||
${PY} "${PIPELINE}/import_store_location.py" \
|
||
--file "${DATA4}/各店信息新（20260508）.xls" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 10: 派生分析数据
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [10] 派生分析数据 ==="
|
||
${PY} "${PIPELINE}/import_derived_data.py" \
|
||
--month "${MONTH}" \
|
||
--db "${DB}" \
|
||
--diagnosis-date "${DIAG_DATE}"
|
||
|
||
# ============================================================
|
||
# Step 11: 物化视图刷新
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [11] 物化视图刷新 ==="
|
||
${PY} "${PIPELINE}/refresh_materialized_views.py" \
|
||
--db "${DB}"
|
||
|
||
# ============================================================
|
||
# Step 12: 验证
|
||
# ============================================================
|
||
echo ""
|
||
echo "=== [12] 数据一致性验证 ==="
|
||
${PY} "${PIPELINE}/verify.py" \
|
||
--db "${DB}" || true
|
||
|
||
echo ""
|
||
echo "============================================================"
|
||
echo " 5月数据导入完成"
|
||
echo "============================================================"
|