feat: 新增db/pipeline数据导入管线，适配5月数据格式
- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
This commit is contained in:
freedakgmail
@@ -0,0 +1,294 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
派生分析数据导入脚本
|
||||
生成以下派生表：
|
||||
- dish_diagnosis_snapshot (菜品诊断快照，从dish_cost_analysis_summary + BOM生成)
|
||||
- central_kitchen_manufacturing_cost_pool (中央厨房制造成本池，从薪资+营业费用+完工报表派生)
|
||||
|
||||
用法:
|
||||
python3 import_derived_data.py --month 2026-04-01 --db bill_query_test
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import sys
|
||||
|
||||
import psycopg2
|
||||
|
||||
|
||||
def generate_dish_diagnosis(conn, diag_date):
|
||||
"""生成菜品诊断快照"""
|
||||
cur = conn.cursor()
|
||||
|
||||
# 检查是否已有数据
|
||||
cur.execute("SELECT count(*) FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
|
||||
existing = cur.fetchone()[0]
|
||||
if existing > 0:
|
||||
print(f" dish_diagnosis_snapshot 已有 {existing} 行 (date={diag_date})，先删除再生成")
|
||||
|
||||
cur.execute("DELETE FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
|
||||
|
||||
# 检查依赖表是否存在且有数据
|
||||
cur.execute("SELECT count(*) FROM public.dish_cost_analysis_summary")
|
||||
summary_count = cur.fetchone()[0]
|
||||
if summary_count == 0:
|
||||
print(" 跳过: dish_cost_analysis_summary 无数据")
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
# 检查 analytics.fact_recipe_bom 是否存在
|
||||
cur.execute("""
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'analytics' AND table_name = 'fact_recipe_bom'
|
||||
)
|
||||
""")
|
||||
has_bom = cur.fetchone()[0]
|
||||
|
||||
bom_join = ""
|
||||
bom_select = "0 AS cnt, 0 AS unique_cnt, 0 AS avg_waste"
|
||||
if has_bom:
|
||||
bom_select = "COALESCE(bom.cnt, 0), COALESCE(bom.unique_cnt, 0), COALESCE(round(bom.avg_waste::numeric, 2), 0)"
|
||||
bom_join = """
|
||||
LEFT JOIN (
|
||||
SELECT b.sku_code,
|
||||
count(*) AS cnt,
|
||||
count(*) FILTER (WHERE r.ref_count = 1) AS unique_cnt,
|
||||
sum(b.waste_rate * b.standard_gross_quantity) / nullif(sum(b.standard_gross_quantity), 0) AS avg_waste
|
||||
FROM analytics.fact_recipe_bom b
|
||||
LEFT JOIN (
|
||||
SELECT material_code, count(DISTINCT sku_code) AS ref_count
|
||||
FROM analytics.fact_recipe_bom GROUP BY material_code
|
||||
) r ON r.material_code = b.material_code
|
||||
GROUP BY b.sku_code
|
||||
) bom ON bom.sku_code = s.dish_code
|
||||
"""
|
||||
|
||||
sql = f"""
|
||||
INSERT INTO public.dish_diagnosis_snapshot (
|
||||
diagnosis_date, dish_code, dish_name, category_l1,
|
||||
sales_amount, sales_quantity,
|
||||
theoretical_margin_pct, actual_margin_pct, cost_variance_amount,
|
||||
cost_tier, bom_complexity_score, unique_material_count, waste_rate_avg,
|
||||
diagnosis_type, diagnosis_detail, suggested_action, priority
|
||||
)
|
||||
SELECT
|
||||
'{diag_date}'::date,
|
||||
s.dish_code,
|
||||
s.dish_name,
|
||||
s.category_level1,
|
||||
round(s.sales_amount::numeric, 2),
|
||||
round(s.sales_quantity::numeric, 2),
|
||||
round(s.theoretical_margin_rate_pct::numeric, 2),
|
||||
round(s.actual_margin_rate_pct::numeric, 2),
|
||||
round(s.cost_variance_amount::numeric, 2),
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
|
||||
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
||||
AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN '紧急'
|
||||
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
||||
AND (s.cost_variance_amount / s.theoretical_cost) > 0.1 THEN '整改'
|
||||
WHEN s.cost_variance_amount > 0 THEN '关注'
|
||||
ELSE '正常'
|
||||
END,
|
||||
{bom_select},
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN '负毛利'
|
||||
WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN '低毛利-定价偏低'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '低毛利-成本超耗'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '高超耗-份量超标'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 100 THEN '高超耗-分摊异常'
|
||||
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN '配方复杂-低销量'
|
||||
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN '独有原料风险'
|
||||
WHEN s.cost_variance_amount > 0 THEN '成本差异'
|
||||
ELSE '正常'
|
||||
END,
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN '实际毛利率为负，需先核查BOM/单位/分摊'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN '理论毛利率为负，定价低于标准成本'
|
||||
WHEN s.theoretical_margin_rate_pct < 30 THEN '理论毛利率低于30%，定价偏低'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '实际成本超理论，存在超耗'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '物料损耗率超过20%'
|
||||
ELSE '成本基本正常'
|
||||
END,
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN 'fix_data'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN 'price_up'
|
||||
WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN 'price_up'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN 'recipe_optimize'
|
||||
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN 'portion_reduce'
|
||||
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'delist'
|
||||
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'evaluate_delist'
|
||||
WHEN s.cost_variance_amount > 0 THEN 'monitor'
|
||||
ELSE 'keep'
|
||||
END,
|
||||
CASE
|
||||
WHEN s.actual_margin_rate_pct < 0 THEN 'P0'
|
||||
WHEN s.theoretical_margin_rate_pct < 0 THEN 'P0'
|
||||
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
||||
AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN 'P0'
|
||||
WHEN s.theoretical_margin_rate_pct < 50 THEN 'P1'
|
||||
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'P2'
|
||||
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'P2'
|
||||
ELSE 'P3'
|
||||
END
|
||||
FROM public.dish_cost_analysis_summary s
|
||||
{bom_join}
|
||||
WHERE s.dish_code IS NOT NULL
|
||||
"""
|
||||
|
||||
cur.execute(sql)
|
||||
inserted = cur.rowcount
|
||||
conn.commit()
|
||||
print(f" dish_diagnosis_snapshot 生成完成: {inserted} 行 (date={diag_date})")
|
||||
cur.close()
|
||||
return inserted
|
||||
|
||||
|
||||
def generate_manufacturing_cost_pool(conn, report_month):
|
||||
"""生成中央厨房制造成本池"""
|
||||
cur = conn.cursor()
|
||||
|
||||
# 检查是否已有数据
|
||||
cur.execute("SELECT count(*) FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
|
||||
existing = cur.fetchone()[0]
|
||||
if existing > 0:
|
||||
print(f" central_kitchen_manufacturing_cost_pool 已有 {existing} 行，先删除再生成")
|
||||
|
||||
cur.execute("DELETE FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
|
||||
|
||||
# 检查依赖表是否有数据
|
||||
cur.execute("SELECT count(*) FROM public.salary_detail_records")
|
||||
salary_count = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM public.operating_expense_records WHERE report_month = %s", (report_month,))
|
||||
expense_count = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM public.central_kitchen_finished_receipt WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date", (report_month, report_month))
|
||||
ck_count = cur.fetchone()[0]
|
||||
|
||||
if salary_count == 0 and expense_count == 0 and ck_count == 0:
|
||||
print(" 跳过: 依赖表无数据")
|
||||
cur.close()
|
||||
return 0
|
||||
|
||||
rows = []
|
||||
|
||||
# 1. 直接人工：中央厨房生产人员工资
|
||||
if salary_count > 0:
|
||||
cur.execute("""
|
||||
SELECT round(coalesce(sum(net_pay), 0)::numeric, 2)
|
||||
FROM public.salary_detail_records
|
||||
WHERE org_level1 LIKE '%加工配送中心%' OR org_level1 LIKE '%中央厨房%'
|
||||
OR org_level2 LIKE '%加工配送中心%' OR org_level2 LIKE '%中央厨房%'
|
||||
OR org_level3 LIKE '%加工配送中心%' OR org_level3 LIKE '%中央厨房%'
|
||||
OR org_level4 LIKE '%加工配送中心%' OR org_level4 LIKE '%中央厨房%'
|
||||
OR org_level5 LIKE '%加工配送中心%' OR org_level5 LIKE '%中央厨房%'
|
||||
OR org_level6 LIKE '%加工配送中心%' OR org_level6 LIKE '%中央厨房%'
|
||||
OR org_level7 LIKE '%加工配送中心%' OR org_level7 LIKE '%中央厨房%'
|
||||
OR org_level8 LIKE '%加工配送中心%' OR org_level8 LIKE '%中央厨房%'
|
||||
""")
|
||||
salary_total = cur.fetchone()[0]
|
||||
if salary_total and float(salary_total) > 0:
|
||||
rows.append((
|
||||
report_month, '直接人工', '中央厨房生产人员工资', '薪资明细',
|
||||
'salary_detail_records:加工配送中心/中央厨房',
|
||||
float(salary_total), 1.0, float(salary_total),
|
||||
'中央厨房组织直接归集', False, True,
|
||||
None
|
||||
))
|
||||
|
||||
# 2. 制造费用：从营业费用中供应链部分按比例分摊
|
||||
if expense_count > 0:
|
||||
# 中央厨房人数/加工配送中心人数比例 = 7/11 ≈ 0.6364
|
||||
ck_ratio = 0.63636364
|
||||
expense_items = [
|
||||
('50302', '餐厅房租', 'operating_expense_records:供应链/50302'),
|
||||
('50307', '水费', 'operating_expense_records:供应链/50307'),
|
||||
('50308', '电费', 'operating_expense_records:供应链/50308'),
|
||||
('50310', '员工宿舍费用', 'operating_expense_records:供应链/50310'),
|
||||
('50315', '维修费', 'operating_expense_records:供应链/50315'),
|
||||
]
|
||||
for acct_code, acct_name, source_ref in expense_items:
|
||||
cur.execute("""
|
||||
SELECT round(coalesce(sum(amount), 0)::numeric, 2)
|
||||
FROM public.operating_expense_records
|
||||
WHERE report_month = %s AND account_code = %s
|
||||
AND cost_unit_source_name LIKE '供应链%%'
|
||||
""", (report_month, acct_code))
|
||||
amount = cur.fetchone()[0]
|
||||
if amount and float(amount) > 0:
|
||||
allocated = round(float(amount) * ck_ratio, 6)
|
||||
rows.append((
|
||||
report_month, '制造费用', acct_name, '营业费用',
|
||||
source_ref, float(amount), ck_ratio, allocated,
|
||||
'中央厨房人数/加工配送中心人数（临时）', True, True,
|
||||
'供应链共享费用，待电表、面积或工时数据后替换'
|
||||
))
|
||||
|
||||
# 3. 源报表费用：完工报表费用成本
|
||||
if ck_count > 0:
|
||||
cur.execute("""
|
||||
SELECT round(coalesce(sum(source_fee_cost), 0)::numeric, 4)
|
||||
FROM public.central_kitchen_finished_receipt
|
||||
WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date
|
||||
""", (report_month, report_month))
|
||||
fee_total = cur.fetchone()[0]
|
||||
if fee_total and float(fee_total) > 0:
|
||||
rows.append((
|
||||
report_month, '源报表费用', '完工报表费用成本', '完工入库报表',
|
||||
'central_kitchen_finished_receipt.source_fee_cost',
|
||||
float(fee_total), 1.0, float(fee_total),
|
||||
'源报表原值', True, False,
|
||||
'无水电人工等明细来源，为避免与工资和营业费用重复，暂不计入重建成本'
|
||||
))
|
||||
|
||||
if rows:
|
||||
from psycopg2.extras import execute_values
|
||||
execute_values(cur, """
|
||||
INSERT INTO public.central_kitchen_manufacturing_cost_pool
|
||||
(report_month, cost_type, cost_subtype, source_type, source_reference,
|
||||
source_amount, central_kitchen_share_pct, allocated_amount,
|
||||
allocation_method, is_provisional, include_in_rebuilt_cost, note)
|
||||
VALUES %s
|
||||
""", rows, page_size=100)
|
||||
|
||||
conn.commit()
|
||||
print(f" central_kitchen_manufacturing_cost_pool 生成完成: {len(rows)} 行")
|
||||
cur.close()
|
||||
return len(rows)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='派生分析数据导入')
|
||||
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
||||
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
||||
parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
|
||||
conn.autocommit = False
|
||||
|
||||
diag_date = args.diagnosis_date or datetime.date.today().isoformat()
|
||||
|
||||
try:
|
||||
print(f"\n=== 派生分析数据生成 ===")
|
||||
print(f"数据库: {args.db}, 月份: {args.month}")
|
||||
|
||||
print("\n--- 菜品诊断快照 ---")
|
||||
generate_dish_diagnosis(conn, diag_date)
|
||||
|
||||
print("\n--- 中央厨房制造成本池 ---")
|
||||
generate_manufacturing_cost_pool(conn, args.month)
|
||||
|
||||
print("\n=== 生成完成 ===")
|
||||
|
||||
except Exception as e:
|
||||
conn.rollback()
|
||||
print(f"错误: {e}", file=sys.stderr)
|
||||
raise
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user