freedakgmail
2fc1b96f44
- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
295 lines
14 KiB
Python
295 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
派生分析数据导入脚本
|
|
生成以下派生表：
|
|
- dish_diagnosis_snapshot (菜品诊断快照，从dish_cost_analysis_summary + BOM生成)
|
|
- central_kitchen_manufacturing_cost_pool (中央厨房制造成本池，从薪资+营业费用+完工报表派生)
|
|
|
|
用法:
|
|
python3 import_derived_data.py --month 2026-04-01 --db bill_query_test
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import sys
|
|
|
|
import psycopg2
|
|
|
|
|
|
def generate_dish_diagnosis(conn, diag_date):
|
|
"""生成菜品诊断快照"""
|
|
cur = conn.cursor()
|
|
|
|
# 检查是否已有数据
|
|
cur.execute("SELECT count(*) FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
|
|
existing = cur.fetchone()[0]
|
|
if existing > 0:
|
|
print(f" dish_diagnosis_snapshot 已有 {existing} 行 (date={diag_date})，先删除再生成")
|
|
|
|
cur.execute("DELETE FROM public.dish_diagnosis_snapshot WHERE diagnosis_date = %s", (diag_date,))
|
|
|
|
# 检查依赖表是否存在且有数据
|
|
cur.execute("SELECT count(*) FROM public.dish_cost_analysis_summary")
|
|
summary_count = cur.fetchone()[0]
|
|
if summary_count == 0:
|
|
print(" 跳过: dish_cost_analysis_summary 无数据")
|
|
cur.close()
|
|
return 0
|
|
|
|
# 检查 analytics.fact_recipe_bom 是否存在
|
|
cur.execute("""
|
|
SELECT EXISTS (
|
|
SELECT 1 FROM information_schema.tables
|
|
WHERE table_schema = 'analytics' AND table_name = 'fact_recipe_bom'
|
|
)
|
|
""")
|
|
has_bom = cur.fetchone()[0]
|
|
|
|
bom_join = ""
|
|
bom_select = "0 AS cnt, 0 AS unique_cnt, 0 AS avg_waste"
|
|
if has_bom:
|
|
bom_select = "COALESCE(bom.cnt, 0), COALESCE(bom.unique_cnt, 0), COALESCE(round(bom.avg_waste::numeric, 2), 0)"
|
|
bom_join = """
|
|
LEFT JOIN (
|
|
SELECT b.sku_code,
|
|
count(*) AS cnt,
|
|
count(*) FILTER (WHERE r.ref_count = 1) AS unique_cnt,
|
|
sum(b.waste_rate * b.standard_gross_quantity) / nullif(sum(b.standard_gross_quantity), 0) AS avg_waste
|
|
FROM analytics.fact_recipe_bom b
|
|
LEFT JOIN (
|
|
SELECT material_code, count(DISTINCT sku_code) AS ref_count
|
|
FROM analytics.fact_recipe_bom GROUP BY material_code
|
|
) r ON r.material_code = b.material_code
|
|
GROUP BY b.sku_code
|
|
) bom ON bom.sku_code = s.dish_code
|
|
"""
|
|
|
|
sql = f"""
|
|
INSERT INTO public.dish_diagnosis_snapshot (
|
|
diagnosis_date, dish_code, dish_name, category_l1,
|
|
sales_amount, sales_quantity,
|
|
theoretical_margin_pct, actual_margin_pct, cost_variance_amount,
|
|
cost_tier, bom_complexity_score, unique_material_count, waste_rate_avg,
|
|
diagnosis_type, diagnosis_detail, suggested_action, priority
|
|
)
|
|
SELECT
|
|
'{diag_date}'::date,
|
|
s.dish_code,
|
|
s.dish_name,
|
|
s.category_level1,
|
|
round(s.sales_amount::numeric, 2),
|
|
round(s.sales_quantity::numeric, 2),
|
|
round(s.theoretical_margin_rate_pct::numeric, 2),
|
|
round(s.actual_margin_rate_pct::numeric, 2),
|
|
round(s.cost_variance_amount::numeric, 2),
|
|
CASE
|
|
WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
|
|
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
|
AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN '紧急'
|
|
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
|
AND (s.cost_variance_amount / s.theoretical_cost) > 0.1 THEN '整改'
|
|
WHEN s.cost_variance_amount > 0 THEN '关注'
|
|
ELSE '正常'
|
|
END,
|
|
{bom_select},
|
|
CASE
|
|
WHEN s.actual_margin_rate_pct < 0 THEN '数据异常'
|
|
WHEN s.theoretical_margin_rate_pct < 0 THEN '负毛利'
|
|
WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN '低毛利-定价偏低'
|
|
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '低毛利-成本超耗'
|
|
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '高超耗-份量超标'
|
|
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 100 THEN '高超耗-分摊异常'
|
|
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN '配方复杂-低销量'
|
|
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN '独有原料风险'
|
|
WHEN s.cost_variance_amount > 0 THEN '成本差异'
|
|
ELSE '正常'
|
|
END,
|
|
CASE
|
|
WHEN s.actual_margin_rate_pct < 0 THEN '实际毛利率为负，需先核查BOM/单位/分摊'
|
|
WHEN s.theoretical_margin_rate_pct < 0 THEN '理论毛利率为负，定价低于标准成本'
|
|
WHEN s.theoretical_margin_rate_pct < 30 THEN '理论毛利率低于30%，定价偏低'
|
|
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN '实际成本超理论，存在超耗'
|
|
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN '物料损耗率超过20%'
|
|
ELSE '成本基本正常'
|
|
END,
|
|
CASE
|
|
WHEN s.actual_margin_rate_pct < 0 THEN 'fix_data'
|
|
WHEN s.theoretical_margin_rate_pct < 0 THEN 'price_up'
|
|
WHEN s.theoretical_margin_rate_pct < 30 AND s.actual_margin_rate_pct > s.theoretical_margin_rate_pct THEN 'price_up'
|
|
WHEN s.theoretical_margin_rate_pct < 50 AND s.actual_margin_rate_pct < s.theoretical_margin_rate_pct THEN 'recipe_optimize'
|
|
WHEN s.cost_variance_amount > 0 AND COALESCE(bom.avg_waste, 0) > 20 THEN 'portion_reduce'
|
|
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'delist'
|
|
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'evaluate_delist'
|
|
WHEN s.cost_variance_amount > 0 THEN 'monitor'
|
|
ELSE 'keep'
|
|
END,
|
|
CASE
|
|
WHEN s.actual_margin_rate_pct < 0 THEN 'P0'
|
|
WHEN s.theoretical_margin_rate_pct < 0 THEN 'P0'
|
|
WHEN s.cost_variance_amount > 0 AND s.theoretical_cost > 0
|
|
AND (s.cost_variance_amount / s.theoretical_cost) > 0.2 THEN 'P0'
|
|
WHEN s.theoretical_margin_rate_pct < 50 THEN 'P1'
|
|
WHEN COALESCE(bom.cnt, 0) > 15 AND s.sales_quantity < 5 THEN 'P2'
|
|
WHEN COALESCE(bom.unique_cnt, 0) > 3 AND s.sales_amount < 5000 THEN 'P2'
|
|
ELSE 'P3'
|
|
END
|
|
FROM public.dish_cost_analysis_summary s
|
|
{bom_join}
|
|
WHERE s.dish_code IS NOT NULL
|
|
"""
|
|
|
|
cur.execute(sql)
|
|
inserted = cur.rowcount
|
|
conn.commit()
|
|
print(f" dish_diagnosis_snapshot 生成完成: {inserted} 行 (date={diag_date})")
|
|
cur.close()
|
|
return inserted
|
|
|
|
|
|
def generate_manufacturing_cost_pool(conn, report_month):
|
|
"""生成中央厨房制造成本池"""
|
|
cur = conn.cursor()
|
|
|
|
# 检查是否已有数据
|
|
cur.execute("SELECT count(*) FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
|
|
existing = cur.fetchone()[0]
|
|
if existing > 0:
|
|
print(f" central_kitchen_manufacturing_cost_pool 已有 {existing} 行，先删除再生成")
|
|
|
|
cur.execute("DELETE FROM public.central_kitchen_manufacturing_cost_pool WHERE report_month = %s", (report_month,))
|
|
|
|
# 检查依赖表是否有数据
|
|
cur.execute("SELECT count(*) FROM public.salary_detail_records")
|
|
salary_count = cur.fetchone()[0]
|
|
cur.execute("SELECT count(*) FROM public.operating_expense_records WHERE report_month = %s", (report_month,))
|
|
expense_count = cur.fetchone()[0]
|
|
cur.execute("SELECT count(*) FROM public.central_kitchen_finished_receipt WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date", (report_month, report_month))
|
|
ck_count = cur.fetchone()[0]
|
|
|
|
if salary_count == 0 and expense_count == 0 and ck_count == 0:
|
|
print(" 跳过: 依赖表无数据")
|
|
cur.close()
|
|
return 0
|
|
|
|
rows = []
|
|
|
|
# 1. 直接人工：中央厨房生产人员工资
|
|
if salary_count > 0:
|
|
cur.execute("""
|
|
SELECT round(coalesce(sum(net_pay), 0)::numeric, 2)
|
|
FROM public.salary_detail_records
|
|
WHERE org_level1 LIKE '%加工配送中心%' OR org_level1 LIKE '%中央厨房%'
|
|
OR org_level2 LIKE '%加工配送中心%' OR org_level2 LIKE '%中央厨房%'
|
|
OR org_level3 LIKE '%加工配送中心%' OR org_level3 LIKE '%中央厨房%'
|
|
OR org_level4 LIKE '%加工配送中心%' OR org_level4 LIKE '%中央厨房%'
|
|
OR org_level5 LIKE '%加工配送中心%' OR org_level5 LIKE '%中央厨房%'
|
|
OR org_level6 LIKE '%加工配送中心%' OR org_level6 LIKE '%中央厨房%'
|
|
OR org_level7 LIKE '%加工配送中心%' OR org_level7 LIKE '%中央厨房%'
|
|
OR org_level8 LIKE '%加工配送中心%' OR org_level8 LIKE '%中央厨房%'
|
|
""")
|
|
salary_total = cur.fetchone()[0]
|
|
if salary_total and float(salary_total) > 0:
|
|
rows.append((
|
|
report_month, '直接人工', '中央厨房生产人员工资', '薪资明细',
|
|
'salary_detail_records:加工配送中心/中央厨房',
|
|
float(salary_total), 1.0, float(salary_total),
|
|
'中央厨房组织直接归集', False, True,
|
|
None
|
|
))
|
|
|
|
# 2. 制造费用：从营业费用中供应链部分按比例分摊
|
|
if expense_count > 0:
|
|
# 中央厨房人数/加工配送中心人数比例 = 7/11 ≈ 0.6364
|
|
ck_ratio = 0.63636364
|
|
expense_items = [
|
|
('50302', '餐厅房租', 'operating_expense_records:供应链/50302'),
|
|
('50307', '水费', 'operating_expense_records:供应链/50307'),
|
|
('50308', '电费', 'operating_expense_records:供应链/50308'),
|
|
('50310', '员工宿舍费用', 'operating_expense_records:供应链/50310'),
|
|
('50315', '维修费', 'operating_expense_records:供应链/50315'),
|
|
]
|
|
for acct_code, acct_name, source_ref in expense_items:
|
|
cur.execute("""
|
|
SELECT round(coalesce(sum(amount), 0)::numeric, 2)
|
|
FROM public.operating_expense_records
|
|
WHERE report_month = %s AND account_code = %s
|
|
AND cost_unit_source_name LIKE '供应链%%'
|
|
""", (report_month, acct_code))
|
|
amount = cur.fetchone()[0]
|
|
if amount and float(amount) > 0:
|
|
allocated = round(float(amount) * ck_ratio, 6)
|
|
rows.append((
|
|
report_month, '制造费用', acct_name, '营业费用',
|
|
source_ref, float(amount), ck_ratio, allocated,
|
|
'中央厨房人数/加工配送中心人数（临时）', True, True,
|
|
'供应链共享费用，待电表、面积或工时数据后替换'
|
|
))
|
|
|
|
# 3. 源报表费用：完工报表费用成本
|
|
if ck_count > 0:
|
|
cur.execute("""
|
|
SELECT round(coalesce(sum(source_fee_cost), 0)::numeric, 4)
|
|
FROM public.central_kitchen_finished_receipt
|
|
WHERE receipt_date >= date %s AND receipt_date < (date %s + interval '1 month')::date
|
|
""", (report_month, report_month))
|
|
fee_total = cur.fetchone()[0]
|
|
if fee_total and float(fee_total) > 0:
|
|
rows.append((
|
|
report_month, '源报表费用', '完工报表费用成本', '完工入库报表',
|
|
'central_kitchen_finished_receipt.source_fee_cost',
|
|
float(fee_total), 1.0, float(fee_total),
|
|
'源报表原值', True, False,
|
|
'无水电人工等明细来源，为避免与工资和营业费用重复，暂不计入重建成本'
|
|
))
|
|
|
|
if rows:
|
|
from psycopg2.extras import execute_values
|
|
execute_values(cur, """
|
|
INSERT INTO public.central_kitchen_manufacturing_cost_pool
|
|
(report_month, cost_type, cost_subtype, source_type, source_reference,
|
|
source_amount, central_kitchen_share_pct, allocated_amount,
|
|
allocation_method, is_provisional, include_in_rebuilt_cost, note)
|
|
VALUES %s
|
|
""", rows, page_size=100)
|
|
|
|
conn.commit()
|
|
print(f" central_kitchen_manufacturing_cost_pool 生成完成: {len(rows)} 行")
|
|
cur.close()
|
|
return len(rows)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='派生分析数据导入')
|
|
parser.add_argument('--month', required=True, help='报告月份 (YYYY-MM-01)')
|
|
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
|
|
parser.add_argument('--diagnosis-date', default=None, help='诊断日期 (默认今天)')
|
|
|
|
args = parser.parse_args()
|
|
|
|
conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
|
|
conn.autocommit = False
|
|
|
|
diag_date = args.diagnosis_date or datetime.date.today().isoformat()
|
|
|
|
try:
|
|
print(f"\n=== 派生分析数据生成 ===")
|
|
print(f"数据库: {args.db}, 月份: {args.month}")
|
|
|
|
print("\n--- 菜品诊断快照 ---")
|
|
generate_dish_diagnosis(conn, diag_date)
|
|
|
|
print("\n--- 中央厨房制造成本池 ---")
|
|
generate_manufacturing_cost_pool(conn, args.month)
|
|
|
|
print("\n=== 生成完成 ===")
|
|
|
|
except Exception as e:
|
|
conn.rollback()
|
|
print(f"错误: {e}", file=sys.stderr)
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|