feat: 新增db/pipeline数据导入管线，适配5月数据格式

- 整合所有导入脚本到db/pipeline/目录 - config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构 - import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details) - import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径 - import_may_data.sh: 5月一键导入脚本(12步全流程) - run_all.py: 一键全量导入+物化视图刷新编排 - refresh_materialized_views.py: 按依赖顺序刷新38个物化视图 - verify.py/verify_import.py: 数据一致性验证 - README.md: 管线文档
2026-08-18 22:17:26 +08:00
parent 9b3a9cdb4b
commit 2fc1b96f44
14 changed files with 4096 additions and 65 deletions
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""
+对比验证脚本：比较测试库与正式库的4月数据
+用法: python3 verify_import.py
+"""
+import psycopg2
+
+def connect(dbname):
+    return psycopg2.connect(host='localhost', port=5432, dbname=dbname, user='freedak')
+
+def compare_table(test_cur, prod_cur, table_name, where_clause='1=1', key_cols=None, sum_cols=None):
+    """对比两个库中同一表的数据"""
+    print(f"\n--- {table_name} ---")
+    
+    # Row count
+    test_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}")
+    test_count = test_cur.fetchone()[0]
+    prod_cur.execute(f"SELECT count(*) FROM public.{table_name} WHERE {where_clause}")
+    prod_count = prod_cur.fetchone()[0]
+    
+    match = "✅" if test_count == prod_count else "❌"
+    print(f"  行数: 测试库={test_count}, 正式库={prod_count} {match}")
+    
+    # Sum comparison
+    if sum_cols:
+        for col in sum_cols:
+            test_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}")
+            test_sum = test_cur.fetchone()[0]
+            prod_cur.execute(f"SELECT round(coalesce(sum({col}),0)::numeric, 2) FROM public.{table_name} WHERE {where_clause}")
+            prod_sum = prod_cur.fetchone()[0]
+            match = "✅" if abs(float(test_sum or 0) - float(prod_sum or 0)) < 0.01 else "❌"
+            print(f"  {col}合计: 测试库={test_sum}, 正式库={prod_sum} {match}")
+    
+    return test_count == prod_count
+
+def main():
+    test_conn = connect('bill_query_test')
+    prod_conn = connect('bill_query')
+    test_cur = test_conn.cursor()
+    prod_cur = prod_conn.cursor()
+    
+    print("=" * 60)
+    print("  数据对比验证: 测试库 vs 正式库 (4月数据)")
+    print("=" * 60)
+    
+    results = []
+    
+    # 1. 库存成本
+    results.append(compare_table(
+        test_cur, prod_cur, 'inventory_cost_records',
+        where_clause="report_month = '2026-04-01'",
+        sum_cols=['consumption_amount', 'opening_amount', 'ending_amount', 'purchase_amount']
+    ))
+    
+    # 2. 菜品销售明细 (测试库只导入了1个文件，按文件对比)
+    test_cur.execute("SELECT DISTINCT source_file FROM public.dish_sales_details")
+    test_files = [r[0] for r in test_cur.fetchall()]
+    for sf in test_files:
+        results.append(compare_table(
+            test_cur, prod_cur, 'dish_sales_details',
+            where_clause=f"source_file = '{sf}'",
+            sum_cols=['gross_amount', 'received_amount']
+        ))
+    
+    # 3. 菜品成本
+    results.append(compare_table(
+        test_cur, prod_cur, 'dish_cost_analysis_summary',
+        where_clause="1=1",
+        sum_cols=['theoretical_cost', 'actual_cost', 'sales_amount']
+    ))
+    # 3b. 菜品成本原料明细
+    results.append(compare_table(
+        test_cur, prod_cur, 'dish_cost_analysis_material_detail',
+        where_clause="1=1",
+        sum_cols=['theoretical_amount', 'actual_amount']
+    ))
+    
+    # 4. 营业费用
+    results.append(compare_table(
+        test_cur, prod_cur, 'operating_expense_records',
+        where_clause="report_month = '2026-04-01'",
+        sum_cols=['amount']
+    ))
+    
+    # 5. 中央厨房
+    ck_tables = {
+        'central_kitchen_finished_receipt': ('receipt_date', ['amount', 'quantity']),
+        'central_kitchen_recipe_consumption': ('business_date', ['theoretical_quantity', 'theoretical_amount', 'issue_quantity', 'issue_amount']),
+        'central_kitchen_material_daily': ('business_date', ['amount']),
+        'central_kitchen_processing_cost': ('report_month', ['theoretical_cost', 'actual_cost']),
+    }
+    for tbl, (date_col, sums) in ck_tables.items():
+        # Check which sum cols actually exist
+        test_cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name='{tbl}'")
+        actual_cols = [r[0] for r in test_cur.fetchall()]
+        valid_sums = [c for c in sums if c in actual_cols]
+        if date_col == 'report_month':
+            where = f"{date_col} = '2026-04-01'"
+        else:
+            where = f"{date_col} >= '2026-04-01' AND {date_col} < '2026-05-01'"
+        results.append(compare_table(
+            test_cur, prod_cur, tbl,
+            where_clause=where,
+            sum_cols=valid_sums if valid_sums else None
+        ))
+    
+    # 6. 配送明细
+    results.append(compare_table(
+        test_cur, prod_cur, 'distribution_detail_records',
+        where_clause="report_month = '2026-04-01'",
+        sum_cols=['cost_total_amount', 'outbound_total_amount', 'total_quantity']
+    ))
+    
+    # 7. 薪资
+    results.append(compare_table(
+        test_cur, prod_cur, 'salary_detail_records',
+        where_clause="1=1",
+        sum_cols=['net_pay', 'gross_pay']
+    ))
+    
+    # 8. 考勤
+    results.append(compare_table(
+        test_cur, prod_cur, 'attendance_records',
+        where_clause="1=1",
+        sum_cols=None
+    ))
+    
+    # Summary
+    print("\n" + "=" * 60)
+    passed = sum(results)
+    total = len(results)
+    print(f"  通过: {passed}/{total}")
+    if passed == total:
+        print("  ✅ 所有对比通过")
+    else:
+        print(f"  ❌ {total - passed} 项对比失败")
+    print("=" * 60)
+    
+    test_cur.close()
+    prod_cur.close()
+    test_conn.close()
+    prod_conn.close()
+
+if __name__ == '__main__':
+    main()