freedak/SBrainCO
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 新增db/pipeline数据导入管线，适配5月数据格式

Browse Source 
- 整合所有导入脚本到db/pipeline/目录
- config.py: 按月份动态配置数据源路径，支持4月/5月不同目录结构
- import_bill_records_may.py: 5月专用账单导入(51列全渠道订单明细→bill_records + 50列品项销售明细→dish_sales_details)
- import_salary_attendance.py: 添加--salary-file/--attendance-file参数支持动态文件路径
- import_may_data.sh: 5月一键导入脚本(12步全流程)
- run_all.py: 一键全量导入+物化视图刷新编排
- refresh_materialized_views.py: 按依赖顺序刷新38个物化视图
- verify.py/verify_import.py: 数据一致性验证
- README.md: 管线文档
This commit is contained in:
freedakgmail
2026-08-18 22:17:26 +08:00
parent 9b3a9cdb4b
commit 2fc1b96f44
14 changed files with 4096 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/pipeline/import_store_location.py
+619
View File
@@ -0,0 +1,619 @@
#!/usr/bin/env python3
"""
门店位置与映射表导入脚本
从 各店信息新（20260508）.xls 导入以下表：
- store_location_source_rows (原始行)
- store_location_master (门店主表，含地理编码)
- sales_store_location_mapping (销售门店→位置映射)
同时导入静态映射表：
- store_name_mapping (薪资名称↔账单名称映射，8条手工数据)
- inventory_store_mapping (库存成本单位→销售门店映射)
- operating_expense_store_mapping (费用单位→销售门店映射)
用法:
python3 import_store_location.py --file <各店信息.xls> --db bill_query_test
python3 import_store_location.py --file <各店信息.xls> --db bill_query_test --skip-mappings
"""
import argparse
import math
import os
import re
import sys
from datetime import date, datetime
from difflib import SequenceMatcher
import pandas as pd
import psycopg2
from psycopg2.extras import Json, execute_values
# ============================================================
# 静态数据
# ============================================================
# WGS84行政区中心点（低精度兜底）
ADMIN_CENTROIDS = {
"北京市东城区": (39.92855, 116.41637),
"北京市西城区": (39.91231, 116.36679),
"北京市朝阳区": (39.92149, 116.44355),
"北京市海淀区": (39.95933, 116.29845),
"北京市丰台区": (39.85856, 116.28625),
"北京市石景山区": (39.90569, 116.22298),
"北京市昌平区": (40.22077, 116.23128),
"北京市大兴区": (39.72684, 116.34159),
"北京市通州区": (39.90249, 116.65643),
"北京市房山区": (39.74788, 116.14327),
"北京市怀柔区": (40.31600, 116.63170),
"北京市北京经济技术开发区": (39.79500, 116.50600),
"上海市浦东新区": (31.22114, 121.54409),
"上海市青浦区": (31.15074, 121.12417),
"浙江省杭州市余杭区": (30.41875, 120.29940),
"陕西省西安市": (34.34157, 108.93977),
}
MANUAL_ALIAS = {
"火锅北三环店": "火锅",
"双安总店": "双安",
"温泉店": "温泉西部马华",
"生命园路店": "北大生命园",
"永丰悦界店": "永丰路",
"哈马尔罕总部基地店": "哈马尔罕",
}
# 薪资名称 → 账单名称 手工映射
STORE_NAME_MAPPING = [
("双安店", "双安总店"),
("安宁庄快手店", "安宁庄快手"),
("海淀大街店", "海淀大街"),
("百子湾店", "百子湾路店"),
("哈马尔罕大钟寺店", "大钟寺店"),
("大钟寺店", "大钟寺店"),
("阿里疆（温泉路店）", "温泉店"),
("温泉店", "温泉店"),
]
# ============================================================
# 工具函数
# ============================================================
def clean(v):
if v is None:
return None
s = str(v).strip()
return s or None
def normalize_name(v):
s = clean(v) or ""
s = re.sub(r"[（(].*?[）)]", "", s)
for token in ("西部马华", "牛肉面", "餐饮店", "餐厅", "总店"):
s = s.replace(token, "")
s = re.sub(r"店$", "", s)
return re.sub(r"\s+", "", s)
def parse_date(v):
if isinstance(v, datetime):
return v.date()
if isinstance(v, date):
return v
if isinstance(v, (int, float)):
try:
return (datetime(1899, 12, 30) + datetime.timedelta(days=float(v))).date()
except Exception:
return None
s = clean(v)
if not s or s in {"长期", "未开业"}:
return None
try:
return datetime.fromisoformat(s).date()
except ValueError:
pass
for fmt in ("%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d"):
try:
return datetime.strptime(s, fmt).date()
except ValueError:
pass
return None
def parse_area(v):
if isinstance(v, (int, float)):
return float(v), "numeric"
s = clean(v)
if not s:
return None, "missing"
nums = [float(x) for x in re.findall(r"\d+(?:\.\d+)?", s)]
if len(nums) >= 2 and ("-" in s or "至" in s or "~" in s):
return sum(nums[:2]) / 2, "range_midpoint"
if nums:
return nums[0], "text_numeric"
return None, "unparsed"
def parse_admin(address):
a = clean(address) or ""
if "上海市" in a:
province, city = "上海市", "上海市"
elif "浙江省" in a or "杭州市" in a:
province, city = "浙江省", "杭州市"
elif "陕西省" in a or "西安市" in a:
province, city = "陕西省", "西安市"
else:
province, city = "北京市", "北京市"
district = None
candidates = ["东城区", "西城区", "朝阳区", "海淀区", "丰台区", "石景山区",
"昌平区", "大兴区", "通州区", "房山区", "怀柔区",
"浦东新区", "青浦区", "余杭区"]
for item in candidates:
if item in a:
district = item
break
if "北京经济技术开发区" in a:
district = "北京经济技术开发区"
return province, city, district
def wgs84_to_gcj02(lat, lon):
if lat is None or lon is None:
return None, None
if not (72.004 <= lon <= 137.8347 and 0.8293 <= lat <= 55.8271):
return lat, lon
a, ee = 6378245.0, 0.00669342162296594323
dlat = _transform_lat(lon - 105.0, lat - 35.0)
dlon = _transform_lon(lon - 105.0, lat - 35.0)
radlat = lat / 180.0 * math.pi
magic = math.sin(radlat)
magic = 1 - ee * magic * magic
sqrtmagic = math.sqrt(magic)
dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * math.pi)
dlon = (dlon * 180.0) / (a / sqrtmagic * math.cos(radlat) * math.pi)
return lat + dlat, lon + dlon
def _transform_lat(x, y):
ret = -100.0 + 2.0*x + 3.0*y + 0.2*y*y + 0.1*x*y + 0.2*math.sqrt(abs(x))
ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0
ret += (20.0*math.sin(y*math.pi) + 40.0*math.sin(y/3.0*math.pi))*2.0/3.0
ret += (160.0*math.sin(y/12.0*math.pi) + 320*math.sin(y*math.pi/30.0))*2.0/3.0
return ret
def _transform_lon(x, y):
ret = 300.0 + x + 2.0*y + 0.1*x*x + 0.1*x*y + 0.1*math.sqrt(abs(x))
ret += (20.0*math.sin(6.0*x*math.pi) + 20.0*math.sin(2.0*x*math.pi))*2.0/3.0
ret += (20.0*math.sin(x*math.pi) + 40.0*math.sin(x/3.0*math.pi))*2.0/3.0
ret += (150.0*math.sin(x/12.0*math.pi) + 300.0*math.sin(x/30.0*math.pi))*2.0/3.0
return ret
def fallback_geocode(province, city, district, address):
if not address:
return None, None, None, "pending_no_address", 0.0
key = f"{city}{district}" if district else city
if province not in {"北京市", "上海市"} and city:
key = f"{province}{city}{district or ''}"
coord = ADMIN_CENTROIDS.get(key)
if coord:
precision = "district_centroid" if district else "city_centroid"
confidence = 0.25 if district else 0.10
return coord[0], coord[1], key, precision, confidence
city_key = f"{province}{city}" if province != city else city
coord = ADMIN_CENTROIDS.get(city_key)
if coord:
return coord[0], coord[1], city_key, "city_centroid", 0.10
return None, None, None, "pending_exact_geocode", 0.0
# ============================================================
# 导入函数
# ============================================================
def import_store_location(conn, filepath):
"""导入门店位置主表和源行表"""
source_file = os.path.basename(filepath)
print(f"\n=== 门店位置导入 ===")
print(f"文件: {source_file}")
cur = conn.cursor()
# 幂等检查
cur.execute("SELECT 1 FROM public.store_location_master WHERE source_file = %s LIMIT 1", (source_file,))
if cur.fetchone():
print(" 位置数据已导入过，跳过位置导入")
# 仍然需要重建销售门店映射
_rebuild_sales_mapping(conn)
cur.close()
return
# 读取Excel
df = pd.read_excel(filepath, header=0, engine='xlrd')
print(f" 总行数: {len(df)}")
# 清理数据
raw_rows = []
numbered = []
for idx, row in df.iterrows():
row_no = idx + 2 # 1-indexed from row 2
seq = row.iloc[0]
seq_int = int(seq) if isinstance(seq, (int, float)) and not pd.isna(seq) else None
raw = {
"row": row_no, "seq": seq_int,
"name": clean(row.iloc[1]) if len(row) > 1 else None,
"company": clean(row.iloc[2]) if len(row) > 2 else None,
"brand": clean(row.iloc[3]) if len(row) > 3 else None,
"address": clean(row.iloc[4]) if len(row) > 4 else None,
"area": clean(row.iloc[5]) if len(row) > 5 else None,
"opened": clean(row.iloc[6]) if len(row) > 6 else None,
"lease": clean(row.iloc[7]) if len(row) > 7 else None,
"license": clean(row.iloc[8]) if len(row) > 8 else None,
"note": clean(row.iloc[9]) if len(row) > 9 else None,
}
raw_rows.append(raw)
if seq_int is not None:
numbered.append(raw)
print(f" 编号门店数: {len(numbered)}")
# 清理旧数据
cur.execute("TRUNCATE public.sales_store_location_mapping RESTART IDENTITY CASCADE")
cur.execute("DELETE FROM public.store_location_master WHERE source_file = %s OR source_file = 'sales_db_placeholder'", (source_file,))
cur.execute("DELETE FROM public.store_location_source_rows WHERE source_file = %s", (source_file,))
# 导入源行
source_values = [(
source_file, r["row"], r["seq"], r["name"], r["company"], r["brand"], r["address"],
r["area"], r["opened"], r["lease"], r["license"], r["note"]
) for r in raw_rows]
execute_values(cur, """
INSERT INTO public.store_location_source_rows
(source_file, source_row, source_store_no, store_short_name, company_name, brand_name,
business_address, area_raw, opened_raw, lease_expiry_raw, license_raw, note)
VALUES %s
""", source_values, page_size=500)
print(f" 源行导入: {len(source_values)} 行")
# 构建主表数据
masters = []
for r in numbered:
area_sqm, area_method = parse_area(r["area"])
province, city, district = parse_admin(r["address"])
lat, lon, display, precision, confidence = fallback_geocode(province, city, district, r["address"])
gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
name = r["name"] or f"未命名门店{r['seq']}"
if "停业" in name:
status = "停业"
elif r["opened"] == "未开业":
status = "未开业"
else:
status = "在册"
masters.append({
**r, "area_sqm": area_sqm, "area_method": area_method,
"opened_date": parse_date(r["opened"]), "lease_date": parse_date(r["lease"]),
"license_date": parse_date(r["license"]), "status": status,
"province": province, "city": city, "district": district,
"lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon,
"display": display, "precision": precision, "confidence": confidence,
})
# 占位门店
placeholders = [
{"name": "甄选商城店", "address": None, "province": None, "city": None, "district": None,
"status": "线上虚拟门店", "precision": "not_applicable", "display": None, "confidence": 0.0},
{"name": "西安含光店", "address": "陕西省西安市含光路（具体门牌待补）",
"province": "陕西省", "city": "西安市", "district": None,
"status": "地址待补", "precision": "city_centroid", "display": "陕西省西安市", "confidence": 0.10},
]
for i, p in enumerate(placeholders, start=1):
coord = ADMIN_CENTROIDS.get(f"{p['province']}{p['city']}") if p["province"] else None
lat, lon = coord if coord else (None, None)
gcj_lat, gcj_lon = wgs84_to_gcj02(lat, lon)
masters.append({
"row": None, "seq": 9000 + i, "name": p["name"], "company": None,
"brand": "西部马华牛肉面", "address": p["address"], "area": None,
"opened": None, "lease": None, "license": None, "note": "经营数据占位记录",
"area_sqm": None, "area_method": "missing", "opened_date": None,
"lease_date": None, "license_date": None, "status": p["status"],
"province": p["province"], "city": p["city"], "district": p["district"],
"lat": lat, "lon": lon, "gcj_lat": gcj_lat, "gcj_lon": gcj_lon,
"display": p["display"], "precision": p["precision"], "confidence": p["confidence"],
})
# 导入主表
location_ids = {}
for m in masters:
provider = "offline_admin_centroid" if m["lat"] is not None else "none"
geocode_status = "fallback_low_precision" if m["lat"] is not None else m["precision"]
src_file = source_file if m["seq"] < 9000 else "sales_db_placeholder"
cur.execute("""
INSERT INTO public.store_location_master
(source_file, source_row, source_store_no, store_short_name, company_name, brand_name,
business_address, area_raw, area_sqm, area_parse_method, opened_raw, opened_date,
lease_expiry_raw, lease_expiry_date, license_raw, license_date, operating_status,
province, city, district, geocode_query, geocode_provider, geocode_status,
geocode_precision, geocode_confidence, geocode_display_name,
latitude_wgs84, longitude_wgs84, latitude_gcj02, longitude_gcj02, geocode_raw, geocoded_at)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,
%s,%s,%s,%s,%s, now())
RETURNING location_id
""", (
src_file, m["row"], m["seq"], m["name"], m["company"], m["brand"], m["address"],
m["area"], m["area_sqm"], m["area_method"], m["opened"], m["opened_date"],
m["lease"], m["lease_date"], m["license"], m["license_date"], m["status"],
m["province"], m["city"], m["district"], m["address"],
provider, geocode_status, m["precision"], m["confidence"], m["display"],
m["lat"], m["lon"], m["gcj_lat"], m["gcj_lon"],
Json({"notice": "行政区中心点兜底，不是门店精确坐标"}) if m["lat"] is not None else None
))
location_ids[m["name"]] = cur.fetchone()[0]
print(f" 主表导入: {len(masters)} 家门店")
# 从DB读取location_ids用于映射
cur.execute("SELECT location_id, store_short_name FROM public.store_location_master")
location_ids = {name: lid for lid, name in cur.fetchall()}
_rebuild_sales_mapping(conn, location_ids, masters)
conn.commit()
cur.close()
print(f"门店位置导入完成")
def _rebuild_sales_mapping(conn, location_ids=None, masters=None):
"""重建销售门店→位置映射表"""
cur = conn.cursor()
# 如果没有传入location_ids，从DB读取
if location_ids is None:
cur.execute("SELECT location_id, store_short_name FROM public.store_location_master")
location_ids = {name: lid for lid, name in cur.fetchall()}
if not location_ids:
print(" 跳过销售门店映射: 无位置数据")
cur.close()
return
# 清理旧映射
cur.execute("TRUNCATE public.sales_store_location_mapping")
# 获取销售门店列表
try:
cur.execute("SELECT store_code, store_name FROM analytics.v_store_scorecard ORDER BY store_name")
sales_stores = cur.fetchall()
except Exception:
sales_stores = []
print(" 警告: analytics.v_store_scorecard 不存在，跳过销售门店映射")
cur.close()
return
# 构建候选列表
if masters:
source_candidates = [(m["name"], normalize_name(m["name"])) for m in masters]
else:
source_candidates = [(name, normalize_name(name)) for name in location_ids.keys()]
mapped = []
for store_code, store_name in sales_stores:
target_name = MANUAL_ALIAS.get(store_name)
method = "manual_alias" if target_name else "normalized_name"
score = 1.0 if target_name else 0.0
if not target_name:
nn = normalize_name(store_name)
exact = [x for x in source_candidates if x[1] == nn and nn]
if exact:
target_name, _ = exact[0]
score = 1.0
else:
ranked = []
for candidate, cn in source_candidates:
s = SequenceMatcher(None, nn, cn).ratio()
if nn and cn and (nn in cn or cn in nn):
s = max(s, 0.92)
ranked.append((s, candidate))
if ranked:
score, target_name = max(ranked)
method = "fuzzy_name"
location_id = location_ids.get(target_name)
if not location_id or score < 0.60:
print(f" 警告: 经营门店无法映射: {store_code} {store_name} -> {target_name} score={score}")
continue
status = "confirmed" if method in {"manual_alias", "normalized_name"} or score >= 0.85 else "reviewed_low_confidence"
note = None if status == "confirmed" else "名称相似匹配，建议业务复核"
mapped.append((store_code, store_name, location_id, method, score, status, note))
if mapped:
execute_values(cur, """
INSERT INTO public.sales_store_location_mapping
(sales_store_code, sales_store_name, location_id, mapping_method, mapping_confidence, mapping_status, review_note)
VALUES %s
""", mapped, page_size=500)
conn.commit()
print(f" 销售门店映射: {len(mapped)} 家")
cur.close()
def import_store_name_mapping(conn):
"""导入薪资名称↔账单名称映射（静态8条）"""
cur = conn.cursor()
cur.execute("SELECT count(*) FROM public.store_name_mapping")
if cur.fetchone()[0] > 0:
print("store_name_mapping已有数据，跳过")
cur.close()
return
execute_values(
cur,
"INSERT INTO public.store_name_mapping (salary_name, bill_name) VALUES %s",
STORE_NAME_MAPPING,
page_size=100
)
conn.commit()
print(f"store_name_mapping导入完成: {len(STORE_NAME_MAPPING)} 条")
cur.close()
def import_inventory_store_mapping(conn):
"""从库存成本数据推导库存门店映射"""
cur = conn.cursor()
cur.execute("SELECT count(*) FROM public.inventory_store_mapping")
if cur.fetchone()[0] > 0:
print("inventory_store_mapping已有数据，跳过")
cur.close()
return
# 从库存成本记录中提取唯一的成本单位编码和名称
cur.execute("""
SELECT DISTINCT cost_unit_code, cost_unit_name
FROM public.inventory_cost_records
WHERE report_month = '2026-04-01'
AND cost_unit_code IS NOT NULL
ORDER BY cost_unit_code
""")
cost_units = cur.fetchall()
mappings = []
for code, name in cost_units:
# 成本单位编码与销售门店编码相同的直接映射
mappings.append((
code, name, code, name,
'经营门店', True, '门店编码精确匹配', 100.00, None
))
if mappings:
execute_values(cur, """
INSERT INTO public.inventory_store_mapping
(cost_unit_code, cost_unit_name, sales_store_code, sales_store_name,
unit_type, include_in_operating_cost, mapping_method, mapping_confidence, review_note)
VALUES %s
""", mappings, page_size=500)
conn.commit()
print(f"inventory_store_mapping导入完成: {len(mappings)} 条")
else:
print("inventory_store_mapping: 无库存成本数据可推导")
cur.close()
def import_operating_expense_store_mapping(conn):
"""从营业费用数据推导费用门店映射"""
cur = conn.cursor()
cur.execute("SELECT count(*) FROM public.operating_expense_store_mapping")
if cur.fetchone()[0] > 0:
print("operating_expense_store_mapping已有数据，跳过")
cur.close()
return
# 从营业费用记录中提取唯一的成本单位名称
cur.execute("""
SELECT DISTINCT cost_unit_source_name
FROM public.operating_expense_records
WHERE report_month = '2026-04-01'
AND cost_unit_source_name IS NOT NULL
ORDER BY cost_unit_source_name
""")
expense_units = cur.fetchall()
# 尝试从销售数据获取门店列表做匹配
try:
cur.execute("SELECT store_code, store_name FROM analytics.dim_store ORDER BY store_code")
sales_stores = cur.fetchall()
except Exception:
sales_stores = []
mappings = []
for (source_name,) in expense_units:
# 标准化名称：去掉"金额"后缀
normalized = re.sub(r"金额$", "", source_name).strip()
# 尝试精确匹配
matched_code = None
matched_name = None
method = "unmapped"
confidence = 0.0
for sc, sn in sales_stores:
if normalized == sn or normalized in sn or sn in normalized:
matched_code = sc
matched_name = sn
method = "标准名称精确匹配"
confidence = 100.0
break
if not matched_code:
# 尝试编码前缀匹配
m = re.match(r"^(\d+)\s+", normalized)
if m:
prefix = m.group(1)
for sc, sn in sales_stores:
if sc == prefix:
matched_code = sc
matched_name = sn
method = "编码前缀匹配"
confidence = 90.0
break
mappings.append((
source_name, normalized,
matched_code, matched_name,
'经营门店', matched_code is not None,
method if matched_code else "待人工确认",
confidence,
None if matched_code else "需人工确认匹配关系"
))
if mappings:
execute_values(cur, """
INSERT INTO public.operating_expense_store_mapping
(cost_unit_source_name, normalized_cost_unit_name, sales_store_code, sales_store_name,
unit_type, include_in_operating_analysis, mapping_method, mapping_confidence, review_note)
VALUES %s
""", mappings, page_size=500)
conn.commit()
print(f"operating_expense_store_mapping导入完成: {len(mappings)} 条")
else:
print("operating_expense_store_mapping: 无费用数据可推导")
cur.close()
# ============================================================
# Main
# ============================================================
def main():
parser = argparse.ArgumentParser(description='门店位置与映射表导入')
parser.add_argument('--file', required=True, help='各店信息Excel文件路径')
parser.add_argument('--db', default='bill_query', help='目标数据库 (默认bill_query)')
parser.add_argument('--skip-location', action='store_true', help='跳过门店位置导入')
parser.add_argument('--skip-mappings', action='store_true', help='跳过映射表导入')
args = parser.parse_args()
conn = psycopg2.connect(host='localhost', port=5432, dbname=args.db, user='freedak')
conn.autocommit = False
try:
if not args.skip_location:
import_store_location(conn, args.file)
if not args.skip_mappings:
print("\n=== 映射表导入 ===")
import_store_name_mapping(conn)
import_inventory_store_mapping(conn)
import_operating_expense_store_mapping(conn)
print("\n=== 导入完成 ===")
except Exception as e:
conn.rollback()
print(f"错误: {e}", file=sys.stderr)
raise
finally:
conn.close()
if __name__ == '__main__':
main()