feat: 热词热力图改为Treemap矩形堆砌布局，面积代表提及次数；赛道分析页面布局调整

backend/crawler.py

@@ -36,6 +36,7 @@ load_dotenv(BASE_DIR / ".env")
 
 DEFAULT_LIST_URL = "https://m.canyin88.com/zixun/"
 ARTICLE_PATH_RE = re.compile(r"/zixun/\d{4}/\d{1,2}/\d{1,2}/\d+\.html$")
+CANYIN168_ARTICLE_RE = re.compile(r"/Article/[a-z]+/\d+\.html$")
 DATE_RE = re.compile(r"(20\d{2})[-/.年](\d{1,2})[-/.月](\d{1,2})(?:日)?(?:\s+(\d{1,2}):(\d{2})(?::(\d{2}))?)?")
 
 
@@ -47,9 +48,11 @@ def normalize_url(raw_url: str, base_url: str = DEFAULT_LIST_URL) -> str | None:
     absolute = urljoin(base_url, raw_url.strip())
     parts = urlsplit(absolute)
     host = parts.netloc.lower().removeprefix("m.").removeprefix("www.")
-    if host != "canyin88.com" or not ARTICLE_PATH_RE.search(parts.path):
-        return None
-    return urlunsplit(("https", "www.canyin88.com", parts.path, "", ""))
+    if host == "canyin88.com" and ARTICLE_PATH_RE.search(parts.path):
+        return urlunsplit(("https", "www.canyin88.com", parts.path, "", ""))
+    if host == "canyin168.com" and CANYIN168_ARTICLE_RE.search(parts.path):
+        return urlunsplit(("http", "www.canyin168.com", parts.path, "", ""))
+    return None
 
 
 def clean_text(value: str | None) -> str:
@@ -138,7 +141,7 @@ def extract_article(page, url: str) -> dict[str, str | None]:
     page.goto(url, wait_until="domcontentloaded", timeout=60_000)
     page.wait_for_timeout(800)
     document_title = clean_text(page.title())
-    document_title = re.sub(r"(?:[_-]红餐网.*)$", "", document_title).strip()
+    document_title = re.sub(r"(?:[_-](?:红餐网|职业餐饮网).*)$", "", document_title).strip()
     title = meta(page, 'meta[property="og:title"]') or document_title
     if not title or title == "相关推荐":
         title = first_text(page, [".title", "h1"])
@@ -197,17 +200,17 @@ def pending_urls(conn: sqlite3.Connection, discovered: list[str], max_retries: i
     return list(dict.fromkeys(discovered + [row[0] for row in rows]))
 
 
-def insert_discoveries(conn: sqlite3.Connection, urls: list[str], source_type: str = "公众号") -> int:
+def insert_discoveries(conn: sqlite3.Connection, urls: list[str], source_type: str = "公众号", source_name: str = "") -> int:
     timestamp = now_iso()
     inserted = 0
     for url in urls:
         cursor = conn.execute(
             """
             INSERT OR IGNORE INTO articles
-                (canonical_url, source_type, discovered_at, updated_at)
-            VALUES (?, ?, ?, ?)
+                (canonical_url, source_type, source_name, discovered_at, updated_at)
+            VALUES (?, ?, ?, ?, ?)
             """,
-            (url, source_type, timestamp, timestamp),
+            (url, source_type, source_name, timestamp, timestamp),
         )
         inserted += cursor.rowcount
     conn.commit()
@@ -439,7 +442,7 @@ def run_crawl(args: argparse.Namespace) -> int:
                             print(f"--- 采集公众号: {source_name} ({list_url}) ---", flush=True)
                             urls = discover_urls(page, list_url, max_scrolls, list_pause_ms)
                             discovered_count += len(urls)
-                            inserted_count += insert_discoveries(conn, urls, source_name)
+                            inserted_count += insert_discoveries(conn, urls, "公众号", source_name)
                             queue = pending_urls(conn, urls, args.max_retries)
                             cutoff = datetime.now() - timedelta(days=lookback_days)
                             for index, url in enumerate(queue, start=1):
@@ -551,7 +554,7 @@ def run_crawl(args: argparse.Namespace) -> int:
             "status": status, "source_type": args.source_type,
             "discovered": discovered_count, "inserted": inserted_count,
             "updated": updated_count, "failed": failed_count,
-        }, ensure_ascii=False, flush=True))
+        }, ensure_ascii=False))
         return 0 if status in {"success", "partial"} else 1