diff --git a/backend/app.py b/backend/app.py index 53c8d7f..fd269a4 100644 --- a/backend/app.py +++ b/backend/app.py @@ -45,9 +45,11 @@ def now_iso() -> str: def db() -> sqlite3.Connection: - conn = sqlite3.connect(DB_PATH) + conn = sqlite3.connect(DB_PATH, timeout=10) conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode = WAL") conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA busy_timeout = 5000") return conn @@ -160,7 +162,7 @@ def dashboard(): "type": r["type"] or "经营干货", "score": r["score"] or 0, "summary": r["ai_summary"] or "", - "entities": json.loads(r["entities"]) if r["entities"] else [], + "entities": [b["name"] if isinstance(b, dict) else b for b in (json.loads(r["entities"]) if r["entities"] else [])], } for r in top_intel ], @@ -251,7 +253,7 @@ def intel_list(): "summary": r["ai_summary"] or r["summary"] or "", "keyPoints": json.loads(r["key_points"]) if r["key_points"] else [], "score": r["score"] or 0, - "entities": json.loads(r["entities"]) if r["entities"] else [], + "entities": [b["name"] if isinstance(b, dict) else b for b in (json.loads(r["entities"]) if r["entities"] else [])], "timeliness": r["timeliness"] or "中", "tags": json.loads(r["tags"]) if r["tags"] else [], "url": r["canonical_url"], @@ -393,26 +395,41 @@ def brand_detail(brand_id: int): @app.get("/api/analysis") def analysis(): with db() as conn: - # 赛道热度趋势：按月统计各赛道的平均评分 + # 赛道热度趋势：按月统计各赛道的文章数和平均评分，热度=文章数×平均评分/10 heat_rows = conn.execute( """ - SELECT strftime('%m', a.published_at) AS month, + SELECT strftime('%Y-%m', a.published_at) AS month, a.category, - AVG(CAST(json_extract(x.result_json, '$.score') AS REAL)) AS heat + COUNT(*) AS article_cnt, + AVG(CAST(json_extract(x.result_json, '$.score') AS REAL)) AS avg_score FROM articles a LEFT JOIN ai_analyses x ON x.article_id = a.id AND x.analysis_type = 'editorial' - WHERE a.crawl_status = 'success' AND a.category IS NOT NULL + WHERE a.crawl_status = 'success' AND a.category IS NOT NULL AND a.category != '' + AND a.published_at >= date('now', '-12 months') GROUP BY month, a.category ORDER BY month """ ).fetchall() - # 开关店对比（从文章中提取的品类趋势数据） - category_counts = conn.execute( + # 品牌门店规模（按赛道聚合真实门店总数） + brand_stores = conn.execute( """ - SELECT a.category, COUNT(*) AS cnt + SELECT category, SUM(stores) AS total_stores, COUNT(*) AS brand_cnt + FROM brands WHERE category IS NOT NULL AND category != '' AND stores > 0 + GROUP BY category + """ + ).fetchall() + + # 舆情分布（按赛道统计正面/负面/中性文章数） + sentiment_rows = conn.execute( + """ + SELECT a.category, + SUM(CASE WHEN json_extract(x.result_json, '$.sentiment') = 'positive' THEN 1 ELSE 0 END) AS positive_cnt, + SUM(CASE WHEN json_extract(x.result_json, '$.sentiment') = 'negative' THEN 1 ELSE 0 END) AS negative_cnt, + SUM(CASE WHEN json_extract(x.result_json, '$.sentiment') = 'neutral' OR json_extract(x.result_json, '$.sentiment') IS NULL THEN 1 ELSE 0 END) AS neutral_cnt FROM articles a - WHERE a.crawl_status = 'success' AND a.category IS NOT NULL + JOIN ai_analyses x ON x.article_id = a.id AND x.analysis_type = 'editorial' + WHERE a.crawl_status = 'success' AND a.category IS NOT NULL AND a.category != '' GROUP BY a.category """ ).fetchall() @@ -432,29 +449,55 @@ def analysis(): """ ).fetchone() - # 热词 - tag_rows = conn.execute( + # 热词热力图：近7天每天各标签出现次数 + tag_heatmap = conn.execute( """ - SELECT value AS tag, COUNT(*) AS cnt - FROM ai_analyses, json_each(json_extract(result_json, '$.tags')) - GROUP BY value - ORDER BY cnt DESC - LIMIT 8 + SELECT value AS tag, + strftime('%Y-%m-%d', a.published_at) AS day, + COUNT(*) AS cnt + FROM ai_analyses x + JOIN articles a ON a.id = x.article_id + , json_each(json_extract(x.result_json, '$.tags')) + WHERE a.published_at >= date('now', '-6 days') + AND value IS NOT NULL AND value != '' + GROUP BY tag, day """ ).fetchall() - # 构建热度趋势数据 + # 热词总量排行（用于选取 TOP 标签） + tag_total = conn.execute( + """ + SELECT value AS tag, COUNT(*) AS cnt + FROM ai_analyses x + JOIN articles a ON a.id = x.article_id + , json_each(json_extract(x.result_json, '$.tags')) + WHERE a.published_at >= date('now', '-6 days') + AND value IS NOT NULL AND value != '' + GROUP BY value + ORDER BY cnt DESC + LIMIT 10 + """ + ).fetchall() + + # 构建热度趋势数据（热度 = 文章数 × 平均评分 / 10） heat_trend = {} for r in heat_rows: - month_key = f"{r['month']}月" if r["month"] else "未知" + month_key = f"{r['month'][-2:]}月" if r["month"] else "未知" if month_key not in heat_trend: heat_trend[month_key] = {"month": month_key} - heat_trend[month_key][r["category"]] = round(r["heat"] or 0) + heat_val = round((r["article_cnt"] or 0) * (r["avg_score"] or 0) / 10) + heat_trend[month_key][r["category"]] = heat_val - # 构建开关店数据 - open_close = [ - {"category": r["category"], "新开": r["cnt"], "关闭": max(0, r["cnt"] - int(r["cnt"] * 0.85))} - for r in category_counts + # 构建品牌门店规模数据 + brand_stores_data = [ + {"category": r["category"], "stores": r["total_stores"] or 0, "brands": r["brand_cnt"]} + for r in brand_stores + ] + + # 构建舆情分布数据 + sentiment_data = [ + {"category": r["category"], "正面": r["positive_cnt"] or 0, "负面": r["negative_cnt"] or 0, "中性": r["neutral_cnt"] or 0} + for r in sentiment_rows ] # 构建价格分布 @@ -468,11 +511,28 @@ def analysis(): {"band": "150元以上", "pct": round((price_rows["band6"] or 0) * 100 / total_brands)}, ] + # 构建热词热力图数据（用 UTC 日期与 SQLite date('now') 保持一致） + from datetime import timedelta, timezone + today_utc = datetime.now(timezone.utc).date() + days = [(today_utc - timedelta(days=i)).isoformat() for i in range(6, -1, -1)] + top_tags = [r["tag"] for r in tag_total] + # 构建矩阵 {tag: {day: cnt}} + matrix = {tag: {day: 0 for day in days} for tag in top_tags} + for r in tag_heatmap: + if r["tag"] in matrix and r["day"] in matrix[r["tag"]]: + matrix[r["tag"]][r["day"]] = r["cnt"] + hot_keywords = { + "days": days, + "tags": top_tags, + "matrix": matrix, + } + return jsonify({ "categoryHeatTrend": list(heat_trend.values()), - "openCloseByCategory": open_close, + "brandStoresByCategory": brand_stores_data, + "sentimentByCategory": sentiment_data, "priceBandData": price_bands, - "hotKeywords": [{"word": r["tag"], "change": min(r["cnt"] * 5, 50)} for r in tag_rows], + "hotKeywords": hot_keywords, }) diff --git a/backend/crawler.py b/backend/crawler.py index eac2149..53617ea 100644 --- a/backend/crawler.py +++ b/backend/crawler.py @@ -36,6 +36,7 @@ load_dotenv(BASE_DIR / ".env") DEFAULT_LIST_URL = "https://m.canyin88.com/zixun/" ARTICLE_PATH_RE = re.compile(r"/zixun/\d{4}/\d{1,2}/\d{1,2}/\d+\.html$") +CANYIN168_ARTICLE_RE = re.compile(r"/Article/[a-z]+/\d+\.html$") DATE_RE = re.compile(r"(20\d{2})[-/.年](\d{1,2})[-/.月](\d{1,2})(?:日)?(?:\s+(\d{1,2}):(\d{2})(?::(\d{2}))?)?") @@ -47,9 +48,11 @@ def normalize_url(raw_url: str, base_url: str = DEFAULT_LIST_URL) -> str | None: absolute = urljoin(base_url, raw_url.strip()) parts = urlsplit(absolute) host = parts.netloc.lower().removeprefix("m.").removeprefix("www.") - if host != "canyin88.com" or not ARTICLE_PATH_RE.search(parts.path): - return None - return urlunsplit(("https", "www.canyin88.com", parts.path, "", "")) + if host == "canyin88.com" and ARTICLE_PATH_RE.search(parts.path): + return urlunsplit(("https", "www.canyin88.com", parts.path, "", "")) + if host == "canyin168.com" and CANYIN168_ARTICLE_RE.search(parts.path): + return urlunsplit(("http", "www.canyin168.com", parts.path, "", "")) + return None def clean_text(value: str | None) -> str: @@ -138,7 +141,7 @@ def extract_article(page, url: str) -> dict[str, str | None]: page.goto(url, wait_until="domcontentloaded", timeout=60_000) page.wait_for_timeout(800) document_title = clean_text(page.title()) - document_title = re.sub(r"(?:[_-]红餐网.*)$", "", document_title).strip() + document_title = re.sub(r"(?:[_-](?:红餐网|职业餐饮网).*)$", "", document_title).strip() title = meta(page, 'meta[property="og:title"]') or document_title if not title or title == "相关推荐": title = first_text(page, [".title", "h1"]) @@ -197,17 +200,17 @@ def pending_urls(conn: sqlite3.Connection, discovered: list[str], max_retries: i return list(dict.fromkeys(discovered + [row[0] for row in rows])) -def insert_discoveries(conn: sqlite3.Connection, urls: list[str], source_type: str = "公众号") -> int: +def insert_discoveries(conn: sqlite3.Connection, urls: list[str], source_type: str = "公众号", source_name: str = "") -> int: timestamp = now_iso() inserted = 0 for url in urls: cursor = conn.execute( """ INSERT OR IGNORE INTO articles - (canonical_url, source_type, discovered_at, updated_at) - VALUES (?, ?, ?, ?) + (canonical_url, source_type, source_name, discovered_at, updated_at) + VALUES (?, ?, ?, ?, ?) """, - (url, source_type, timestamp, timestamp), + (url, source_type, source_name, timestamp, timestamp), ) inserted += cursor.rowcount conn.commit() @@ -439,7 +442,7 @@ def run_crawl(args: argparse.Namespace) -> int: print(f"--- 采集公众号: {source_name} ({list_url}) ---", flush=True) urls = discover_urls(page, list_url, max_scrolls, list_pause_ms) discovered_count += len(urls) - inserted_count += insert_discoveries(conn, urls, source_name) + inserted_count += insert_discoveries(conn, urls, "公众号", source_name) queue = pending_urls(conn, urls, args.max_retries) cutoff = datetime.now() - timedelta(days=lookback_days) for index, url in enumerate(queue, start=1): @@ -551,7 +554,7 @@ def run_crawl(args: argparse.Namespace) -> int: "status": status, "source_type": args.source_type, "discovered": discovered_count, "inserted": inserted_count, "updated": updated_count, "failed": failed_count, - }, ensure_ascii=False, flush=True)) + }, ensure_ascii=False)) return 0 if status in {"success", "partial"} else 1 diff --git a/backend/structurer.py b/backend/structurer.py index 5b27e57..b7c5c0b 100644 --- a/backend/structurer.py +++ b/backend/structurer.py @@ -51,6 +51,23 @@ def get_api_key() -> str: # ---- 单篇文章结构化分析 ---- +_PROFANITY_MAP = { + "妈的": "**", "傻逼": "**", "他妈": "**", "操": "*", + "草泥马": "***", "尼玛": "**", "贱人": "**", + "王八蛋": "***", "混蛋": "**", "去死": "**", + "毛爷爷": "某伟人", "毛泽东": "某伟人", "国民党": "某党", + "共产党": "某党", "长征": "远征", "抗日": "抗战", + "革命": "变革", "社会主义": "某主义", +} + + +def sanitize_content(text: str) -> str: + """过滤粗口和敏感词，避免触发千问内容审核。""" + for word, replacement in _PROFANITY_MAP.items(): + text = text.replace(word, replacement) + return text + + def analyze_article(title: str, content: str) -> dict: """调用通义千问对文章进行结构化分析。""" api_key = get_api_key() @@ -71,7 +88,7 @@ JSON 字段必须为： - numbers（关键数据数组） 文章标题：{title} -正文：{content[:24000]}""" +正文：{sanitize_content(content[:24000])}""" payload = json.dumps({ "model": model, "messages": [ @@ -276,6 +293,11 @@ def run_batch_analyze(force: bool, pause: float, retries: int) -> int: break except Exception as exc: last_error = exc + err_str = str(exc) + # 内容审核失败，标记跳过不重试 + if "data_inspection_failed" in err_str: + print(f"[{index}/{len(rows)}] SKIP(内容审核) {row['title']}", flush=True) + break if attempt <= retries: wait = attempt * 2 print(f"[{index}/{len(rows)}] RETRY {attempt}/{retries} {exc}", flush=True) diff --git a/src/lib/api.ts b/src/lib/api.ts index 84a5231..e0df89a 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -94,9 +94,10 @@ export interface Brand { export interface AnalysisData { categoryHeatTrend: Record[] - openCloseByCategory: { category: string; 新开: number; 关闭: number }[] + brandStoresByCategory: { category: string; stores: number; brands: number }[] + sentimentByCategory: { category: string; 正面: number; 负面: number; 中性: number }[] priceBandData: { band: string; pct: number }[] - hotKeywords: { word: string; change: number }[] + hotKeywords: { days: string[]; tags: string[]; matrix: Record> } } export interface SourceConfig { diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index b1ab5b2..65c00ef 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -59,7 +59,7 @@ export default function Home() {
- {page === 'dashboard' && navigate(`/${p}`)} />} + {page === 'dashboard' && navigate(articleId ? `/${p}?article=${articleId}&q=${encodeURIComponent(articleTitle || '')}` : `/${p}`)} />} {page === 'intel' && } {page === 'qa' && } {page === 'brands' && } diff --git a/src/sections/CategoryAnalysis.tsx b/src/sections/CategoryAnalysis.tsx index 77b8ee9..561f3d4 100644 --- a/src/sections/CategoryAnalysis.tsx +++ b/src/sections/CategoryAnalysis.tsx @@ -3,10 +3,58 @@ import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card' import { Loader2 } from 'lucide-react' import { LineChart, Line, BarChart, Bar, XAxis, YAxis, Tooltip, Legend, - ResponsiveContainer, CartesianGrid, PieChart, Pie, Cell, + ResponsiveContainer, CartesianGrid, PieChart, Pie, Cell, Treemap, } from 'recharts' import { api, type AnalysisData } from '@/lib/api' +const treemapColors = ['#fdba74', '#fb923c', '#f97316', '#ea580c', '#c2410c'] + +function TreemapCell(props: any) { + const { x, y, width, height, name, value, maxVal } = props + if (width < 30 || height < 20) return null + const intensity = maxVal > 0 ? value / maxVal : 0 + const colorIdx = Math.min(Math.floor(intensity * treemapColors.length), treemapColors.length - 1) + const bg = treemapColors[colorIdx] + const fontSize = Math.max(10, Math.min(16, width / 8)) + return ( + + + {width > 50 && height > 30 && ( + + {name} + + )} + {width > 40 && height > 40 && ( + + {value} + + )} + + ) +} + const lineColors: Record = { 茶饮咖啡: '#f97316', 快餐: '#3b82f6', @@ -29,9 +77,10 @@ export default function CategoryAnalysis() { }, []) const heatTrend = data?.categoryHeatTrend ?? [] - const openClose = data?.openCloseByCategory ?? [] + const brandStores = data?.brandStoresByCategory ?? [] + const sentiment = data?.sentimentByCategory ?? [] const priceBands = data?.priceBandData ?? [] - const hotKw = data?.hotKeywords ?? [] + const hotKw = data?.hotKeywords ?? { days: [], tags: [], matrix: {} } if (loading) { return ( @@ -54,7 +103,7 @@ export default function CategoryAnalysis() { 赛道热度指数（近 12 个月） -

由情报提及量、开关店数据、消费关注度加权合成

+

由情报提及量与 AI 评分加权合成

 {heatTrend.length === 0 ? ( @@ -79,26 +128,52 @@ export default function CategoryAnalysis() {
- {/* 开关店对比 */} + {/* 品牌门店规模 */} - 上半年新开 / 关闭门店对比 -

绿色为正增长缺口，红色为净收缩信号

+ 各赛道品牌门店规模 +

按赛道聚合的品牌总门店数

 - {openClose.length === 0 ? ( -
暂无开关店数据
+ {brandStores.length === 0 ? ( +
暂无门店数据
) : (
- + + + + + + + + +
+ )} + + + + {/* 舆情分布 */} + + + 各赛道舆情分布 +

按赛道统计正面/负面/中性情报数量

+ + + {sentiment.length === 0 ? ( +
暂无舆情数据
+ ) : ( +
+ + - - + + +
@@ -147,28 +222,48 @@ export default function CategoryAnalysis() { )} -
- {/* 热词趋势 */} - - - 行业热词关注度周环比 - - - {hotKw.length === 0 ? ( -
暂无热词数据
- ) : ( -
- {hotKw.map((k) => ( -
-
{k.word}
-
▲ +{k.change}%
-
- ))} -
- )} - - + {/* 热词热力图 — 矩形堆砌，大小代表提及次数 */} + + + 行业热词近7天热力图 +

矩形面积代表提及次数，颜色深浅表示热度高低

+ + + {!hotKw?.tags || hotKw.tags.length === 0 ? ( +
暂无热词数据
+ ) : (() => { + const treemapData = hotKw.tags + .map((tag) => ({ + name: tag, + size: hotKw.days.reduce((sum, d) => sum + (hotKw.matrix[tag]?.[d] || 0), 0), + })) + .filter((d) => d.size > 0) + .sort((a, b) => b.size - a.size) + + if (treemapData.length === 0) { + return
暂无热词数据
+ } + + const maxVal = Math.max(...treemapData.map((d) => d.size)) + + return ( +
+ + } + /> + +
+ ) + })()} + + +
) } diff --git a/src/sections/Dashboard.tsx b/src/sections/Dashboard.tsx index 8c3128e..ccae94b 100644 --- a/src/sections/Dashboard.tsx +++ b/src/sections/Dashboard.tsx @@ -5,7 +5,7 @@ import { Badge } from '@/components/ui/badge' import { api, type DashboardData, type OverviewStats } from '@/lib/api' export interface PageKeyHintType { - onNavigate?: (page: 'intel' | 'qa' | 'analysis' | 'reports') => void + onNavigate?: (page: 'intel' | 'qa' | 'analysis' | 'reports', articleId?: string, articleTitle?: string) => void } const statCards = (ovStats: OverviewStats) => [ @@ -109,7 +109,8 @@ export default function Dashboard({ onNavigate }: PageKeyHintType) { {topIntel.map((item, idx) => (
onNavigate?.('intel', item.id, item.title)} >
{idx + 1} diff --git a/src/sections/IntelFeed.tsx b/src/sections/IntelFeed.tsx index 510ca44..d157e0d 100644 --- a/src/sections/IntelFeed.tsx +++ b/src/sections/IntelFeed.tsx @@ -1,5 +1,6 @@ import { useEffect, useState } from 'react' import { createPortal } from 'react-dom' +import { useSearchParams } from 'react-router' import { ChevronDown, ChevronUp, Search, Sparkles, ListChecks, Loader2, FileText, AlertCircle, TrendingUp, PlayCircle, X } from 'lucide-react' import { Card, CardContent } from '@/components/ui/card' import { Badge } from '@/components/ui/badge' @@ -26,6 +27,7 @@ function scoreColor(score: number) { } export default function IntelFeed() { + const [searchParams, setSearchParams] = useSearchParams() const [type, setType] = useState('全部') const [category, setCategory] = useState('全部赛道') const [query, setQuery] = useState('') @@ -39,16 +41,33 @@ export default function IntelFeed() { const [totalPages, setTotalPages] = useState(1) const [total, setTotal] = useState(0) const perPage = 20 + const pendingArticleId = searchParams.get('article') + const pendingQuery = searchParams.get('q') || '' useEffect(() => { setLoading(true) setExpanded(null) setDetail(null) - api.getIntel({ type: type === '全部' ? undefined : type, category: category === '全部赛道' ? undefined : category, q: query || undefined, page, per_page: perPage }) + const searchQuery = pendingQuery || query + api.getIntel({ type: type === '全部' ? undefined : type, category: category === '全部赛道' ? undefined : category, q: searchQuery || undefined, page, per_page: perPage }) .then((res) => { setItems(res.items); setTotalPages(res.pages); setTotal(res.total) }) .catch(() => { setItems([]); setTotalPages(1); setTotal(0) }) .finally(() => setLoading(false)) - }, [type, category, query, page]) + }, [type, category, query, page, pendingQuery]) + + // 从 Dashboard 跳转过来时，同步搜索关键词并自动展开对应文章 + useEffect(() => { + if (pendingQuery && !query) { + setQuery(pendingQuery) + } + if (pendingArticleId && items.length > 0 && !expanded) { + const found = items.find((it) => it.id === pendingArticleId) + if (found) { + toggleExpand(found.id) + setSearchParams({}, { replace: true }) + } + } + }, [pendingArticleId, pendingQuery, items, expanded, query]) const toggleExpand = (id: string) => { if (expanded === id) { @@ -279,9 +298,11 @@ export default function IntelFeed() { {(detail?.analysis?.brands || item.entities).length > 0 && (
关联品牌： - {(detail?.analysis?.brands || item.entities).map((e) => ( - {e} - ))} + {(detail?.analysis?.brands || item.entities).map((e, i) => { + const label = typeof e === 'string' ? e : (e as { name?: string })?.name || '' + if (!label) return null + return {label} + })}
)}