freedakgmail
694 lines
29 KiB
Python
694 lines
29 KiB
Python
#!/usr/bin/env python3
|
||
"""结构化引擎：大模型自动打标签、摘要、提取要点、识别品牌实体。
|
||
|
||
对每篇采集入库的文章调用通义千问，输出结构化JSON：
|
||
- summary: 120字内摘要
|
||
- key_points: 3-6条关键要点
|
||
- tags: 5-10个标签
|
||
- brands: 识别到的品牌实体
|
||
- category: 赛道分类
|
||
- type: 情报类型（政策监管/品牌动态/品类趋势/经营干货/供应链/消费洞察）
|
||
- score: 价值评分 0-100
|
||
- timeliness: 时效性（高/中/低）
|
||
- sentiment: 情感倾向
|
||
- content_angles: 可延展选题
|
||
- risk_notes: 事实或表述风险
|
||
- numbers: 关键数据
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
|
||
BASE_DIR = Path(__file__).resolve().parent
|
||
load_dotenv(BASE_DIR / ".env")
|
||
|
||
DASHSCOPE_CHAT_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1/chat/completions"
|
||
DASHSCOPE_EMBED_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings"
|
||
|
||
|
||
def now_iso() -> str:
|
||
return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")
|
||
|
||
|
||
def get_api_key() -> str:
|
||
key = os.getenv("DASHSCOPE_API_KEY", "").strip()
|
||
if not key:
|
||
raise RuntimeError("尚未配置 DASHSCOPE_API_KEY，请在 .env 中设置")
|
||
return key
|
||
|
||
|
||
# ---- 单篇文章结构化分析 ----
|
||
|
||
def analyze_article(title: str, content: str) -> dict:
|
||
"""调用通义千问对文章进行结构化分析。"""
|
||
api_key = get_api_key()
|
||
model = os.getenv("QWEN_MODEL", "qwen-plus")
|
||
prompt = f"""你是一名资深餐饮行业研究编辑。分析下面这篇文章，只返回合法 JSON，不要 Markdown 代码块。
|
||
JSON 字段必须为：
|
||
- summary（120字内摘要）
|
||
- key_points（3-6条字符串，关键要点）
|
||
- tags（5-10个标签）
|
||
- brands（品牌数组，每个元素为对象：{{"name":"品牌名","stores":门店数整数或0,"avg_price":人均消费整数或0,"model":"直营/加盟/直营+加盟"}}，仅填文中明确提及的数据，未提及的填0或空字符串）
|
||
- category（赛道分类，从以下选择：茶饮咖啡/快餐/火锅/正餐/烘焙/供应链/综合）
|
||
- type（情报类型，从以下选择：政策监管/品牌动态/品类趋势/经营干货/供应链/消费洞察）
|
||
- score（价值评分0-100整数，越高越有经营参考价值）
|
||
- timeliness（时效性：高/中/低）
|
||
- sentiment（positive/neutral/negative）
|
||
- content_angles（3条可延展选题）
|
||
- risk_notes（事实或表述风险数组）
|
||
- numbers（关键数据数组）
|
||
|
||
文章标题：{title}
|
||
正文：{content[:24000]}"""
|
||
payload = json.dumps({
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": "你输出严谨、简洁、可供编辑部直接使用的结构化行业分析。"},
|
||
{"role": "user", "content": prompt},
|
||
],
|
||
"temperature": 0.25,
|
||
"response_format": {"type": "json_object"},
|
||
}, ensure_ascii=False).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
DASHSCOPE_CHAT_URL,
|
||
data=payload,
|
||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=90) as response:
|
||
body = json.loads(response.read().decode("utf-8"))
|
||
except urllib.error.HTTPError as exc:
|
||
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
||
raise RuntimeError(f"千问接口返回 {exc.code}：{detail}") from exc
|
||
if "choices" not in body or not body["choices"]:
|
||
raise RuntimeError(f"千问返回异常：{json.dumps(body, ensure_ascii=False)[:300]}")
|
||
text = body["choices"][0]["message"]["content"].strip()
|
||
if text.startswith("```"):
|
||
text = text.strip("`").removeprefix("json").strip()
|
||
try:
|
||
return json.loads(text)
|
||
except Exception as e:
|
||
raise RuntimeError(f"JSON解析失败: {e}\n{text[:200]}") from e
|
||
|
||
|
||
# ---- 跨文章行业信号提取 ----
|
||
|
||
def extract_industry_signals(items: list[dict], model: str) -> list[dict]:
|
||
"""从多篇文章的结构化分析中提取跨文章行业信号。"""
|
||
api_key = get_api_key()
|
||
prompt = f"""你是餐饮产业首席分析师。根据以下多篇文章的结构化分析，识别跨文章、可验证、有经营意义的行业信号。
|
||
只返回合法 JSON：{{"signals":[...]}}。每个 signal 必须包含：
|
||
- title（短标题）
|
||
- summary（100-180字）
|
||
- trend（emerging/accelerating/stable/declining）
|
||
- confidence（0到1）
|
||
- article_ids（至少2个证据文章ID）
|
||
- implications（2-4条经营启示）
|
||
- tags（3-6个标签）
|
||
不要把单一品牌新闻简单改写成行业信号；合并重复主题；最多输出8条。
|
||
|
||
输入：{json.dumps(items, ensure_ascii=False)}"""
|
||
body = json.dumps({
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": "你进行基于证据的餐饮行业趋势聚类，避免空泛结论。"},
|
||
{"role": "user", "content": prompt},
|
||
],
|
||
"temperature": 0.2,
|
||
"response_format": {"type": "json_object"},
|
||
}, ensure_ascii=False).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
DASHSCOPE_CHAT_URL,
|
||
data=body,
|
||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=180) as response:
|
||
result = json.loads(response.read().decode("utf-8"))
|
||
except urllib.error.HTTPError as exc:
|
||
raise RuntimeError(f"千问接口返回 {exc.code}：{exc.read().decode('utf-8', errors='replace')[:500]}") from exc
|
||
except urllib.error.URLError as exc:
|
||
raise RuntimeError(f"千问接口网络超时：{exc}") from exc
|
||
text = result["choices"][0]["message"]["content"].strip()
|
||
if text.startswith("```"):
|
||
text = text.strip("`").removeprefix("json").strip()
|
||
return json.loads(text).get("signals", [])
|
||
|
||
|
||
# ---- 文本向量嵌入 ----
|
||
|
||
def get_embeddings(texts: list[str]) -> list[list[float]]:
|
||
"""调用通义千问文本向量模型，批量获取向量。"""
|
||
api_key = get_api_key()
|
||
model = os.getenv("EMBEDDING_MODEL", "text-embedding-v3")
|
||
payload = json.dumps({
|
||
"model": model,
|
||
"input": texts,
|
||
}, ensure_ascii=False).encode("utf-8")
|
||
req = urllib.request.Request(
|
||
DASHSCOPE_EMBED_URL,
|
||
data=payload,
|
||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=60) as response:
|
||
body = json.loads(response.read().decode("utf-8"))
|
||
except urllib.error.HTTPError as exc:
|
||
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
||
raise RuntimeError(f"向量接口返回 {exc.code}：{detail}") from exc
|
||
if "output" in body:
|
||
return [item["embedding"] for item in body["output"]["embeddings"]]
|
||
return [item["embedding"] for item in body["data"]]
|
||
|
||
|
||
# ---- 品牌实体入库 ----
|
||
|
||
def upsert_brand(conn: sqlite3.Connection, brand_name: str, category: str | None = None,
|
||
stores: int = 0, avg_price: int = 0, model: str | None = None,
|
||
latest_news: str | None = None, news_date: str | None = None) -> None:
|
||
"""将识别到的品牌实体写入品牌表，有新数据时更新。"""
|
||
existing = conn.execute("SELECT id, stores, avg_price FROM brands WHERE name=?", (brand_name,)).fetchone()
|
||
timestamp = now_iso()
|
||
if existing:
|
||
# 只在新值更大或非空时更新，避免旧数据被覆盖
|
||
new_stores = max(existing["stores"], stores) if stores > 0 else existing["stores"]
|
||
new_avg_price = avg_price if avg_price > 0 else existing["avg_price"]
|
||
conn.execute(
|
||
"""
|
||
UPDATE brands SET category=COALESCE(?, category),
|
||
stores=?, avg_price=?,
|
||
model=COALESCE(NULLIF(?, ''), model),
|
||
latest_news=COALESCE(NULLIF(?, ''), latest_news),
|
||
news_date=COALESCE(NULLIF(?, ''), news_date),
|
||
updated_at=?
|
||
WHERE id=?
|
||
""",
|
||
(category, new_stores, new_avg_price, model or "", latest_news or "", news_date or "", timestamp, existing["id"]),
|
||
)
|
||
else:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO brands (name, category, stores, avg_price, model, city_tier_json, trend_json,
|
||
latest_news, news_date, growth, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, NULLIF(?, ''), '[]', '[]', NULLIF(?, ''), NULLIF(?, ''), 0, ?, ?)
|
||
""",
|
||
(brand_name, category, stores, avg_price, model or "", latest_news or "", news_date or "", timestamp, timestamp),
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
# ---- 批量分析主流程 ----
|
||
|
||
def run_batch_analyze(force: bool, pause: float, retries: int) -> int:
|
||
"""批量对所有成功采集的文章执行结构化分析。"""
|
||
db_path = Path(os.getenv("CIBANK_DB", str(BASE_DIR / "data" / "cibank.db")))
|
||
conn = sqlite3.connect(db_path)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.executescript((BASE_DIR / "schema.sql").read_text(encoding="utf-8"))
|
||
|
||
model = os.getenv("QWEN_MODEL", "qwen-plus")
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT a.id, a.title, a.content, a.content_hash, a.published_at, x.content_hash analyzed_hash
|
||
FROM articles a
|
||
LEFT JOIN ai_analyses x
|
||
ON x.article_id = a.id AND x.model = ? AND x.analysis_type = 'editorial'
|
||
WHERE a.crawl_status = 'success'
|
||
ORDER BY a.published_at DESC, a.id DESC
|
||
""",
|
||
(model,),
|
||
).fetchall()
|
||
|
||
pending = [r for r in rows if force or not r["analyzed_hash"] or r["analyzed_hash"] != r["content_hash"]]
|
||
print(json.dumps({"total": len(rows), "pending": len(pending), "model": model}, ensure_ascii=False), flush=True)
|
||
|
||
success = skipped = failed = 0
|
||
for index, row in enumerate(rows, start=1):
|
||
if not force and row["analyzed_hash"] == row["content_hash"]:
|
||
skipped += 1
|
||
continue
|
||
last_error = None
|
||
for attempt in range(1, retries + 2):
|
||
try:
|
||
result = analyze_article(row["title"], row["content"])
|
||
timestamp = now_iso()
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO ai_analyses(article_id, model, analysis_type, result_json, content_hash, created_at, updated_at)
|
||
VALUES (?, ?, 'editorial', ?, ?, ?, ?)
|
||
ON CONFLICT(article_id, model, analysis_type) DO UPDATE SET
|
||
result_json = excluded.result_json, content_hash = excluded.content_hash, updated_at = excluded.updated_at
|
||
""",
|
||
(row["id"], model, json.dumps(result, ensure_ascii=False), row["content_hash"], timestamp, timestamp),
|
||
)
|
||
# 更新文章的category字段
|
||
if result.get("category"):
|
||
conn.execute("UPDATE articles SET category=? WHERE id=?", (result["category"], row["id"]))
|
||
|
||
# 品牌实体入库
|
||
for brand in result.get("brands", []):
|
||
if isinstance(brand, str):
|
||
upsert_brand(conn, brand, result.get("category"))
|
||
elif isinstance(brand, dict):
|
||
upsert_brand(conn, brand.get("name", ""), result.get("category"),
|
||
int(brand.get("stores", 0) or 0), int(brand.get("avg_price", 0) or 0),
|
||
brand.get("model"), row["title"], (row["published_at"] or "")[:10])
|
||
|
||
conn.commit()
|
||
success += 1
|
||
print(f"[{index}/{len(rows)}] OK {row['title']}", flush=True)
|
||
last_error = None
|
||
break
|
||
except Exception as exc:
|
||
last_error = exc
|
||
if attempt <= retries:
|
||
wait = attempt * 2
|
||
print(f"[{index}/{len(rows)}] RETRY {attempt}/{retries} {exc}", flush=True)
|
||
time.sleep(wait)
|
||
if last_error is not None:
|
||
failed += 1
|
||
print(f"[{index}/{len(rows)}] FAIL {row['title']}: {last_error}", flush=True)
|
||
time.sleep(pause)
|
||
|
||
total_analyzed = conn.execute("SELECT COUNT(DISTINCT article_id) FROM ai_analyses").fetchone()[0]
|
||
conn.close()
|
||
print(json.dumps({
|
||
"success": success, "skipped": skipped, "failed": failed,
|
||
"total_analyzed": total_analyzed,
|
||
}, ensure_ascii=False), flush=True)
|
||
return 0 if failed == 0 else 1
|
||
|
||
|
||
def run_extract_signals(days: int) -> int:
|
||
"""提取跨文章行业信号。"""
|
||
db_path = Path(os.getenv("CIBANK_DB", str(BASE_DIR / "data" / "cibank.db")))
|
||
conn = sqlite3.connect(db_path, timeout=30)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.executescript((BASE_DIR / "schema.sql").read_text(encoding="utf-8"))
|
||
|
||
model = os.getenv("QWEN_MODEL", "qwen-plus")
|
||
from datetime import datetime, timedelta
|
||
cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%S")
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT a.id, a.title, a.published_at, x.result_json
|
||
FROM articles a JOIN ai_analyses x ON x.article_id = a.id
|
||
WHERE a.crawl_status = 'success' AND a.published_at >= ?
|
||
ORDER BY a.published_at DESC
|
||
""",
|
||
(cutoff,),
|
||
).fetchall()
|
||
|
||
items = []
|
||
for row in rows:
|
||
analysis = json.loads(row["result_json"])
|
||
items.append({
|
||
"article_id": row["id"], "title": row["title"], "published_at": row["published_at"],
|
||
"summary": analysis.get("summary"), "key_points": analysis.get("key_points", []),
|
||
"type": analysis.get("type"), "tags": analysis.get("tags", []),
|
||
"brands": analysis.get("brands", []), "score": analysis.get("score", 0),
|
||
})
|
||
|
||
if not items:
|
||
print(json.dumps({"status": "skipped", "reason": "没有已分析文章"}, ensure_ascii=False))
|
||
return 0
|
||
|
||
# 分批提取信号，避免一次性传入过多内容触发内容审查
|
||
signal_date = datetime.now().astimezone().date().isoformat()
|
||
timestamp = now_iso()
|
||
conn.execute("DELETE FROM industry_signals WHERE signal_date=?", (signal_date,))
|
||
conn.commit()
|
||
|
||
all_signals = []
|
||
batch_size = 20
|
||
total_batches = (len(items) + batch_size - 1) // batch_size
|
||
for i in range(0, len(items), batch_size):
|
||
batch_num = i // batch_size + 1
|
||
batch = items[i:i + batch_size]
|
||
print(f" 信号批次 {batch_num}/{total_batches}，{len(batch)} 篇", flush=True)
|
||
for attempt in range(2):
|
||
try:
|
||
batch_signals = extract_industry_signals(batch, model)
|
||
all_signals.extend(batch_signals)
|
||
# 每批成功后立即写入数据库
|
||
for signal in batch_signals:
|
||
conn.execute(
|
||
"""
|
||
INSERT OR IGNORE INTO industry_signals
|
||
(signal_date, title, summary, trend, confidence, implications_json, article_ids_json, tags_json, model, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
signal_date, signal["title"], signal["summary"], signal.get("trend", "emerging"),
|
||
float(signal.get("confidence", 0.5)), json.dumps(signal.get("implications", []), ensure_ascii=False),
|
||
json.dumps(signal.get("article_ids", [])), json.dumps(signal.get("tags", []), ensure_ascii=False),
|
||
model, timestamp, timestamp,
|
||
),
|
||
)
|
||
conn.commit()
|
||
break
|
||
except RuntimeError as exc:
|
||
print(f" 批次 {batch_num} 失败(尝试{attempt+1}): {exc}", flush=True)
|
||
if attempt == 0:
|
||
import time; time.sleep(2)
|
||
continue
|
||
except Exception as exc:
|
||
print(f" 批次 {batch_num} 异常(尝试{attempt+1}): {type(exc).__name__}: {exc}", flush=True)
|
||
if attempt == 0:
|
||
import time; time.sleep(2)
|
||
continue
|
||
|
||
# 合并去重：按标题去重，保留置信度更高的
|
||
seen = {}
|
||
for sig in all_signals:
|
||
title = sig.get("title", "")
|
||
conf = float(sig.get("confidence", 0))
|
||
if title not in seen or conf > float(seen[title].get("confidence", 0)):
|
||
seen[title] = sig
|
||
signals = list(seen.values())[:8]
|
||
|
||
# 用去重后的结果替换当天信号
|
||
conn.execute("DELETE FROM industry_signals WHERE signal_date=?", (signal_date,))
|
||
for signal in signals:
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO industry_signals
|
||
(signal_date, title, summary, trend, confidence, implications_json, article_ids_json, tags_json, model, created_at, updated_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
signal_date, signal["title"], signal["summary"], signal.get("trend", "emerging"),
|
||
float(signal.get("confidence", 0.5)), json.dumps(signal.get("implications", []), ensure_ascii=False),
|
||
json.dumps(signal.get("article_ids", [])), json.dumps(signal.get("tags", []), ensure_ascii=False),
|
||
model, timestamp, timestamp,
|
||
),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
print(json.dumps({"status": "success", "input_articles": len(items), "signals": len(signals), "date": signal_date}, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
# ---- 向量嵌入生成 ----
|
||
|
||
def ensure_embedding_schema(conn: sqlite3.Connection) -> None:
|
||
"""兼容已有数据库，为向量表补齐增量更新所需字段与唯一索引。"""
|
||
columns = {row[1] for row in conn.execute("PRAGMA table_info(embeddings)").fetchall()}
|
||
if "content_hash" not in columns:
|
||
conn.execute("ALTER TABLE embeddings ADD COLUMN content_hash TEXT")
|
||
conn.execute(
|
||
"""
|
||
DELETE FROM embeddings
|
||
WHERE id NOT IN (
|
||
SELECT MAX(id) FROM embeddings GROUP BY article_id, chunk_index, model
|
||
)
|
||
"""
|
||
)
|
||
conn.execute(
|
||
"CREATE UNIQUE INDEX IF NOT EXISTS idx_embeddings_unique_chunk "
|
||
"ON embeddings(article_id, chunk_index, model)"
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
def split_text(text: str, chunk_size: int = 800, overlap: int = 120) -> list[str]:
|
||
"""将长文本切分为有重叠的中文检索片段。"""
|
||
normalized = "\n".join(line.strip() for line in text.splitlines() if line.strip())
|
||
if not normalized:
|
||
return []
|
||
chunks = []
|
||
start = 0
|
||
while start < len(normalized):
|
||
end = min(len(normalized), start + chunk_size)
|
||
if end < len(normalized):
|
||
candidates = [normalized.rfind(mark, start + chunk_size // 2, end) for mark in "。！？；\n"]
|
||
boundary = max(candidates)
|
||
if boundary > start:
|
||
end = boundary + 1
|
||
chunks.append(normalized[start:end])
|
||
if end >= len(normalized):
|
||
break
|
||
start = max(start + 1, end - overlap)
|
||
return chunks
|
||
|
||
|
||
def build_embedding_chunks(row: sqlite3.Row) -> tuple[list[str], str]:
|
||
"""构建包含标题、AI摘要和完整原文的向量分块。"""
|
||
analysis = json.loads(row["result_json"]) if row["result_json"] else {}
|
||
header_parts = [f"标题：{row['title'] or ''}"]
|
||
summary = analysis.get("summary") or row["summary"] or ""
|
||
if summary:
|
||
header_parts.append(f"摘要：{summary}")
|
||
key_points = analysis.get("key_points") or []
|
||
if key_points:
|
||
header_parts.append("要点：" + "；".join(str(item) for item in key_points))
|
||
tags = analysis.get("tags") or []
|
||
if tags:
|
||
header_parts.append("标签：" + "、".join(str(item) for item in tags))
|
||
header = "\n".join(header_parts)
|
||
content_chunks = split_text(row["content"] or "") or [""]
|
||
chunks = [f"{header}\n正文片段：{content}".strip() for content in content_chunks]
|
||
source_hash = hashlib.sha256("\n\n".join(chunks).encode("utf-8")).hexdigest()
|
||
return chunks, source_hash
|
||
|
||
|
||
def run_generate_embeddings(force: bool) -> int:
|
||
"""为所有成功文章增量生成分块向量，用于RAG检索。"""
|
||
db_path = Path(os.getenv("CIBANK_DB", str(BASE_DIR / "data" / "cibank.db")))
|
||
conn = sqlite3.connect(db_path)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.executescript((BASE_DIR / "schema.sql").read_text(encoding="utf-8"))
|
||
ensure_embedding_schema(conn)
|
||
|
||
model = os.getenv("EMBEDDING_MODEL", "text-embedding-v3")
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT a.id, a.title, a.summary, a.content, x.result_json
|
||
FROM articles a
|
||
LEFT JOIN ai_analyses x ON x.id = (
|
||
SELECT x2.id FROM ai_analyses x2
|
||
WHERE x2.article_id = a.id AND x2.analysis_type = 'editorial'
|
||
ORDER BY x2.updated_at DESC, x2.id DESC LIMIT 1
|
||
)
|
||
WHERE a.crawl_status = 'success'
|
||
ORDER BY a.id
|
||
""",
|
||
).fetchall()
|
||
|
||
success = skipped = failed = chunks_written = 0
|
||
for row in rows:
|
||
chunks, source_hash = build_embedding_chunks(row)
|
||
existing = conn.execute(
|
||
"SELECT content_hash FROM embeddings WHERE article_id=? AND model=? LIMIT 1",
|
||
(row["id"], model),
|
||
).fetchone()
|
||
if not force and existing and existing["content_hash"] == source_hash:
|
||
skipped += 1
|
||
continue
|
||
|
||
try:
|
||
vectors = []
|
||
for start in range(0, len(chunks), 10):
|
||
vectors.extend(get_embeddings(chunks[start:start + 10]))
|
||
if len(vectors) != len(chunks):
|
||
raise RuntimeError(f"向量数量异常：期望 {len(chunks)}，实际 {len(vectors)}")
|
||
timestamp = now_iso()
|
||
conn.execute("BEGIN")
|
||
conn.execute("DELETE FROM embeddings WHERE article_id=? AND model=?", (row["id"], model))
|
||
conn.executemany(
|
||
"""
|
||
INSERT INTO embeddings
|
||
(article_id, chunk_index, chunk_text, embedding, model, content_hash, created_at)
|
||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||
""",
|
||
[
|
||
(row["id"], index, chunk, json.dumps(vector), model, source_hash, timestamp)
|
||
for index, (chunk, vector) in enumerate(zip(chunks, vectors))
|
||
],
|
||
)
|
||
conn.commit()
|
||
success += 1
|
||
chunks_written += len(chunks)
|
||
except Exception as exc:
|
||
conn.rollback()
|
||
failed += 1
|
||
print(f"FAIL article {row['id']}: {exc}", flush=True)
|
||
|
||
conn.close()
|
||
print(json.dumps({
|
||
"success": success, "skipped": skipped, "failed": failed,
|
||
"chunks_written": chunks_written, "model": model,
|
||
}, ensure_ascii=False))
|
||
return 0 if failed == 0 else 1
|
||
|
||
|
||
def run_generate_report(days: int = 7) -> int:
|
||
"""生成周期报告：聚合近期文章分析结果，调用LLM生成结构化报告。"""
|
||
db_path = Path(os.getenv("CIBANK_DB", str(BASE_DIR / "data" / "cibank.db")))
|
||
conn = sqlite3.connect(db_path, timeout=30)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.executescript((BASE_DIR / "schema.sql").read_text(encoding="utf-8"))
|
||
|
||
model = os.getenv("QWEN_MODEL", "qwen-plus")
|
||
from datetime import datetime, timedelta
|
||
cutoff = (datetime.now() - timedelta(days=days)).strftime("%Y-%m-%d")
|
||
|
||
rows = conn.execute(
|
||
"""
|
||
SELECT a.id, a.title, a.source_name, a.published_at, a.category,
|
||
x.result_json
|
||
FROM articles a
|
||
JOIN ai_analyses x ON x.article_id = a.id AND x.analysis_type = 'editorial'
|
||
WHERE a.crawl_status = 'success' AND a.published_at >= ?
|
||
ORDER BY a.published_at DESC
|
||
LIMIT 50
|
||
""",
|
||
(cutoff,),
|
||
).fetchall()
|
||
|
||
if not rows:
|
||
print(json.dumps({"status": "skip", "reason": "近期无分析数据"}, ensure_ascii=False))
|
||
return 0
|
||
|
||
items = []
|
||
for r in rows:
|
||
analysis = json.loads(r["result_json"]) if r["result_json"] else {}
|
||
items.append({
|
||
"title": r["title"],
|
||
"source": r["source_name"] or "",
|
||
"date": (r["published_at"] or "")[:10],
|
||
"category": r["category"] or analysis.get("category", ""),
|
||
"type": analysis.get("type", ""),
|
||
"summary": analysis.get("summary", ""),
|
||
"key_points": analysis.get("key_points", [])[:3],
|
||
"score": analysis.get("score", 0),
|
||
"brands": [b["name"] if isinstance(b, dict) else b for b in analysis.get("brands", [])][:3],
|
||
})
|
||
|
||
api_key = get_api_key()
|
||
period_label = f"近{days}天"
|
||
prompt = f"""你是餐饮行业首席分析师。根据以下{len(items)}篇情报的结构化分析，生成一份{period_label}餐饮行业经营决策简报。
|
||
只返回合法 JSON，不要 Markdown 代码块。格式：
|
||
{{
|
||
"title": "报告标题（含日期范围）",
|
||
"highlights": ["核心要点1", "核心要点2", "核心要点3", "核心要点4"],
|
||
"sections": [
|
||
{{"heading": "板块标题", "body": "详细分析正文（200-400字）"}},
|
||
{{"heading": "板块标题", "body": "详细分析正文（200-400字）"}},
|
||
{{"heading": "板块标题", "body": "详细分析正文（200-400字）"}},
|
||
{{"heading": "经营建议", "body": "基于以上分析的经营建议（200-300字）"}}
|
||
]
|
||
}}
|
||
|
||
要求：
|
||
1. highlights 为4-6条核心洞察，每条15-30字
|
||
2. sections 至少4个板块，覆盖赛道趋势、品牌动态、政策/供应链、经营建议
|
||
3. 内容必须基于提供的情报数据，不要编造
|
||
4. 语言专业、简洁、有决策参考价值
|
||
|
||
情报数据：
|
||
{json.dumps(items, ensure_ascii=False)}"""
|
||
|
||
payload = json.dumps({
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": "你是餐饮行业首席分析师，输出严谨的经营决策简报。"},
|
||
{"role": "user", "content": prompt},
|
||
],
|
||
"temperature": 0.3,
|
||
"response_format": {"type": "json_object"},
|
||
}, ensure_ascii=False).encode("utf-8")
|
||
|
||
req = urllib.request.Request(
|
||
DASHSCOPE_CHAT_URL,
|
||
data=payload,
|
||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=120) as response:
|
||
body = json.loads(response.read().decode("utf-8"))
|
||
except urllib.error.HTTPError as exc:
|
||
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
||
print(json.dumps({"status": "error", "error": f"千问接口返回 {exc.code}：{detail}"}, ensure_ascii=False))
|
||
return 1
|
||
|
||
text = body["choices"][0]["message"]["content"].strip()
|
||
if text.startswith("```"):
|
||
text = text.strip("`").removeprefix("json").strip()
|
||
result = json.loads(text)
|
||
|
||
report_date = datetime.now().strftime("%Y-%m-%d")
|
||
timestamp = now_iso()
|
||
conn.execute(
|
||
"""
|
||
INSERT INTO reports (title, report_date, period, highlights_json, sections_json, created_at)
|
||
VALUES (?, ?, ?, ?, ?, ?)
|
||
""",
|
||
(
|
||
result["title"],
|
||
report_date,
|
||
period_label,
|
||
json.dumps(result["highlights"], ensure_ascii=False),
|
||
json.dumps(result["sections"], ensure_ascii=False),
|
||
timestamp,
|
||
),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
print(json.dumps({
|
||
"status": "success",
|
||
"title": result["title"],
|
||
"date": report_date,
|
||
"highlights": len(result["highlights"]),
|
||
"sections": len(result["sections"]),
|
||
}, ensure_ascii=False))
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
parser = argparse.ArgumentParser(description="结构化引擎：LLM分析+品牌识别+信号提取+向量嵌入")
|
||
sub = parser.add_subparsers(dest="command")
|
||
|
||
p_analyze = sub.add_parser("analyze", help="批量文章结构化分析")
|
||
p_analyze.add_argument("--force", action="store_true")
|
||
p_analyze.add_argument("--pause", type=float, default=0.35)
|
||
p_analyze.add_argument("--retries", type=int, default=2)
|
||
|
||
p_signals = sub.add_parser("signals", help="提取跨文章行业信号")
|
||
p_signals.add_argument("--days", type=int, default=30)
|
||
|
||
p_embed = sub.add_parser("embed", help="生成向量嵌入")
|
||
p_embed.add_argument("--force", action="store_true")
|
||
|
||
p_report = sub.add_parser("report", help="生成周期报告")
|
||
p_report.add_argument("--days", type=int, default=7)
|
||
|
||
args = parser.parse_args()
|
||
if args.command == "analyze":
|
||
raise SystemExit(run_batch_analyze(args.force, args.pause, args.retries))
|
||
elif args.command == "signals":
|
||
raise SystemExit(run_extract_signals(args.days))
|
||
elif args.command == "embed":
|
||
raise SystemExit(run_generate_embeddings(args.force))
|
||
elif args.command == "report":
|
||
raise SystemExit(run_generate_report(args.days))
|
||
else:
|
||
parser.print_help()
|