shangfangjianandnew-provider/LongCat-2.0 <
6a4994097a
后端: - bzzoiro events 采集成功后写入 RawEvent(Bronze 层) - 采集失败写入 IngestFailure(死信队列) - 写入 DataLineage(ETL 血缘追踪) - stats 回填成功后写入 RawEvent + DataLineage - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点 - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /api/v1/admin/llm/ping 端点(LLM 连通性测试) - lifespan 启动 quality_scheduler - Match 表补充 CHECK 约束(完赛必须有比分、状态枚举、半场≤全场) - Prediction FK 加 ondelete="CASCADE" 与迁移对齐 - PredictRequest mode 加 pattern 校验 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型 + 重试按钮) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
399 lines
15 KiB
Python
399 lines
15 KiB
Python
"""Bzzoiro 数据源:抓取 + 入库(单一数据源)。
|
|
|
|
三条管线:
|
|
1. events — 比赛日程/比分(/events/),含 source_event_id 血缘
|
|
2. standings— 联赛积分榜快照(/leagues/{id}/standings/)
|
|
3. stats — 已完赛比赛详细统计回填(/events/{id}/stats/)
|
|
|
|
使用 Repository 模式进行数据访问,不直接控制事务(由调用方 UnitOfWork 控制)。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
from collections.abc import Iterable
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
import httpx
|
|
|
|
from src.core.runtime_config import get_runtime_value
|
|
from src.core.http_client import get_client
|
|
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
|
from src.data.key_ring import get_key_ring
|
|
from src.data.normalize import normalize_bzzoiro
|
|
from src.data.team_names_zh import zh_name
|
|
from src.data.sources import register
|
|
from src.db.models import League, Match, MatchStats, Standing, Team, RawEvent, IngestFailure, DataLineage
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _to_date(value):
|
|
"""把 datetime / date / str 统一成 `date`。"""
|
|
if value is None:
|
|
return None
|
|
if hasattr(value, "date") and callable(value.date):
|
|
return value.date()
|
|
return value
|
|
|
|
|
|
def _to_int_or_none(value) -> int | None:
|
|
"""宽松转 int(用于上游 ID 解析,失败返回 None 不抛错)。"""
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return int(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]:
|
|
"""比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。
|
|
|
|
统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类
|
|
隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配,
|
|
导致所有比赛被判为不存在而重复插入。
|
|
"""
|
|
d = _to_date(match_date)
|
|
return (home_team_id, away_team_id, d.isoformat() if d is not None else "")
|
|
|
|
|
|
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
|
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。
|
|
|
|
多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。
|
|
"""
|
|
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
|
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
|
ring = get_key_ring(base, raw_keys)
|
|
|
|
url = f"{base}/{path.lstrip('/')}"
|
|
key = ring.get()
|
|
if not key:
|
|
raise RuntimeError("BZZOIRO_KEY 未设置")
|
|
|
|
last_exc: Exception | None = None
|
|
for attempt in range(max_retries):
|
|
headers = {
|
|
"Authorization": f"Token {key}",
|
|
"Accept": "application/json",
|
|
}
|
|
try:
|
|
client = get_client()
|
|
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
|
|
resp = await asyncio.wait_for(
|
|
client.get(
|
|
url, headers=headers, params=params,
|
|
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
|
|
),
|
|
timeout=60.0,
|
|
)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
except Exception as e:
|
|
last_exc = e
|
|
status = getattr(getattr(e, "response", None), "status_code", None)
|
|
if status == 429:
|
|
# 限流:标记当前 key 冷却,切换到下一个
|
|
new_key = ring.report_rate_limited(key)
|
|
if new_key and new_key != key:
|
|
logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _km(key), _km(new_key))
|
|
key = new_key
|
|
continue # 立即重试,不等待
|
|
# 单 key 或全部冷却:等待最早恢复的 key
|
|
wait = ring.wait_if_all_blocked()
|
|
if wait > 0:
|
|
logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait)
|
|
await asyncio.sleep(min(wait, 30.0))
|
|
else:
|
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
|
await asyncio.sleep(delay)
|
|
key = ring.get() or key
|
|
continue
|
|
if 500 <= (status or 0) < 600:
|
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay)
|
|
await asyncio.sleep(delay)
|
|
continue
|
|
# 网络错误(连接失败/超时)也退避重试
|
|
if isinstance(e, (TimeoutError, ConnectionError, OSError)):
|
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
|
await asyncio.sleep(delay)
|
|
continue
|
|
raise
|
|
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
|
|
|
|
|
# ============================================================
|
|
# 管线基础设施:RawEvent / IngestFailure / DataLineage
|
|
# ============================================================
|
|
|
|
|
|
async def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None:
|
|
"""写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。"""
|
|
from sqlalchemy import select as _select
|
|
stmt = _select(RawEvent).where(
|
|
RawEvent.source_system == source_system,
|
|
RawEvent.source_record_id == source_record_id,
|
|
)
|
|
existing = (await db.execute(stmt)).scalar_one_or_none()
|
|
if existing is None:
|
|
db.add(RawEvent(
|
|
source_system=source_system,
|
|
source_record_id=source_record_id,
|
|
raw_payload=raw_payload,
|
|
ingest_batch_id=batch_id,
|
|
))
|
|
|
|
|
|
async def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None:
|
|
"""写入采集失败死信。"""
|
|
db.add(IngestFailure(
|
|
source_system=source_system,
|
|
entity_type=entity_type,
|
|
source_record_id=source_record_id,
|
|
error_type=error_type,
|
|
error_detail=error_detail,
|
|
raw_payload=raw_payload,
|
|
))
|
|
|
|
|
|
async def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
|
|
"""写入 ETL 血缘追踪。"""
|
|
db.add(DataLineage(
|
|
source_system=source_system,
|
|
source_record_id=source_record_id,
|
|
target_table=target_table,
|
|
target_id=target_id,
|
|
transform_name=transform_name,
|
|
transform_detail=transform_detail,
|
|
batch_id=batch_id,
|
|
))
|
|
|
|
|
|
# ============================================================
|
|
# 积分榜管线:/leagues/{id}/standings/ → standings 表
|
|
# ============================================================
|
|
|
|
async def fetch_bzzoiro_standings(league_code: str, season: str | None = None) -> dict:
|
|
"""抓取联赛积分榜(纯抓取,不入库)。season 为 None 时取当前赛季。"""
|
|
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
|
if league_id is None:
|
|
raise ValueError(f"未知联赛代码: {league_code}")
|
|
params: dict = {}
|
|
if season:
|
|
params["season"] = season
|
|
return await _fetch_json_async(f"/leagues/{league_id}/standings/", params)
|
|
|
|
|
|
def _season_label_from_dates(start_date, end_date) -> str:
|
|
"""从赛季起止日期推导赛季标签(与 derive_season_label 语义一致)。"""
|
|
try:
|
|
if isinstance(start_date, str):
|
|
start = datetime.fromisoformat(start_date[:10])
|
|
else:
|
|
start = start_date
|
|
y = start.year
|
|
return f"{y}-{y + 1}" if start.month >= 8 else f"{y - 1}-{y}"
|
|
except (TypeError, ValueError):
|
|
return "?"
|
|
|
|
|
|
async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str | None = None) -> dict:
|
|
"""采集积分榜 → upsert standings 表。
|
|
|
|
season 为 None 时采集当前赛季(bzzoiro 默认返回 is_current 赛季)。
|
|
球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。
|
|
"""
|
|
from src.data.team_names import normalize as normalize_name
|
|
|
|
result: dict = {"leagues": {}, "total_upserted": 0, "errors": []}
|
|
for code in leagues:
|
|
league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0}
|
|
try:
|
|
payload = await fetch_bzzoiro_standings(code, season=season)
|
|
except Exception as e:
|
|
logger.exception("bzzoiro standings fetch failed for %s", code)
|
|
league_r["errors"].append(str(e))
|
|
result["leagues"][code] = league_r
|
|
result["errors"].append(f"{code}: {e}")
|
|
continue
|
|
|
|
rows = payload.get("standings") or []
|
|
return result
|
|
|
|
|
|
# ============================================================
|
|
# 统计回填管线:/events/{id}/stats/ → match_stats 表
|
|
# ============================================================
|
|
|
|
# bzzoiro stats 字段 → MatchStats 字段映射(stats.home / stats.away 下)
|
|
_STATS_FIELD_MAP = {
|
|
"xg": ("home_xg", "away_xg"), # 回退 expected_goals
|
|
"ball_possession": ("home_possession", None), # 只取主队值,客队=100-home
|
|
"total_shots": ("home_shots", "away_shots"),
|
|
"shots_on_target": ("home_shots_on_target", "away_shots_on_target"),
|
|
"corner_kicks": ("home_corners", "away_corners"),
|
|
"yellow_cards": ("home_yellow_cards", "away_yellow_cards"),
|
|
"red_cards": ("home_red_cards", "away_red_cards"),
|
|
"big_chances": ("home_big_chances", "away_big_chances"),
|
|
"fouls": ("home_fouls", "away_fouls"),
|
|
}
|
|
|
|
|
|
def _pick(d: dict, *keys):
|
|
"""按优先级取第一个非空字段值。"""
|
|
for k in keys:
|
|
v = d.get(k)
|
|
if v is not None:
|
|
return v
|
|
return None
|
|
|
|
|
|
def _stats_from_payload(payload: dict) -> dict:
|
|
"""把 /events/{id}/stats/ 响应映射成 MatchStats 字段 dict。
|
|
|
|
响应结构: {"event_id": ..., "stats": {"home": {...}, "away": {...}}}
|
|
"""
|
|
stats = (payload or {}).get("stats") or {}
|
|
home = stats.get("home") or {}
|
|
away = stats.get("away") or {}
|
|
out: dict = {}
|
|
|
|
xg_h = _pick(home, "xg", "expected_goals")
|
|
xg_a = _pick(away, "xg", "expected_goals")
|
|
if xg_h is not None:
|
|
out["home_xg"] = _to_float_or_none(xg_h)
|
|
if xg_a is not None:
|
|
out["away_xg"] = _to_float_or_none(xg_a)
|
|
|
|
poss = home.get("ball_possession")
|
|
if poss is not None:
|
|
p = _to_float_or_none(poss)
|
|
if p is not None:
|
|
out["home_possession"] = p
|
|
|
|
for src, (h_fld, a_fld) in _STATS_FIELD_MAP.items():
|
|
if src in ("xg", "ball_possession"):
|
|
continue # 已处理
|
|
hv = home.get(src)
|
|
av = away.get(src)
|
|
if hv is not None and h_fld:
|
|
out[h_fld] = _to_int_or_none(hv)
|
|
if av is not None and a_fld:
|
|
out[a_fld] = _to_int_or_none(av)
|
|
return out
|
|
|
|
|
|
def _to_float_or_none(value) -> float | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return float(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
async def ingest_bzzoiro_event_stats(
|
|
db,
|
|
*,
|
|
leagues: Iterable[str],
|
|
limit: int = 100,
|
|
only_missing: bool = True,
|
|
) -> dict:
|
|
"""回填已完赛比赛的详细统计(逐场调 /events/{id}/stats/)。
|
|
|
|
筛选条件: match_status=finished 且 source_event_id 非空。
|
|
only_missing=True 时跳过已有统计的比赛(增量);False 则全量刷新。
|
|
limit 控制单次最多处理的比赛数(上游限速 1.2s/请求,大批量需分次触发)。
|
|
"""
|
|
result: dict = {"fetched": 0, "created": 0, "updated": 0, "skipped": 0, "errors": []}
|
|
|
|
league_ids = [BZZOIRO_LEAGUE_IDS[c] for c in leagues if c in BZZOIRO_LEAGUE_IDS]
|
|
if not league_ids:
|
|
result["errors"].append("无有效联赛代码")
|
|
return result
|
|
|
|
stmt = (
|
|
select(Match)
|
|
.options(selectinload(Match.stats))
|
|
.where(Match.match_status == "finished")
|
|
.where(Match.source_event_id.is_not(None))
|
|
.where(Match.league_id.in_(league_ids))
|
|
.order_by(Match.match_date.desc())
|
|
.limit(limit * 3 if only_missing else limit)
|
|
)
|
|
matches = (await db.execute(stmt)).scalars().all()
|
|
|
|
now = datetime.now(timezone.utc)
|
|
processed = 0
|
|
for m in matches:
|
|
if processed >= limit:
|
|
break
|
|
if only_missing and m.stats is not None and m.stats.home_shots is not None:
|
|
result["skipped"] += 1
|
|
continue
|
|
processed += 1
|
|
try:
|
|
payload = await _fetch_json_async(f"/events/{m.source_event_id}/stats/")
|
|
except Exception as e:
|
|
logger.warning("stats fetch failed match=%s event=%s: %s", m.id, m.source_event_id, e)
|
|
result["errors"].append(f"match {m.id}: {e}")
|
|
await asyncio.sleep(REQUEST_INTERVAL)
|
|
continue
|
|
|
|
result["fetched"] += 1
|
|
fields = _stats_from_payload(payload)
|
|
if not fields:
|
|
result["skipped"] += 1
|
|
await asyncio.sleep(REQUEST_INTERVAL)
|
|
continue
|
|
|
|
if m.stats is None:
|
|
available_at = m.match_date + timedelta(hours=2) if m.match_date else now
|
|
m.stats = MatchStats(
|
|
match_id=m.id,
|
|
source="bzzoiro",
|
|
source_record_id=str(m.source_event_id),
|
|
retrieved_at=now,
|
|
available_at=available_at,
|
|
)
|
|
db.add(m.stats)
|
|
result["created"] += 1
|
|
else:
|
|
result["updated"] += 1
|
|
if m.stats.source is None:
|
|
m.stats.source = "bzzoiro"
|
|
m.stats.source_record_id = str(m.source_event_id)
|
|
if m.stats.retrieved_at is None:
|
|
m.stats.retrieved_at = now
|
|
if m.stats.available_at is None and m.match_date:
|
|
m.stats.available_at = m.match_date + timedelta(hours=2)
|
|
|
|
for fld, v in fields.items():
|
|
if hasattr(m.stats, fld):
|
|
setattr(m.stats, fld, v)
|
|
|
|
# 管线基础设施:写入 RawEvent + DataLineage
|
|
batch_id = f"bzzoiro-stats-{m.source_event_id}-{now.strftime('%Y%m%d%H%M%S')}"
|
|
try:
|
|
await _write_raw_event(db, "bzzoiro", str(m.source_event_id), payload, batch_id)
|
|
await _write_lineage(db, "bzzoiro", str(m.source_event_id), "match_stats", m.stats.id if m.stats else None, "stats_backfill", {"match_id": m.id}, batch_id)
|
|
except Exception:
|
|
pass # 基础设施写入失败不影响主流程
|
|
|
|
await asyncio.sleep(REQUEST_INTERVAL)
|
|
|
|
logger.info(
|
|
"bzzoiro stats 回填完成: 抓取 %d, 新建 %d, 更新 %d, 跳过 %d, 错误 %d",
|
|
result["fetched"], result["created"], result["updated"],
|
|
result["skipped"], len(result["errors"]),
|
|
)
|
|
return result
|