"""Bzzoiro 数据源:抓取 + 入库(单一数据源)。 三条管线: 1. events — 比赛日程/比分(/events/),含 source_event_id 血缘 2. standings— 联赛积分榜快照(/leagues/{id}/standings/) 3. stats — 已完赛比赛详细统计回填(/events/{id}/stats/) 使用 Repository 模式进行数据访问,不直接控制事务(由调用方 UnitOfWork 控制)。 """ from __future__ import annotations import asyncio import logging import random from collections.abc import Iterable from datetime import datetime, timedelta, timezone from sqlalchemy import select from sqlalchemy.orm import selectinload import httpx from src.core.runtime_config import get_runtime_value from src.core.http_client import get_client from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL from src.data.key_ring import get_key_ring from src.data.normalize import normalize_bzzoiro from src.data.team_names_zh import zh_name from src.data.sources import register from src.db.models import League, Match, MatchStats, Standing, Team, RawEvent, IngestFailure, DataLineage logger = logging.getLogger(__name__) def _to_date(value): """把 datetime / date / str 统一成 `date`。""" if value is None: return None if hasattr(value, "date") and callable(value.date): return value.date() return value def _to_int_or_none(value) -> int | None: """宽松转 int(用于上游 ID 解析,失败返回 None 不抛错)。""" if value is None: return None try: return int(str(value).strip()) except (TypeError, ValueError): return None def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]: """比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。 统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类 隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配, 导致所有比赛被判为不存在而重复插入。 """ d = _to_date(match_date) return (home_team_id, away_team_id, d.isoformat() if d is not None else "") async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list: """异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。 多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。 """ base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/") raw_keys = await get_runtime_value("BZZOIRO_KEY") ring = get_key_ring(base, raw_keys) url = f"{base}/{path.lstrip('/')}" key = ring.get() if not key: raise RuntimeError("BZZOIRO_KEY 未设置") last_exc: Exception | None = None for attempt in range(max_retries): headers = { "Authorization": f"Token {key}", "Accept": "application/json", } try: client = get_client() # 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死 resp = await asyncio.wait_for( client.get( url, headers=headers, params=params, timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0), ), timeout=60.0, ) resp.raise_for_status() return resp.json() except Exception as e: last_exc = e status = getattr(getattr(e, "response", None), "status_code", None) if status == 429: # 限流:标记当前 key 冷却,切换到下一个 new_key = ring.report_rate_limited(key) if new_key and new_key != key: logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _km(key), _km(new_key)) key = new_key continue # 立即重试,不等待 # 单 key 或全部冷却:等待最早恢复的 key wait = ring.wait_if_all_blocked() if wait > 0: logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait) await asyncio.sleep(min(wait, 30.0)) else: delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay) await asyncio.sleep(delay) key = ring.get() or key continue if 500 <= (status or 0) < 600: delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay) await asyncio.sleep(delay) continue # 网络错误(连接失败/超时)也退避重试 if isinstance(e, (TimeoutError, ConnectionError, OSError)): delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e) await asyncio.sleep(delay) continue raise raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}") # ============================================================ # 管线基础设施:RawEvent / IngestFailure / DataLineage # ============================================================ async def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None: """写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。""" from sqlalchemy import select as _select stmt = _select(RawEvent).where( RawEvent.source_system == source_system, RawEvent.source_record_id == source_record_id, ) existing = (await db.execute(stmt)).scalar_one_or_none() if existing is None: db.add(RawEvent( source_system=source_system, source_record_id=source_record_id, raw_payload=raw_payload, ingest_batch_id=batch_id, )) async def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None: """写入采集失败死信。""" db.add(IngestFailure( source_system=source_system, entity_type=entity_type, source_record_id=source_record_id, error_type=error_type, error_detail=error_detail, raw_payload=raw_payload, )) async def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None: """写入 ETL 血缘追踪。""" db.add(DataLineage( source_system=source_system, source_record_id=source_record_id, target_table=target_table, target_id=target_id, transform_name=transform_name, transform_detail=transform_detail, batch_id=batch_id, )) # ============================================================ # 积分榜管线:/leagues/{id}/standings/ → standings 表 # ============================================================ async def fetch_bzzoiro_standings(league_code: str, season: str | None = None) -> dict: """抓取联赛积分榜(纯抓取,不入库)。season 为 None 时取当前赛季。""" league_id = BZZOIRO_LEAGUE_IDS.get(league_code) if league_id is None: raise ValueError(f"未知联赛代码: {league_code}") params: dict = {} if season: params["season"] = season return await _fetch_json_async(f"/leagues/{league_id}/standings/", params) def _season_label_from_dates(start_date, end_date) -> str: """从赛季起止日期推导赛季标签(与 derive_season_label 语义一致)。""" try: if isinstance(start_date, str): start = datetime.fromisoformat(start_date[:10]) else: start = start_date y = start.year return f"{y}-{y + 1}" if start.month >= 8 else f"{y - 1}-{y}" except (TypeError, ValueError): return "?" async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str | None = None) -> dict: """采集积分榜 → upsert standings 表。 season 为 None 时采集当前赛季(bzzoiro 默认返回 is_current 赛季)。 球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。 """ from src.data.team_names import normalize as normalize_name result: dict = {"leagues": {}, "total_upserted": 0, "errors": []} for code in leagues: league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0} try: payload = await fetch_bzzoiro_standings(code, season=season) except Exception as e: logger.exception("bzzoiro standings fetch failed for %s", code) league_r["errors"].append(str(e)) result["leagues"][code] = league_r result["errors"].append(f"{code}: {e}") continue rows = payload.get("standings") or [] return result # ============================================================ # 统计回填管线:/events/{id}/stats/ → match_stats 表 # ============================================================ # bzzoiro stats 字段 → MatchStats 字段映射(stats.home / stats.away 下) _STATS_FIELD_MAP = { "xg": ("home_xg", "away_xg"), # 回退 expected_goals "ball_possession": ("home_possession", None), # 只取主队值,客队=100-home "total_shots": ("home_shots", "away_shots"), "shots_on_target": ("home_shots_on_target", "away_shots_on_target"), "corner_kicks": ("home_corners", "away_corners"), "yellow_cards": ("home_yellow_cards", "away_yellow_cards"), "red_cards": ("home_red_cards", "away_red_cards"), "big_chances": ("home_big_chances", "away_big_chances"), "fouls": ("home_fouls", "away_fouls"), } def _pick(d: dict, *keys): """按优先级取第一个非空字段值。""" for k in keys: v = d.get(k) if v is not None: return v return None def _stats_from_payload(payload: dict) -> dict: """把 /events/{id}/stats/ 响应映射成 MatchStats 字段 dict。 响应结构: {"event_id": ..., "stats": {"home": {...}, "away": {...}}} """ stats = (payload or {}).get("stats") or {} home = stats.get("home") or {} away = stats.get("away") or {} out: dict = {} xg_h = _pick(home, "xg", "expected_goals") xg_a = _pick(away, "xg", "expected_goals") if xg_h is not None: out["home_xg"] = _to_float_or_none(xg_h) if xg_a is not None: out["away_xg"] = _to_float_or_none(xg_a) poss = home.get("ball_possession") if poss is not None: p = _to_float_or_none(poss) if p is not None: out["home_possession"] = p for src, (h_fld, a_fld) in _STATS_FIELD_MAP.items(): if src in ("xg", "ball_possession"): continue # 已处理 hv = home.get(src) av = away.get(src) if hv is not None and h_fld: out[h_fld] = _to_int_or_none(hv) if av is not None and a_fld: out[a_fld] = _to_int_or_none(av) return out def _to_float_or_none(value) -> float | None: if value is None: return None try: return float(str(value).strip()) except (TypeError, ValueError): return None async def ingest_bzzoiro_event_stats( db, *, leagues: Iterable[str], limit: int = 100, only_missing: bool = True, ) -> dict: """回填已完赛比赛的详细统计(逐场调 /events/{id}/stats/)。 筛选条件: match_status=finished 且 source_event_id 非空。 only_missing=True 时跳过已有统计的比赛(增量);False 则全量刷新。 limit 控制单次最多处理的比赛数(上游限速 1.2s/请求,大批量需分次触发)。 """ result: dict = {"fetched": 0, "created": 0, "updated": 0, "skipped": 0, "errors": []} league_ids = [BZZOIRO_LEAGUE_IDS[c] for c in leagues if c in BZZOIRO_LEAGUE_IDS] if not league_ids: result["errors"].append("无有效联赛代码") return result stmt = ( select(Match) .options(selectinload(Match.stats)) .where(Match.match_status == "finished") .where(Match.source_event_id.is_not(None)) .where(Match.league_id.in_(league_ids)) .order_by(Match.match_date.desc()) .limit(limit * 3 if only_missing else limit) ) matches = (await db.execute(stmt)).scalars().all() now = datetime.now(timezone.utc) processed = 0 for m in matches: if processed >= limit: break if only_missing and m.stats is not None and m.stats.home_shots is not None: result["skipped"] += 1 continue processed += 1 try: payload = await _fetch_json_async(f"/events/{m.source_event_id}/stats/") except Exception as e: logger.warning("stats fetch failed match=%s event=%s: %s", m.id, m.source_event_id, e) result["errors"].append(f"match {m.id}: {e}") await asyncio.sleep(REQUEST_INTERVAL) continue result["fetched"] += 1 fields = _stats_from_payload(payload) if not fields: result["skipped"] += 1 await asyncio.sleep(REQUEST_INTERVAL) continue if m.stats is None: available_at = m.match_date + timedelta(hours=2) if m.match_date else now m.stats = MatchStats( match_id=m.id, source="bzzoiro", source_record_id=str(m.source_event_id), retrieved_at=now, available_at=available_at, ) db.add(m.stats) result["created"] += 1 else: result["updated"] += 1 if m.stats.source is None: m.stats.source = "bzzoiro" m.stats.source_record_id = str(m.source_event_id) if m.stats.retrieved_at is None: m.stats.retrieved_at = now if m.stats.available_at is None and m.match_date: m.stats.available_at = m.match_date + timedelta(hours=2) for fld, v in fields.items(): if hasattr(m.stats, fld): setattr(m.stats, fld, v) # 管线基础设施:写入 RawEvent + DataLineage batch_id = f"bzzoiro-stats-{m.source_event_id}-{now.strftime('%Y%m%d%H%M%S')}" try: await _write_raw_event(db, "bzzoiro", str(m.source_event_id), payload, batch_id) await _write_lineage(db, "bzzoiro", str(m.source_event_id), "match_stats", m.stats.id if m.stats else None, "stats_backfill", {"match_id": m.id}, batch_id) except Exception: pass # 基础设施写入失败不影响主流程 await asyncio.sleep(REQUEST_INTERVAL) logger.info( "bzzoiro stats 回填完成: 抓取 %d, 新建 %d, 更新 %d, 跳过 %d, 错误 %d", result["fetched"], result["created"], result["updated"], result["skipped"], len(result["errors"]), ) return result