Profeto/src/data/bzzoiro_stats.py

"""bzzoiro stats 回填管线:已完赛比赛详细统计(/events/{id}/stats/)→ match_stats 表。

从 bzzoiro.py 拆出。上游限速(REQUEST_INTERVAL 秒/请求),大批量回填需分次触发;
只 add/flush 不 commit,事务由调用方 UnitOfWork 控制。

可替换协作者(_fetch_json_async / Bronze 写入助手 / REQUEST_INTERVAL)在运行期
经聚合门面 src.data.bzzoiro 解析 —— 与拆分前的单文件 monkeypatch 语义一致。
"""
from __future__ import annotations

import asyncio
import logging
from collections.abc import Iterable
from datetime import datetime, timedelta, timezone

from src.data.bzzoiro_common import _to_float_or_none, _to_int_or_none
from src.data.config import BZZOIRO_LEAGUE_IDS
from src.data.pipeline_write import _safe_write_ingest_failure, _write_lineage, _write_raw_event
from src.db.models import MatchStats
from src.db.repositories import MatchRepository

logger = logging.getLogger(__name__)


# bzzoiro stats 字段 → MatchStats 字段映射(stats.home / stats.away 下)
_STATS_FIELD_MAP = {
    "xg": ("home_xg", "away_xg"),                    # 回退 expected_goals
    "ball_possession": ("home_possession", None),     # 只取主队值,客队=100-home
    "total_shots": ("home_shots", "away_shots"),
    "shots_on_target": ("home_shots_on_target", "away_shots_on_target"),
    "corner_kicks": ("home_corners", "away_corners"),
    "yellow_cards": ("home_yellow_cards", "away_yellow_cards"),
    "red_cards": ("home_red_cards", "away_red_cards"),
    "big_chances": ("home_big_chances", "away_big_chances"),
    "fouls": ("home_fouls", "away_fouls"),
}


def _pick(d: dict, *keys):
    """按优先级取第一个非空字段值。"""
    for k in keys:
        v = d.get(k)
        if v is not None:
            return v
    return None


def _stats_from_payload(payload: dict) -> dict:
    """把 /events/{id}/stats/ 响应映射成 MatchStats 字段 dict。

    响应结构: {"event_id": ..., "stats": {"home": {...}, "away": {...}}}
    """
    stats = (payload or {}).get("stats") or {}
    home = stats.get("home") or {}
    away = stats.get("away") or {}
    out: dict = {}

    xg_h = _pick(home, "xg", "expected_goals")
    xg_a = _pick(away, "xg", "expected_goals")
    if xg_h is not None:
        out["home_xg"] = _to_float_or_none(xg_h)
    if xg_a is not None:
        out["away_xg"] = _to_float_or_none(xg_a)

    poss = home.get("ball_possession")
    if poss is not None:
        p = _to_float_or_none(poss)
        if p is not None:
            out["home_possession"] = p

    for src, (h_fld, a_fld) in _STATS_FIELD_MAP.items():
        if src in ("xg", "ball_possession"):
            continue  # 已处理
        hv = home.get(src)
        av = away.get(src)
        if hv is not None and h_fld:
            out[h_fld] = _to_int_or_none(hv)
        if av is not None and a_fld:
            out[a_fld] = _to_int_or_none(av)
    return out


async def ingest_bzzoiro_event_stats(
    db,
    *,
    leagues: Iterable[str],
    limit: int = 100,
    only_missing: bool = True,
) -> dict:
    """回填已完赛比赛的详细统计(逐场调 /events/{id}/stats/)。

    筛选条件: match_status=finished 且 source_event_id 非空。
    only_missing=True 时跳过已有统计的比赛(增量);False 则全量刷新。
    limit 控制单次最多处理的比赛数(上游限速约 1.2s/请求,大批量需分次触发)。
    """
    from src.data import bzzoiro as bz

    result: dict = {"fetched": 0, "created": 0, "updated": 0, "skipped": 0, "errors": []}

    league_ids = [BZZOIRO_LEAGUE_IDS[c] for c in leagues if c in BZZOIRO_LEAGUE_IDS]
    if not league_ids:
        result["errors"].append("无有效联赛代码")
        return result

    # D4: 候选比赛查询经 MatchRepository(含 stats 预加载,筛选/排序/limit 语义不变)
    matches = await MatchRepository(db).find_finished_with_stats(
        league_ids, limit=limit * 3 if only_missing else limit
    )

    now = datetime.now(timezone.utc)
    processed = 0
    for m in matches:
        if processed >= limit:
            break
        if only_missing and m.stats is not None and m.stats.home_shots is not None:
            result["skipped"] += 1
            continue
        processed += 1
        try:
            payload = await bz._fetch_json_async(f"/events/{m.source_event_id}/stats/")
        except Exception as e:
            logger.warning("stats fetch failed match=%s event=%s: %s", m.id, m.source_event_id, e)
            result["errors"].append(f"match {m.id}: {e}")
            await _safe_write_ingest_failure(
                db,
                entity_type="match_stats",
                source_record_id=str(m.source_event_id),
                error=e,
                raw_payload={"match_id": m.id},
            )
            await asyncio.sleep(bz.REQUEST_INTERVAL)
            continue

        result["fetched"] += 1
        fields = _stats_from_payload(payload)
        if not fields:
            result["skipped"] += 1
            await asyncio.sleep(bz.REQUEST_INTERVAL)
            continue

        if m.stats is None:
            available_at = m.match_date + timedelta(hours=2) if m.match_date else now
            m.stats = MatchStats(
                match_id=m.id,
                source="bzzoiro",
                source_record_id=str(m.source_event_id),
                retrieved_at=now,
                available_at=available_at,
            )
            db.add(m.stats)
            result["created"] += 1
        else:
            result["updated"] += 1
            if m.stats.source is None:
                m.stats.source = "bzzoiro"
                m.stats.source_record_id = str(m.source_event_id)
            if m.stats.retrieved_at is None:
                m.stats.retrieved_at = now
            if m.stats.available_at is None and m.match_date:
                m.stats.available_at = m.match_date + timedelta(hours=2)

        for fld, v in fields.items():
            if hasattr(m.stats, fld):
                setattr(m.stats, fld, v)

        # 管线基础设施:写入 RawEvent + DataLineage
        batch_id = f"bzzoiro-stats-{m.source_event_id}-{now.strftime('%Y%m%d%H%M%S')}"
        try:
            await _write_raw_event(db, "bzzoiro", str(m.source_event_id), payload, batch_id)
            await _write_lineage(db, "bzzoiro", str(m.source_event_id), "match_stats", m.stats.id if m.stats else None, "stats_backfill", {"match_id": m.id}, batch_id)
        except Exception:
            pass  # 基础设施写入失败不影响主流程

        await asyncio.sleep(bz.REQUEST_INTERVAL)

    logger.info(
        "bzzoiro stats 回填完成: 抓取 %d, 新建 %d, 更新 %d, 跳过 %d, 错误 %d",
        result["fetched"], result["created"], result["updated"],
        result["skipped"], len(result["errors"]),
    )
    return result