Profeto/src/data/normalize.py

"""数据规范化:任意数据源原始记录 → NormalizedMatch。

迁移自旧项目 app/data/normalize.py,简化:
    - 去掉 XGBackoff 双轨(不再需要独立回填)
    - 去掉 PIT 时间契约(无训练集要防泄漏)
    - 保留核心清洗契约(队名归一、日期解析、数值范围)
"""
from __future__ import annotations

import logging
import math
from dataclasses import dataclass
from datetime import datetime, timezone

logger = logging.getLogger(__name__)

VALID_STATUS = {"finished", "scheduled", "in_play", "paused", "postponed", "cancelled", "suspended"}

STATUS_MAP = {
    "finished": "finished", "completed": "finished", "done": "finished", "awarded": "finished",
    "scheduled": "scheduled", "upcoming": "scheduled",
    "in_play": "in_play", "live": "in_play",
    "paused": "paused", "postponed": "postponed",
    "cancelled": "cancelled", "canceled": "cancelled", "abandoned": "cancelled",
    "suspended": "suspended",
}


@dataclass
class NormalizedMatch:
    """清洗后的统一比赛记录(入库中间格式)。"""

    league_type: str
    date: datetime
    home_team: str
    away_team: str
    match_status: str = "finished"
    home_goals: int | None = None
    away_goals: int | None = None
    season_label: str = ""
    home_xg: float | None = None
    away_xg: float | None = None
    home_shots: int | None = None
    away_shots: int | None = None
    home_shots_on_target: int | None = None
    away_shots_on_target: int | None = None
    home_corners: int | None = None
    away_corners: int | None = None
    home_possession: float | None = None
    home_yellow_cards: int | None = None
    away_yellow_cards: int | None = None
    home_red_cards: int | None = None
    away_red_cards: int | None = None
    home_ht_goals: int | None = None
    away_ht_goals: int | None = None
    match_stage: str | None = None

    def validate(self) -> None:
        """完整数据契约校验。"""
        if self.match_status == "finished" and (self.home_goals is None or self.away_goals is None):
            raise ValueError(f"Finished match must have score: {self.home_team} vs {self.away_team}")

        def _finite(n, v):
            if v is not None and isinstance(v, float) and not math.isfinite(v):
                raise ValueError(f"{n} must be finite, got {v}")

        def _range(n, v, lo, hi):
            if v is not None and not (lo <= v <= hi):
                raise ValueError(f"{n} out of range [{lo}, {hi}]: {v}")

        for side in ("home", "away"):
            _finite(f"{side}_goals", getattr(self, f"{side}_goals"))
            _range(f"{side}_goals", getattr(self, f"{side}_goals"), 0, 30)
            _finite(f"{side}_xg", getattr(self, f"{side}_xg"))
            _range(f"{side}_xg", getattr(self, f"{side}_xg"), 0, 20)
            for fld in ("shots", "shots_on_target", "corners"):
                _range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 100)
            for fld in ("yellow_cards", "red_cards"):
                _range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 20)
        _range("home_possession", self.home_possession, 0, 100)
        if self.home_ht_goals is not None and self.home_goals is not None and self.home_ht_goals > self.home_goals:
            raise ValueError(f"home_ht_goals({self.home_ht_goals}) > home_goals({self.home_goals})")
        if self.away_ht_goals is not None and self.away_goals is not None and self.away_ht_goals > self.away_goals:
            raise ValueError(f"away_ht_goals({self.away_ht_goals}) > away_goals({self.away_goals})")


def derive_season_label(date: datetime) -> str:
    y = date.year
    return f"{y}-{y + 1}" if date.month >= 8 else f"{y - 1}-{y}"


def _parse_date(value) -> datetime | None:
    """日期解析 → UTC datetime(带 tzinfo)。

    P2-2: 解析失败时记录 warning,避免静默丢数据而无感知。
    """
    if value in (None, ""):
        return None
    if isinstance(value, (int, float)):
        return datetime.fromtimestamp(value, tz=timezone.utc)
    s = str(value).strip()
    if not s:
        return None
    iso_s = s[:-1] + "+00:00" if s.endswith("Z") else s
    try:
        dt = datetime.fromisoformat(iso_s)
        if dt.tzinfo is not None:
            return dt.astimezone(timezone.utc)
        return dt.replace(tzinfo=timezone.utc)
    except ValueError:
        pass
    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", "%d/%m/%Y", "%d/%m/%y"):
        try:
            return datetime.strptime(s[:19], fmt).replace(tzinfo=timezone.utc)
        except ValueError:
            continue
    # P2-2 修复: 记录被丢弃的原始值,便于排查数据源格式变更
    logger.warning("_parse_date failed, dropping record: %r", value)
    return None


def _to_int(v) -> int | None:
    if v is None or (isinstance(v, str) and v.strip() in ("", "-")):
        return None
    if isinstance(v, bool):
        return None
    if isinstance(v, int):
        return v
    try:
        f = float(str(v).strip())
    except (TypeError, ValueError):
        return None
    if not f.is_integer():
        return None
    return int(f)


def _to_float(v) -> float | None:
    if v is None or (isinstance(v, str) and v.strip() in ("", "-")):
        return None
    try:
        return float(str(v).strip())
    except (TypeError, ValueError):
        return None


def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
    """bzzoiro event → NormalizedMatch。"""
    from src.data.team_names import normalize as normalize_name

    date = _parse_date(raw.get("event_date"))
    if date is None:
        return None
    raw_status = str(raw.get("status", "")).lower()
    status = STATUS_MAP.get(raw_status)
    if status is None:
        return None
    home = normalize_name(raw.get("home_team", ""))
    away = normalize_name(raw.get("away_team", ""))
    if not home or not away or home == away:
        return None
    m = NormalizedMatch(
        league_type=league_type,
        date=date,
        home_team=home,
        away_team=away,
        match_status=status,
        season_label=derive_season_label(date),
    )
    m.home_goals = _to_int(raw.get("home_score", raw.get("home_goals")))
    m.away_goals = _to_int(raw.get("away_score", raw.get("away_goals")))
    m.home_ht_goals = _to_int(raw.get("home_score_ht", raw.get("home_ht_goals")))
    m.away_ht_goals = _to_int(raw.get("away_score_ht", raw.get("away_ht_goals")))
    _rn = _to_int(raw.get("round_number"))
    _rn_name = str(raw.get("round_name") or "").strip()
    if _rn_name:
        m.match_stage = _rn_name
    elif _rn:
        m.match_stage = f"第 {_rn} 轮"
    if m.match_status == "finished" and m.home_goals is None:
        m.match_status = "scheduled"
    return m


def normalize_understat(raw: dict, league_type: str) -> NormalizedMatch | None:
    """understat 单场 → NormalizedMatch(仅 xG)。"""
    from src.data.team_names import normalize as normalize_name

    dt_str = raw.get("datetime") or raw.get("date")
    if not dt_str:
        return None
    dt = _parse_date(dt_str)
    if dt is None:
        return None
    home_info = raw.get("h", {})
    away_info = raw.get("a", {})
    home_name = home_info.get("title", "") if isinstance(home_info, dict) else ""
    away_name = away_info.get("title", "") if isinstance(away_info, dict) else ""
    home = normalize_name(home_name)
    away = normalize_name(away_name)
    if not home or not away or home == away:
        return None
    home_xg = raw.get("xG", {}).get("h") if isinstance(raw.get("xG"), dict) else None
    away_xg = raw.get("xG", {}).get("a") if isinstance(raw.get("xG"), dict) else None
    return NormalizedMatch(
        league_type=league_type,
        date=dt,
        home_team=home,
        away_team=away,
        match_status="finished",
        season_label=derive_season_label(dt),
        home_xg=home_xg,
        away_xg=away_xg,
    )