Profeto/src/data/normalize.py

"""数据规范化:任意数据源原始记录 → NormalizedMatch。

迁移自旧项目 app/data/normalize.py,简化:
    - 去掉 XGBackoff 双轨(不再需要独立回填)
    - 去掉 PIT 时间契约(无训练集要防泄漏)
    - 保留核心清洗契约(队名归一、日期解析、数值范围)
"""
from __future__ import annotations

import logging
import math
from dataclasses import dataclass
from datetime import datetime, timezone

logger = logging.getLogger(__name__)

VALID_STATUS = {"finished", "scheduled", "in_play", "paused", "postponed", "cancelled", "suspended"}

STATUS_MAP = {
    "finished": "finished", "completed": "finished", "done": "finished", "awarded": "finished",
    "scheduled": "scheduled", "upcoming": "scheduled", "notstarted": "scheduled", "not_started": "scheduled",
    "in_play": "in_play", "live": "in_play",
    "paused": "paused", "postponed": "postponed",
    "cancelled": "cancelled", "canceled": "cancelled", "abandoned": "cancelled",
    "suspended": "suspended",
}


@dataclass
class NormalizedMatch:
    """清洗后的统一比赛记录(入库中间格式)。"""

    league_type: str
    date: datetime
    home_team: str
    away_team: str
    match_status: str = "finished"
    home_goals: int | None = None
    away_goals: int | None = None
    season_label: str = ""
    home_xg: float | None = None
    away_xg: float | None = None
    home_shots: int | None = None
    away_shots: int | None = None
    home_shots_on_target: int | None = None
    away_shots_on_target: int | None = None
    home_corners: int | None = None
    away_corners: int | None = None
    home_possession: float | None = None
    home_yellow_cards: int | None = None
    away_yellow_cards: int | None = None
    home_red_cards: int | None = None
    away_red_cards: int | None = None
    home_ht_goals: int | None = None
    away_ht_goals: int | None = None
    match_stage: str | None = None

    def validate(self) -> None:
        """完整数据契约校验。"""
        if self.match_status == "finished" and (self.home_goals is None or self.away_goals is None):
            raise ValueError(f"Finished match must have score: {self.home_team} vs {self.away_team}")

        def _finite(n, v):
            if v is not None and isinstance(v, float) and not math.isfinite(v):
                raise ValueError(f"{n} must be finite, got {v}")

        def _range(n, v, lo, hi):
            if v is not None and not (lo <= v <= hi):
                raise ValueError(f"{n} out of range [{lo}, {hi}]: {v}")

        for side in ("home", "away"):
            _finite(f"{side}_goals", getattr(self, f"{side}_goals"))
            _range(f"{side}_goals", getattr(self, f"{side}_goals"), 0, 30)
            _finite(f"{side}_xg", getattr(self, f"{side}_xg"))
            _range(f"{side}_xg", getattr(self, f"{side}_xg"), 0, 20)
            for fld in ("shots", "shots_on_target", "corners"):
                _range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 100)
            for fld in ("yellow_cards", "red_cards"):
                _range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 20)
        _range("home_possession", self.home_possession, 0, 100)
        if self.home_ht_goals is not None and self.home_goals is not None and self.home_ht_goals > self.home_goals:
            raise ValueError(f"home_ht_goals({self.home_ht_goals}) > home_goals({self.home_goals})")
        if self.away_ht_goals is not None and self.away_goals is not None and self.away_ht_goals > self.away_goals:
            raise ValueError(f"away_ht_goals({self.away_ht_goals}) > away_goals({self.away_goals})")


def derive_season_label(date: datetime) -> str:
    y = date.year
    return f"{y}-{y + 1}" if date.month >= 8 else f"{y - 1}-{y}"


def _parse_date(value) -> datetime | None:
    """日期解析 → UTC datetime(带 tzinfo)。

    P2-2: 解析失败时记录 warning,避免静默丢数据而无感知。
    """
    if value in (None, ""):
        return None
    if isinstance(value, (int, float)):
        return datetime.fromtimestamp(value, tz=timezone.utc)
    s = str(value).strip()
    if not s:
        return None
    iso_s = s[:-1] + "+00:00" if s.endswith("Z") else s
    try:
        dt = datetime.fromisoformat(iso_s)
        if dt.tzinfo is not None:
            return dt.astimezone(timezone.utc)
        return dt.replace(tzinfo=timezone.utc)
    except ValueError:
        pass
    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", "%d/%m/%Y", "%d/%m/%y"):
        try:
            return datetime.strptime(s[:19], fmt).replace(tzinfo=timezone.utc)
        except ValueError:
            continue
    # P2-2 修复: 记录被丢弃的原始值,便于排查数据源格式变更
    logger.warning("_parse_date failed, dropping record: %r", value)
    return None


def _to_int(v) -> int | None:
    if v is None or (isinstance(v, str) and v.strip() in ("", "-")):
        return None
    if isinstance(v, bool):
        return None
    if isinstance(v, int):
        return v
    try:
        f = float(str(v).strip())
    except (TypeError, ValueError):
        return None
    if not f.is_integer():
        return None
    return int(f)


def _to_float(v) -> float | None:
    if v is None or (isinstance(v, str) and v.strip() in ("", "-")):
        return None
    try:
        return float(str(v).strip())
    except (TypeError, ValueError):
        return None


def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
    """bzzoiro event → NormalizedMatch。

    统计字段映射说明:
      当前字段名基于常见足球 API 模式推测(home_shots/away_shots 等),
      未经真实 bzzoiro 响应校验。若真实字段不同,映射结果将为 None。

    ⚠️ 待用真实响应核对的字段清单(请提供一份 event 样例验证):
      - 射门: home_shots / away_shots(或 shots_home / shots_away)
      - 射正: home_shots_on_target / away_shots_on_target(或 sot_home / sot_away)
      - 角球: home_corners / away_corners(或 corners_home / corners_away)
      - 控球: home_possession(或 possession,仅主队值)
      - xG: home_xg / away_xg(或 xg_home / xg_away / expected_goals_home / expected_goals_away)
      - 黄牌: home_yellow_cards / away_yellow_cards(或 yellow_cards_home / yellow_cards_away)
      - 红牌: home_red_cards / away_red_cards(或 red_cards_home / red_cards_away)

    映射策略:优先查主字段名,回退到别名。所有字段缺失时保持 None,不伪造。
    """
    from src.data.team_names import normalize as normalize_name

    date = _parse_date(raw.get("event_date"))
    if date is None:
        return None
    raw_status = str(raw.get("status", "")).lower()
    status = STATUS_MAP.get(raw_status)
    if status is None:
        return None
    home = normalize_name(raw.get("home_team", ""))
    away = normalize_name(raw.get("away_team", ""))
    if not home or not away or home == away:
        return None
    m = NormalizedMatch(
        league_type=league_type,
        date=date,
        home_team=home,
        away_team=away,
        match_status=status,
        season_label=derive_season_label(date),
    )
    m.home_goals = _to_int(raw.get("home_score", raw.get("home_goals")))
    m.away_goals = _to_int(raw.get("away_score", raw.get("away_goals")))
    m.home_ht_goals = _to_int(raw.get("home_score_ht", raw.get("home_ht_goals")))
    m.away_ht_goals = _to_int(raw.get("away_score_ht", raw.get("away_ht_goals")))
    _rn = _to_int(raw.get("round_number"))
    _rn_name = str(raw.get("round_name") or "").strip()
    if _rn_name:
        m.match_stage = _rn_name
    elif _rn:
        m.match_stage = f"第 {_rn} 轮"

    # 统计字段映射(API 字段名 → NormalizedMatch)
    # API 可能提供的字段:home_shots/away_shots, shots_on_target, corners, possession, xg, cards
    # API 没有的字段保持 None,不伪造
    m.home_shots = _to_int(raw.get("home_shots", raw.get("shots_home")))
    m.away_shots = _to_int(raw.get("away_shots", raw.get("shots_away")))
    m.home_shots_on_target = _to_int(raw.get("home_shots_on_target", raw.get("sot_home")))
    m.away_shots_on_target = _to_int(raw.get("away_shots_on_target", raw.get("sot_away")))
    m.home_corners = _to_int(raw.get("home_corners", raw.get("corners_home")))
    m.away_corners = _to_int(raw.get("away_corners", raw.get("corners_away")))
    # 控球率:API 通常只给 home 值,away = 100 - home
    possession_home = _to_float(raw.get("home_possession", raw.get("possession")))
    if possession_home is not None:
        m.home_possession = possession_home
    # xG
    m.home_xg = _to_float(raw.get("home_xg", raw.get("xg_home", raw.get("expected_goals_home"))))
    m.away_xg = _to_float(raw.get("away_xg", raw.get("xg_away", raw.get("expected_goals_away"))))
    # 牌
    m.home_yellow_cards = _to_int(raw.get("home_yellow_cards", raw.get("yellow_cards_home")))
    m.away_yellow_cards = _to_int(raw.get("away_yellow_cards", raw.get("yellow_cards_away")))
    m.home_red_cards = _to_int(raw.get("home_red_cards", raw.get("red_cards_home")))
    m.away_red_cards = _to_int(raw.get("away_red_cards", raw.get("red_cards_away")))

    if m.match_status == "finished" and m.home_goals is None:
        m.match_status = "scheduled"
    return m