"""数据规范化:任意数据源原始记录 → NormalizedMatch。 迁移自旧项目 app/data/normalize.py,简化: - 去掉 XGBackoff 双轨(不再需要独立回填) - 去掉 PIT 时间契约(无训练集要防泄漏) - 保留核心清洗契约(队名归一、日期解析、数值范围) """ from __future__ import annotations import logging import math from dataclasses import dataclass from datetime import datetime, timezone logger = logging.getLogger(__name__) VALID_STATUS = {"finished", "scheduled", "in_play", "paused", "postponed", "cancelled", "suspended"} STATUS_MAP = { "finished": "finished", "completed": "finished", "done": "finished", "awarded": "finished", "scheduled": "scheduled", "upcoming": "scheduled", "notstarted": "scheduled", "not_started": "scheduled", "in_play": "in_play", "live": "in_play", "paused": "paused", "postponed": "postponed", "cancelled": "cancelled", "canceled": "cancelled", "abandoned": "cancelled", "suspended": "suspended", } @dataclass class NormalizedMatch: """清洗后的统一比赛记录(入库中间格式)。""" league_type: str date: datetime home_team: str away_team: str match_status: str = "finished" home_goals: int | None = None away_goals: int | None = None season_label: str = "" home_xg: float | None = None away_xg: float | None = None home_shots: int | None = None away_shots: int | None = None home_shots_on_target: int | None = None away_shots_on_target: int | None = None home_corners: int | None = None away_corners: int | None = None home_possession: float | None = None home_yellow_cards: int | None = None away_yellow_cards: int | None = None home_red_cards: int | None = None away_red_cards: int | None = None home_ht_goals: int | None = None away_ht_goals: int | None = None match_stage: str | None = None def validate(self) -> None: """完整数据契约校验。""" if self.match_status == "finished" and (self.home_goals is None or self.away_goals is None): raise ValueError(f"Finished match must have score: {self.home_team} vs {self.away_team}") def _finite(n, v): if v is not None and isinstance(v, float) and not math.isfinite(v): raise ValueError(f"{n} must be finite, got {v}") def _range(n, v, lo, hi): if v is not None and not (lo <= v <= hi): raise ValueError(f"{n} out of range [{lo}, {hi}]: {v}") for side in ("home", "away"): _finite(f"{side}_goals", getattr(self, f"{side}_goals")) _range(f"{side}_goals", getattr(self, f"{side}_goals"), 0, 30) _finite(f"{side}_xg", getattr(self, f"{side}_xg")) _range(f"{side}_xg", getattr(self, f"{side}_xg"), 0, 20) for fld in ("shots", "shots_on_target", "corners"): _range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 100) for fld in ("yellow_cards", "red_cards"): _range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 20) _range("home_possession", self.home_possession, 0, 100) if self.home_ht_goals is not None and self.home_goals is not None and self.home_ht_goals > self.home_goals: raise ValueError(f"home_ht_goals({self.home_ht_goals}) > home_goals({self.home_goals})") if self.away_ht_goals is not None and self.away_goals is not None and self.away_ht_goals > self.away_goals: raise ValueError(f"away_ht_goals({self.away_ht_goals}) > away_goals({self.away_goals})") def derive_season_label(date: datetime) -> str: y = date.year return f"{y}-{y + 1}" if date.month >= 8 else f"{y - 1}-{y}" def _parse_date(value) -> datetime | None: """日期解析 → UTC datetime(带 tzinfo)。 P2-2: 解析失败时记录 warning,避免静默丢数据而无感知。 """ if value in (None, ""): return None if isinstance(value, (int, float)): return datetime.fromtimestamp(value, tz=timezone.utc) s = str(value).strip() if not s: return None iso_s = s[:-1] + "+00:00" if s.endswith("Z") else s try: dt = datetime.fromisoformat(iso_s) if dt.tzinfo is not None: return dt.astimezone(timezone.utc) return dt.replace(tzinfo=timezone.utc) except ValueError: pass for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", "%d/%m/%Y", "%d/%m/%y"): try: return datetime.strptime(s[:19], fmt).replace(tzinfo=timezone.utc) except ValueError: continue # P2-2 修复: 记录被丢弃的原始值,便于排查数据源格式变更 logger.warning("_parse_date failed, dropping record: %r", value) return None def _to_int(v) -> int | None: if v is None or (isinstance(v, str) and v.strip() in ("", "-")): return None if isinstance(v, bool): return None if isinstance(v, int): return v try: f = float(str(v).strip()) except (TypeError, ValueError): return None if not f.is_integer(): return None return int(f) def _to_float(v) -> float | None: if v is None or (isinstance(v, str) and v.strip() in ("", "-")): return None try: return float(str(v).strip()) except (TypeError, ValueError): return None def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None: """bzzoiro event → NormalizedMatch。 统计字段映射说明: 当前字段名基于常见足球 API 模式推测(home_shots/away_shots 等), 未经真实 bzzoiro 响应校验。若真实字段不同,映射结果将为 None。 ⚠️ 待用真实响应核对的字段清单(请提供一份 event 样例验证): - 射门: home_shots / away_shots(或 shots_home / shots_away) - 射正: home_shots_on_target / away_shots_on_target(或 sot_home / sot_away) - 角球: home_corners / away_corners(或 corners_home / corners_away) - 控球: home_possession(或 possession,仅主队值) - xG: home_xg / away_xg(或 xg_home / xg_away / expected_goals_home / expected_goals_away) - 黄牌: home_yellow_cards / away_yellow_cards(或 yellow_cards_home / yellow_cards_away) - 红牌: home_red_cards / away_red_cards(或 red_cards_home / red_cards_away) 映射策略:优先查主字段名,回退到别名。所有字段缺失时保持 None,不伪造。 """ from src.data.team_names import normalize as normalize_name date = _parse_date(raw.get("event_date")) if date is None: return None raw_status = str(raw.get("status", "")).lower() status = STATUS_MAP.get(raw_status) if status is None: return None home = normalize_name(raw.get("home_team", "")) away = normalize_name(raw.get("away_team", "")) if not home or not away or home == away: return None m = NormalizedMatch( league_type=league_type, date=date, home_team=home, away_team=away, match_status=status, season_label=derive_season_label(date), ) m.home_goals = _to_int(raw.get("home_score", raw.get("home_goals"))) m.away_goals = _to_int(raw.get("away_score", raw.get("away_goals"))) m.home_ht_goals = _to_int(raw.get("home_score_ht", raw.get("home_ht_goals"))) m.away_ht_goals = _to_int(raw.get("away_score_ht", raw.get("away_ht_goals"))) _rn = _to_int(raw.get("round_number")) _rn_name = str(raw.get("round_name") or "").strip() if _rn_name: m.match_stage = _rn_name elif _rn: m.match_stage = f"第 {_rn} 轮" # 统计字段映射(API 字段名 → NormalizedMatch) # API 可能提供的字段:home_shots/away_shots, shots_on_target, corners, possession, xg, cards # API 没有的字段保持 None,不伪造 m.home_shots = _to_int(raw.get("home_shots", raw.get("shots_home"))) m.away_shots = _to_int(raw.get("away_shots", raw.get("shots_away"))) m.home_shots_on_target = _to_int(raw.get("home_shots_on_target", raw.get("sot_home"))) m.away_shots_on_target = _to_int(raw.get("away_shots_on_target", raw.get("sot_away"))) m.home_corners = _to_int(raw.get("home_corners", raw.get("corners_home"))) m.away_corners = _to_int(raw.get("away_corners", raw.get("corners_away"))) # 控球率:API 通常只给 home 值,away = 100 - home possession_home = _to_float(raw.get("home_possession", raw.get("possession"))) if possession_home is not None: m.home_possession = possession_home # xG m.home_xg = _to_float(raw.get("home_xg", raw.get("xg_home", raw.get("expected_goals_home")))) m.away_xg = _to_float(raw.get("away_xg", raw.get("xg_away", raw.get("expected_goals_away")))) # 牌 m.home_yellow_cards = _to_int(raw.get("home_yellow_cards", raw.get("yellow_cards_home"))) m.away_yellow_cards = _to_int(raw.get("away_yellow_cards", raw.get("yellow_cards_away"))) m.home_red_cards = _to_int(raw.get("home_red_cards", raw.get("red_cards_home"))) m.away_red_cards = _to_int(raw.get("away_red_cards", raw.get("red_cards_away"))) if m.match_status == "finished" and m.home_goals is None: m.match_status = "scheduled" return m