"""bzzoiro standings 管线:联赛积分榜快照(/leagues/{id}/standings/)→ standings 表。 从 bzzoiro.py 拆出。同一联赛同一赛季只保留最新快照(按 (league, season, team) upsert);球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。 可替换协作者(抓取函数 / Bronze 写入助手)在运行期经聚合门面 src.data.bzzoiro 解析 —— 与拆分前的单文件 monkeypatch 语义一致。 """ from __future__ import annotations import logging from collections.abc import Iterable from datetime import datetime, timezone from sqlalchemy import select from src.data.bzzoiro_common import _to_float_or_none, _to_int_or_none from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES from src.data.pipeline_write import _safe_write_ingest_failure, _write_lineage, _write_raw_event from src.data.team_names_zh import zh_name from src.db.models import Standing, Team from src.db.repositories import LeagueRepository, TeamRepository logger = logging.getLogger(__name__) async def fetch_bzzoiro_standings(league_code: str, season: str | None = None) -> dict: """抓取联赛积分榜(纯抓取,不入库)。season 为 None 时取当前赛季。""" from src.data import bzzoiro as bz league_id = BZZOIRO_LEAGUE_IDS.get(league_code) if league_id is None: raise ValueError(f"未知联赛代码: {league_code}") params: dict = {} if season: params["season"] = season return await bz._fetch_json_async(f"/leagues/{league_id}/standings/", params) def _season_label_from_dates(start_date, end_date) -> str: """从赛季起止日期推导赛季标签(与 derive_season_label 语义一致)。""" try: if isinstance(start_date, str): start = datetime.fromisoformat(start_date[:10]) else: start = start_date if start is None: return "?" y = start.year return f"{y}-{y + 1}" if start.month >= 8 else f"{y - 1}-{y}" except (TypeError, ValueError): return "?" async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str | None = None) -> dict: """采集积分榜 → upsert standings 表。 season 为 None 时采集当前赛季(bzzoiro 默认返回 is_current 赛季)。 球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。 """ from src.data.team_names import normalize as normalize_name from src.data import bzzoiro as bz result: dict = {"leagues": {}, "total_upserted": 0, "errors": []} for code in leagues: league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0, "errors": []} try: payload = await bz.fetch_bzzoiro_standings(code, season=season) except Exception as e: logger.exception("bzzoiro standings fetch failed for %s", code) league_r["errors"].append(str(e)) await _safe_write_ingest_failure( db, entity_type="standings", source_record_id=None, error=e, raw_payload={"league": code, "season": season}, ) result["leagues"][code] = league_r result["errors"].append(f"{code}: {e}") continue rows = payload.get("standings") or [] if not rows: result["leagues"][code] = {"error": "无积分榜数据(赛季未开始或未提供)"} result["errors"].append(f"{code}: 无积分榜数据") continue # 联赛(get-or-create,D4: 经 LeagueRepository) league = await LeagueRepository(db).get_or_create( code, LEAGUE_NAMES.get(code, code), LEAGUE_COUNTRIES.get(code) ) team_r = TeamRepository(db) # 赛季标签:优先用返回的 season 对象推导 season_obj = payload.get("season") or {} season_label = _season_label_from_dates( season_obj.get("start_date"), season_obj.get("end_date") ) if season_label == "?": season_label = season or "" # 批量预载球队(与 events 管线使用同一 normalize 规则,保证 Team 匹配) names = {normalize_name(str(r.get("team_name", ""))) for r in rows} names.discard("") team_map: dict[str, Team] = await team_r.get_all_by_names(list(names)) now = datetime.now(timezone.utc) for r in rows: team_name = normalize_name(str(r.get("team_name", ""))) if not team_name: continue team = team_map.get(team_name) if team is None: team = await team_r.get_or_create(team_name, name_zh=zh_name(team_name)) team_map[team_name] = team league_r["teams_created"] += 1 zone = r.get("zone") or {} values = dict( position=_to_int_or_none(r.get("position")) or 0, played=_to_int_or_none(r.get("played")) or 0, won=_to_int_or_none(r.get("won")) or 0, drawn=_to_int_or_none(r.get("drawn")) or 0, lost=_to_int_or_none(r.get("lost")) or 0, goals_for=_to_int_or_none(r.get("gf")) or 0, goals_against=_to_int_or_none(r.get("ga")) or 0, goal_diff=_to_int_or_none(r.get("gd")) or 0, points=_to_int_or_none(r.get("pts")) or 0, xg_for=_to_float_or_none(r.get("xgf")), xg_against=_to_float_or_none(r.get("xga")), form=r.get("form") or None, zone=zone.get("label") or zone.get("key") or None, updated_at=now, retrieved_at=now, ) # P0-02: 追加快照——每次采集 INSERT 新行(available_at=now), # ON CONFLICT (league, season, team, available_at) DO NOTHING。 standing = Standing( league_id=league.id, season=season_label, team_id=team.id, available_at=now, **values ) db.add(standing) league_r["upserted"] += 1 league_r["rows"] = len(rows) # D1(对称 events/stats 管线): 联赛成功 upsert → 补写 Bronze 层。 # 幂等键 standings:{league}:{season}:积分榜是联赛级快照,一次成功 # 采集写一条 RawEvent(整份原始载荷)+ 一条血缘。season 用实际入库的 # 标签(由载荷推导,与 Standing.season 同口径),不依赖调用方传参, # 保证不同调用方(season=None 或显式传参)对同一赛季命中同一条 RawEvent。 if league_r["upserted"] > 0: bronze_batch_id = f"bzzoiro-standings-{code}-{now:%Y%m%d%H%M%S}" await _write_standings_bronze( db, source_record_id=f"standings:{code}:{season_label}", raw_payload=payload, league_id=league.id, league_code=code, season_label=season_label, rows_upserted=league_r["upserted"], batch_id=bronze_batch_id, ) result["leagues"][code] = league_r result["total_upserted"] += league_r["upserted"] logger.info( "bzzoiro standings 采集完成: %s 赛季 %s, upsert %d/%d", code, season_label, league_r["upserted"], league_r["rows"], ) return result async def _write_standings_bronze( db, *, source_record_id: str, raw_payload: dict, league_id: int | None, league_code: str, season_label: str, rows_upserted: int, batch_id: str, ) -> None: """standings 成功 upsert 一个联赛后的 Bronze 层补写:RawEvent(幂等) + DataLineage。 与 _write_events_bronze 同级约束:幂等性由 _write_raw_event 的 source_record_id 查重保证(积分榜是联赛级快照,同联赛同赛季重复采集 命中同一条 RawEvent);best-effort:基础设施写入失败只记 warning, 绝不拖垮采集主流程。 """ try: await _write_raw_event(db, "bzzoiro", source_record_id, raw_payload, batch_id) await _write_lineage( db, "bzzoiro", source_record_id, "standings", league_id, "standings_ingest", {"league": league_code, "season": season_label, "rows_upserted": rows_upserted}, batch_id, ) except Exception: logger.warning( "standings Bronze 写入失败(record=%s, league=%s),不影响采集主流程", source_record_id, league_code, exc_info=True, )