"""Bzzoiro 数据源:抓取 + 入库。 迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。 """ from __future__ import annotations import asyncio import json as _json import logging import random import time as _time import urllib.error import urllib.parse import urllib.request from collections.abc import Iterable from sqlalchemy import select from src.core.config import settings from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL from src.data.match_lookup import find_existing_match, get_or_create_team from src.data.normalize import normalize_bzzoiro from src.data.sources import register from src.db.models import League, Match, MatchStats logger = logging.getLogger(__name__) def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list: """同步 HTTP(bzzoiro 客户端保持同步,在 async 函数里 run_in_executor)。""" base = settings.BZZOIRO_BASE.rstrip("/") url = f"{base}/{path.lstrip('/')}" if params: url += "?" + urllib.parse.urlencode(params) key = settings.BZZOIRO_KEY if not key: raise RuntimeError("BZZOIRO_KEY 未设置") last_exc: Exception | None = None for attempt in range(max_retries): try: req = urllib.request.Request(url) req.add_header("Authorization", f"Token {key}") req.add_header("Accept", "application/json") with urllib.request.urlopen(req, timeout=30) as resp: return _json.loads(resp.read().decode("utf-8")) except urllib.error.HTTPError as e: last_exc = e if e.code == 429: # 指数退避: 429 通常意味着限速 delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay) _time.sleep(delay) continue if 500 <= e.code < 600: delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro %d, retry %d in %.1fs", e.code, attempt + 1, delay) _time.sleep(delay) continue raise # 4xx 直接抛 except (urllib.error.URLError, TimeoutError, ConnectionError) as e: last_exc = e delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e) _time.sleep(delay) raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}") async def fetch_bzzoiro_events( league_code: str, *, status: str = "finished", date_from: str | None = None, date_to: str | None = None, limit: int = 200, ) -> list[dict]: """抓取 bzzoiro 原始事件(异步包装)。""" league_id = BZZOIRO_LEAGUE_IDS.get(league_code) if league_id is None: raise ValueError(f"未知联赛代码: {league_code}") loop = asyncio.get_event_loop() rows: list[dict] = [] offset = 0 payload: dict | list = {} while True: params: dict = { "league_id": league_id, "status": status, "limit": limit, "offset": offset, } if date_from: params["date_from"] = str(date_from)[:10] if date_to: params["date_to"] = str(date_to)[:10] # 显式位置参数,避免 lambda 闭包捕获循环变量 payload = await loop.run_in_executor(None, _fetch_json_sync, "/events/", params) batch = payload.get("results") or [] if not batch: break rows.extend(batch) total = payload.get("total") offset += limit if total is not None and offset >= total: break if len(batch) < limit: break await asyncio.sleep(REQUEST_INTERVAL) return rows @register class BzzoiroSource: """bzzoiro 数据源(实现 DataSource 协议)。""" name = "bzzoiro" async def ingest( self, db, *, leagues: Iterable[str], date_from: str | None = None, date_to: str | None = None, status: str = "finished", ) -> dict: """采集 bzzoiro → 入库。返回统计。""" result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []} for code in leagues: league_r: dict = {"inserted": 0, "updated": 0, "errors": []} try: raw_events = await fetch_bzzoiro_events(code, status=status, date_from=date_from, date_to=date_to) except Exception as e: logger.exception("bzzoiro fetch failed for %s", code) league_r["errors"].append(f"fetch failed: {e}") result["leagues"][code] = league_r continue # 获取或创建联赛 stmt = select(League).where(League.code == code) league = (await db.execute(stmt)).scalar_one_or_none() if league is None: league = League(code=code, name=LEAGUE_NAMES.get(code, code), country=LEAGUE_COUNTRIES.get(code)) db.add(league) await db.flush() # === 批量优化: 预加载球队和已有比赛到内存 === team_name_to_id: dict[str, int] = {} existing_match_keys: set[tuple[int, int, str]] = set() if raw_events: # 预加载所有涉及的球队名 all_team_names = set() for raw in raw_events: nm = normalize_bzzoiro(raw, code) if nm: all_team_names.add(nm.home_team) all_team_names.add(nm.away_team) if all_team_names: from sqlalchemy import select from src.db.models import Team stmt = select(Team).where(Team.name.in_(all_team_names)) teams = (await db.execute(stmt)).scalars().all() team_name_to_id = {t.name: t.id for t in teams} # 预加载已有比赛 (league_id + home_id + away_id + date) # 需要先获取球队 ID,所以分批处理 date_strs = set() for raw in raw_events: nm = normalize_bzzoiro(raw, code) if nm and nm.date: date_strs.add(nm.date.date().isoformat() if hasattr(nm.date, "date") else str(nm.date)) if date_strs: from sqlalchemy import func stmt = ( select(Match.home_team_id, Match.away_team_id, func.date(Match.match_date).label("d")) .where(Match.league_id == league.id) ) rows = (await db.execute(stmt)).all() for row in rows: existing_match_keys.add((row.home_team_id, row.away_team_id, str(row.d))) for raw in raw_events: try: nm = normalize_bzzoiro(raw, code) if nm is None: continue nm.validate() except Exception as e: logger.debug("normalize skip: %s", e) league_r["errors"].append(f"normalize: {e}") continue # 球队: 内存查找 + 按需创建 home_team_id = team_name_to_id.get(nm.home_team) if home_team_id is None: home = Team(name=nm.home_team) db.add(home) await db.flush() home_team_id = home.id team_name_to_id[nm.home_team] = home_team_id away_team_id = team_name_to_id.get(nm.away_team) if away_team_id is None: away = Team(name=nm.away_team) db.add(away) await db.flush() away_team_id = away.id team_name_to_id[nm.away_team] = away_team_id # 查找已有比赛: 内存查找 date_key = nm.date.date().isoformat() if hasattr(nm.date, "date") else str(nm.date) match_key = (home_team_id, away_team_id, date_key) existing = None if match_key not in existing_match_keys else "exists" if existing is None: m = Match( league_id=league.id, season=nm.season_label or None, home_team_id=home_team_id, away_team_id=away_team_id, match_date=nm.date, match_date_date=nm.date.date() if hasattr(nm.date, "date") else nm.date, match_status=nm.match_status, home_goals=nm.home_goals, away_goals=nm.away_goals, home_ht_goals=nm.home_ht_goals, away_ht_goals=nm.away_ht_goals, match_stage=nm.match_stage, ) db.add(m) await db.flush() existing_match_keys.add(match_key) # 防止同批重复 if nm.home_xg is not None or nm.away_xg is not None: stats = MatchStats( match_id=m.id, home_xg=nm.home_xg, away_xg=nm.away_xg, home_shots=nm.home_shots, away_shots=nm.away_shots, home_shots_on_target=nm.home_shots_on_target, away_shots_on_target=nm.away_shots_on_target, home_corners=nm.home_corners, away_corners=nm.away_corners, home_possession=nm.home_possession, home_yellow_cards=nm.home_yellow_cards, away_yellow_cards=nm.away_yellow_cards, home_red_cards=nm.home_red_cards, away_red_cards=nm.away_red_cards, ) db.add(stats) league_r["inserted"] += 1 else: # 已有比赛: 需要查询对象来更新 # 注意: 这里为了简化仍查询一次,但只在"已有"时触发 from sqlalchemy import func stmt = ( select(Match) .where(Match.league_id == league.id) .where(Match.home_team_id == home_team_id) .where(Match.away_team_id == away_team_id) .where(func.date(Match.match_date) == date_key) ) existing_match = (await db.execute(stmt)).scalar_one() changed = False if existing_match.match_status != nm.match_status and nm.match_status == "finished": existing_match.match_status = nm.match_status changed = True if existing_match.home_goals is None and nm.home_goals is not None: existing_match.home_goals = nm.home_goals existing_match.away_goals = nm.away_goals existing_match.home_ht_goals = nm.home_ht_goals existing_match.away_ht_goals = nm.away_ht_goals changed = True if existing_match.match_stage is None and nm.match_stage: existing_match.match_stage = nm.match_stage changed = True if existing_match.stats is None and (nm.home_xg is not None or nm.away_xg is not None): existing_match.stats = MatchStats(match_id=existing_match.id) db.add(existing_match.stats) await db.flush() if existing_match.stats is not None: for fld in ("home_xg", "away_xg", "home_shots", "away_shots", "home_shots_on_target", "away_shots_on_target", "home_corners", "away_corners", "home_possession", "home_yellow_cards", "away_yellow_cards", "home_red_cards", "away_red_cards"): if getattr(existing_match.stats, fld, None) is None: v = getattr(nm, fld, None) if v is not None: setattr(existing_match.stats, fld, v) changed = True if changed: league_r["updated"] += 1 await db.commit() result["leagues"][code] = league_r result["total_inserted"] += league_r["inserted"] result["total_updated"] += league_r["updated"] return result