"""bzzoiro 管线共享原语:HTTP 抓取(多 key 轮换)与宽松字段转换。 从 bzzoiro.py 拆出(单文件 → 多模块):仅放无业务语义的共享基础, 三条管线(events/standings/stats)与聚合门面见 bzzoiro.py。 """ from __future__ import annotations import asyncio import logging import random import httpx from src.core.http_client import get_client from src.core.runtime_config import get_runtime_value from src.data.key_ring import _mask, get_key_ring logger = logging.getLogger(__name__) def _to_date(value): """把 datetime / date / str 统一成 `date`。""" if value is None: return None if hasattr(value, "date") and callable(value.date): return value.date() return value def _to_int_or_none(value) -> int | None: """宽松转 int(用于上游 ID 解析,失败返回 None 不抛错)。""" if value is None: return None try: return int(str(value).strip()) except (TypeError, ValueError): return None def _to_float_or_none(value) -> float | None: try: return float(str(value).strip()) except (TypeError, ValueError): return None def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]: """比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。 统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类 隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配, 导致所有比赛被判为不存在而重复插入。 """ d = _to_date(match_date) return (home_team_id, away_team_id, d.isoformat() if d is not None else "") async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list: """异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。 多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。 """ base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/") raw_keys = await get_runtime_value("BZZOIRO_KEY") ring = get_key_ring(base, raw_keys) url = f"{base}/{path.lstrip('/')}" key = ring.get() if not key: raise RuntimeError("BZZOIRO_KEY 未设置") last_exc: Exception | None = None for attempt in range(max_retries): headers = { "Authorization": f"Token {key}", "Accept": "application/json", } try: client = get_client() # 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死 resp = await asyncio.wait_for( client.get( url, headers=headers, params=params, timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0), ), timeout=60.0, ) resp.raise_for_status() return resp.json() except Exception as e: last_exc = e status = getattr(getattr(e, "response", None), "status_code", None) if status == 429: # 限流:标记当前 key 冷却,切换到下一个 new_key = ring.report_rate_limited(key) if new_key and new_key != key: logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _mask(key), _mask(new_key)) key = new_key continue # 立即重试,不等待 # 单 key 或全部冷却:等待最早恢复的 key wait = ring.wait_if_all_blocked() if wait > 0: logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait) await asyncio.sleep(min(wait, 30.0)) else: delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay) await asyncio.sleep(delay) key = ring.get() or key continue if 500 <= (status or 0) < 600: delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay) await asyncio.sleep(delay) continue # 网络错误(连接失败/超时)也退避重试 if isinstance(e, (TimeoutError, ConnectionError, OSError)): delay = min(2 ** attempt, 16) + random.uniform(0, 1) logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e) await asyncio.sleep(delay) continue raise raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")