shangfangjian
317a5e338a
单文件 852 行按职责拆分,保持 BzzoiroSource 与 get_source("bzzoiro") 行为不变:
- bzzoiro_common HTTP 抓取(多 key 轮换) + 字段转换原语
- bzzoiro_events fetch_bzzoiro_events + BzzoiroSource.ingest + Bronze 补写
- bzzoiro_standings standings 管线
- bzzoiro_stats stats 回填
- pipeline_write RawEvent/IngestFailure/DataLineage 写入助手
子模块运行期经聚合门面 src.data.bzzoiro 解析可替换协作者,
单文件时代的 bz.* monkeypatch 语义完全保留。
路由 import 已指向新模块(ingest.py / schedules.py)。
124 lines
4.7 KiB
Python
124 lines
4.7 KiB
Python
"""bzzoiro 管线共享原语:HTTP 抓取(多 key 轮换)与宽松字段转换。
|
|
|
|
从 bzzoiro.py 拆出(单文件 → 多模块):仅放无业务语义的共享基础,
|
|
三条管线(events/standings/stats)与聚合门面见 bzzoiro.py。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import random
|
|
|
|
import httpx
|
|
|
|
from src.core.http_client import get_client
|
|
from src.core.runtime_config import get_runtime_value
|
|
from src.data.key_ring import _mask, get_key_ring
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def _to_date(value):
|
|
"""把 datetime / date / str 统一成 `date`。"""
|
|
if value is None:
|
|
return None
|
|
if hasattr(value, "date") and callable(value.date):
|
|
return value.date()
|
|
return value
|
|
|
|
|
|
def _to_int_or_none(value) -> int | None:
|
|
"""宽松转 int(用于上游 ID 解析,失败返回 None 不抛错)。"""
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return int(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _to_float_or_none(value) -> float | None:
|
|
try:
|
|
return float(str(value).strip())
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]:
|
|
"""比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。
|
|
|
|
统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类
|
|
隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配,
|
|
导致所有比赛被判为不存在而重复插入。
|
|
"""
|
|
d = _to_date(match_date)
|
|
return (home_team_id, away_team_id, d.isoformat() if d is not None else "")
|
|
|
|
|
|
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
|
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。
|
|
|
|
多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。
|
|
"""
|
|
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
|
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
|
ring = get_key_ring(base, raw_keys)
|
|
|
|
url = f"{base}/{path.lstrip('/')}"
|
|
key = ring.get()
|
|
if not key:
|
|
raise RuntimeError("BZZOIRO_KEY 未设置")
|
|
|
|
last_exc: Exception | None = None
|
|
for attempt in range(max_retries):
|
|
headers = {
|
|
"Authorization": f"Token {key}",
|
|
"Accept": "application/json",
|
|
}
|
|
try:
|
|
client = get_client()
|
|
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
|
|
resp = await asyncio.wait_for(
|
|
client.get(
|
|
url, headers=headers, params=params,
|
|
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
|
|
),
|
|
timeout=60.0,
|
|
)
|
|
resp.raise_for_status()
|
|
return resp.json()
|
|
except Exception as e:
|
|
last_exc = e
|
|
status = getattr(getattr(e, "response", None), "status_code", None)
|
|
if status == 429:
|
|
# 限流:标记当前 key 冷却,切换到下一个
|
|
new_key = ring.report_rate_limited(key)
|
|
if new_key and new_key != key:
|
|
logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _mask(key), _mask(new_key))
|
|
key = new_key
|
|
continue # 立即重试,不等待
|
|
# 单 key 或全部冷却:等待最早恢复的 key
|
|
wait = ring.wait_if_all_blocked()
|
|
if wait > 0:
|
|
logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait)
|
|
await asyncio.sleep(min(wait, 30.0))
|
|
else:
|
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
|
await asyncio.sleep(delay)
|
|
key = ring.get() or key
|
|
continue
|
|
if 500 <= (status or 0) < 600:
|
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay)
|
|
await asyncio.sleep(delay)
|
|
continue
|
|
# 网络错误(连接失败/超时)也退避重试
|
|
if isinstance(e, (TimeoutError, ConnectionError, OSError)):
|
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
|
await asyncio.sleep(delay)
|
|
continue
|
|
raise
|
|
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|