feat: bzzoiro 多 API Key 轮换,遇限流自动切换
新增 KeyRing 轮换环(src/data/key_ring.py): - 支持逗号/分号/换行分隔多个 key,单 key 场景零开销 - 遇到 429 自动标记当前 key 冷却(默认 60s)并立即切换到下一个 key - 全部 key 冷却时等待最早恢复的 key,避免无谓重试 - 热更新 key 列表(增删 key 无需重启) - 进程级单例,按 base URL 隔离 后端集成: - bzzoiro._fetch_json_async 接入 KeyRing,429 立即轮换(不等待) - 新增 /admin/keyring/status 端点展示 key 环状态 - 新增 /admin/keyring/cooldown/reset 紧急重置冷却 - BZZOIRO_KEY 配置描述提示多 key 支持 - mask_value 多 key 显示数量(如 "3 个 key(末段 …XXXX)") - 清理已移除 injuries 的 API_FOOTBALL_KEY 配置项 前端: - 数据源页新增 Key Ring 状态面板(每个 key 可用/冷却状态 + 重置按钮) - 配置项脱敏展示支持多 key 计数 测试: 新增 24 个 KeyRing 单元测试 + 2 个 bzzoiro 轮换集成测试,全部通过。 Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
co-authored by
new-provider/LongCat-2.0 <
new-provider/LongCat-2.0 <
parent
f05dc1ae15
commit
f52ec8b963
+36
-10
@@ -22,6 +22,7 @@ import httpx
|
||||
from src.core.runtime_config import get_runtime_value
|
||||
from src.core.http_client import get_client
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||
from src.data.key_ring import get_key_ring
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.team_names_zh import zh_name
|
||||
from src.data.sources import register
|
||||
@@ -61,20 +62,25 @@ def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, i
|
||||
|
||||
|
||||
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。"""
|
||||
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。
|
||||
|
||||
多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。
|
||||
"""
|
||||
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
||||
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
||||
ring = get_key_ring(base, raw_keys)
|
||||
|
||||
url = f"{base}/{path.lstrip('/')}"
|
||||
key = await get_runtime_value("BZZOIRO_KEY")
|
||||
key = ring.get()
|
||||
if not key:
|
||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Token {key}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_retries):
|
||||
headers = {
|
||||
"Authorization": f"Token {key}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
try:
|
||||
client = get_client()
|
||||
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
|
||||
@@ -91,9 +97,22 @@ async def _fetch_json_async(path: str, params: dict | None = None, max_retries:
|
||||
last_exc = e
|
||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status == 429:
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
||||
await asyncio.sleep(delay)
|
||||
# 限流:标记当前 key 冷却,切换到下一个
|
||||
new_key = ring.report_rate_limited(key)
|
||||
if new_key and new_key != key:
|
||||
logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _km(key), _km(new_key))
|
||||
key = new_key
|
||||
continue # 立即重试,不等待
|
||||
# 单 key 或全部冷却:等待最早恢复的 key
|
||||
wait = ring.wait_if_all_blocked()
|
||||
if wait > 0:
|
||||
logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait)
|
||||
await asyncio.sleep(min(wait, 30.0))
|
||||
else:
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
||||
await asyncio.sleep(delay)
|
||||
key = ring.get() or key
|
||||
continue
|
||||
if 500 <= (status or 0) < 600:
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
@@ -110,6 +129,13 @@ async def _fetch_json_async(path: str, params: dict | None = None, max_retries:
|
||||
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
||||
|
||||
|
||||
def _km(key: str) -> str:
|
||||
"""key 脱敏缩写(用于日志)。"""
|
||||
if len(key) <= 8:
|
||||
return key[:2] + "***"
|
||||
return key[:4] + "..." + key[-4:]
|
||||
|
||||
|
||||
async def fetch_bzzoiro_events(
|
||||
league_code: str,
|
||||
*,
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""API Key 轮换环:多 key 自动切换,遇到限流(429)自动跳过已冷却 key。
|
||||
|
||||
设计:
|
||||
- 进程内纯内存状态(限速是短时状态,无需持久化)
|
||||
- 单 key 场景零开销:直接透传
|
||||
- 多 key 场景:429 时把当前 key 标记冷却(默认 60s),轮转到下一个可用 key
|
||||
- 全部 key 都在冷却时:使用最早冷却的那个 key 并等待(退化到单 key 重试)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_COOLDOWN = 60.0 # 单个 key 被限流后的冷却时间(秒)
|
||||
|
||||
|
||||
class KeyRing:
|
||||
"""多 key 轮换环。在 async 单线程事件循环下无需加锁。"""
|
||||
|
||||
def __init__(self, keys: list[str], cooldown_seconds: float = DEFAULT_COOLDOWN) -> None:
|
||||
self._keys: list[str] = [k.strip() for k in keys if k and k.strip()]
|
||||
self._cooldown = cooldown_seconds
|
||||
# key → 冷却过期时间戳(时刻);不在表中表示可用
|
||||
self._blocked_until: dict[str, float] = {}
|
||||
self._index = 0 # 当前轮转位置
|
||||
|
||||
@property
|
||||
def has_multiple(self) -> bool:
|
||||
return len(self._keys) > 1
|
||||
|
||||
@property
|
||||
def all_keys(self) -> list[str]:
|
||||
return list(self._keys)
|
||||
|
||||
@property
|
||||
def active_key(self) -> str | None:
|
||||
"""当前指向的 key(即使正在冷却也返回,用于上报)。"""
|
||||
if not self._keys:
|
||||
return None
|
||||
return self._keys[self._index]
|
||||
|
||||
def get(self) -> str | None:
|
||||
"""获取一个可用 key:优先选不在冷却中的;全部冷却则选最早过期的。"""
|
||||
if not self._keys:
|
||||
return None
|
||||
if len(self._keys) == 1:
|
||||
return self._keys[0]
|
||||
|
||||
now = time.monotonic()
|
||||
n = len(self._keys)
|
||||
# 从当前 index 开始找一圈,找一个可用的
|
||||
for offset in range(n):
|
||||
idx = (self._index + offset) % n
|
||||
key = self._keys[idx]
|
||||
expire = self._blocked_until.get(key, 0.0)
|
||||
if now >= expire:
|
||||
# 可用:把指针移到这里
|
||||
self._index = idx
|
||||
# 清理已过期的冷却记录
|
||||
if key in self._blocked_until:
|
||||
del self._blocked_until[key]
|
||||
return key
|
||||
|
||||
# 全部在冷却中:选最早过期的那个,并等待到它过期
|
||||
earliest_key = min(self._keys, key=lambda k: self._blocked_until.get(k, 0.0))
|
||||
self._index = self._keys.index(earliest_key)
|
||||
return earliest_key
|
||||
|
||||
def report_rate_limited(self, key: str | None = None) -> str | None:
|
||||
"""上报某个 key 被限流(429)。默认是当前 key。返回切换后的新 key。"""
|
||||
target = key or self.active_key
|
||||
if target and len(self._keys) > 1:
|
||||
until = time.monotonic() + self._cooldown
|
||||
self._blocked_until[target] = until
|
||||
logger.warning(
|
||||
"bzzoiro key 被限流(429),冷却 %.0fs: %s", self._cooldown, _mask(target),
|
||||
)
|
||||
# 轮转到下一个(即使只有一个 key 也做一次 get,保持行为一致)
|
||||
return self.get()
|
||||
|
||||
def wait_if_all_blocked(self) -> float:
|
||||
"""如果所有 key 都在冷却中,返回需要等待的秒数;否则返回 0。"""
|
||||
if len(self._keys) <= 1:
|
||||
return 0.0
|
||||
now = time.monotonic()
|
||||
remaining = [self._blocked_until.get(k, 0.0) - now for k in self._keys]
|
||||
if all(r > -0.001 for r in remaining) and any(r > 0.001 for r in remaining):
|
||||
# 全部仍在冷却中
|
||||
return max(remaining)
|
||||
return 0.0
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""当前 key 环状态(用于管理后台展示)。"""
|
||||
now = time.monotonic()
|
||||
return {
|
||||
"total": len(self._keys),
|
||||
"keys": [
|
||||
{
|
||||
"masked": _mask(k),
|
||||
"blocked_remaining": max(0.0, round(self._blocked_until.get(k, 0.0) - now, 1)),
|
||||
}
|
||||
for k in self._keys
|
||||
],
|
||||
"active_index": self._index,
|
||||
}
|
||||
|
||||
|
||||
def _mask(key: str) -> str:
|
||||
"""脱敏:只显示前 4 位和后 4 位。"""
|
||||
if len(key) <= 10:
|
||||
return key[:2] + "***"
|
||||
return key[:4] + "..." + key[-4:]
|
||||
|
||||
|
||||
# ── 全局单例(进程级,按 base URL 隔离) ──────────────────────────
|
||||
_RINGS: dict[str, KeyRing] = {}
|
||||
|
||||
|
||||
def parse_keys(value: str | None) -> list[str]:
|
||||
"""解析 key 配置值:支持逗号、分号、换行分隔的多个 key。"""
|
||||
if not value:
|
||||
return []
|
||||
# 统一替换分隔符为逗号后拆分
|
||||
normalized = value.replace("\n", ",").replace(";", ",")
|
||||
return [k.strip() for k in normalized.split(",") if k.strip()]
|
||||
|
||||
|
||||
def get_key_ring(base: str, raw_keys: str | None, cooldown_seconds: float = DEFAULT_COOLDOWN) -> KeyRing:
|
||||
"""获取(或创建)某 base URL 对应的 KeyRing。"""
|
||||
key = base
|
||||
ring = _RINGS.get(key)
|
||||
parsed = parse_keys(raw_keys)
|
||||
if ring is None:
|
||||
ring = KeyRing(parsed, cooldown_seconds)
|
||||
_RINGS[key] = ring
|
||||
else:
|
||||
# 热更新 key 列表(增删 key 无需重启)
|
||||
if set(ring.all_keys) != set(parsed):
|
||||
ring._keys = parsed
|
||||
ring._blocked_until.clear()
|
||||
ring._index = 0
|
||||
ring._cooldown = cooldown_seconds
|
||||
return ring
|
||||
Reference in New Issue
Block a user