feat: 数据采集健壮性改进

- 新增 src/core/retry.py: 指数退避 + 抖动的重试装饰器
- bzzoiro: 改进重试逻辑
  - 429/5xx/网络错误均触发指数退避重试
  - 4xx 直接抛(不重试)
- understat: 新增重试(原无重试)
- injuries: 新增重试 + 文件缓存 7 天过期
- sources.py: 延迟导入修复循环导入
This commit is contained in:
shangfangjian
2026-09-09 21:47:56 +08:00
parent 1166a157ec
commit 494751bbb0
5 changed files with 166 additions and 18 deletions
+19 -4
View File
@@ -7,6 +7,7 @@ from __future__ import annotations
import asyncio
import json as _json
import logging
import random
import time as _time
import urllib.error
import urllib.parse
@@ -35,6 +36,7 @@ def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3
if not key:
raise RuntimeError("BZZOIRO_KEY 未设置")
last_exc: Exception | None = None
for attempt in range(max_retries):
try:
req = urllib.request.Request(url)
@@ -43,12 +45,25 @@ def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3
with urllib.request.urlopen(req, timeout=30) as resp:
return _json.loads(resp.read().decode("utf-8"))
except urllib.error.HTTPError as e:
last_exc = e
if e.code == 429:
logger.warning("bzzoiro 429, retry %d", attempt + 1)
_time.sleep(1)
# 指数退避: 429 通常意味着限速
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
_time.sleep(delay)
continue
raise
raise RuntimeError("bzzoiro rate limit exceeded")
if 500 <= e.code < 600:
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
logger.warning("bzzoiro %d, retry %d in %.1fs", e.code, attempt + 1, delay)
_time.sleep(delay)
continue
raise # 4xx 直接抛
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
last_exc = e
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
_time.sleep(delay)
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
async def fetch_bzzoiro_events(