feat: 数据采集健壮性改进
- 新增 src/core/retry.py: 指数退避 + 抖动的重试装饰器 - bzzoiro: 改进重试逻辑 - 429/5xx/网络错误均触发指数退避重试 - 4xx 直接抛(不重试) - understat: 新增重试(原无重试) - injuries: 新增重试 + 文件缓存 7 天过期 - sources.py: 延迟导入修复循环导入
This commit is contained in:
+19
-4
@@ -7,6 +7,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json as _json
|
||||
import logging
|
||||
import random
|
||||
import time as _time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
@@ -35,6 +36,7 @@ def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3
|
||||
if not key:
|
||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
@@ -43,12 +45,25 @@ def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return _json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
last_exc = e
|
||||
if e.code == 429:
|
||||
logger.warning("bzzoiro 429, retry %d", attempt + 1)
|
||||
_time.sleep(1)
|
||||
# 指数退避: 429 通常意味着限速
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
||||
_time.sleep(delay)
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("bzzoiro rate limit exceeded")
|
||||
if 500 <= e.code < 600:
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro %d, retry %d in %.1fs", e.code, attempt + 1, delay)
|
||||
_time.sleep(delay)
|
||||
continue
|
||||
raise # 4xx 直接抛
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
||||
last_exc = e
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||
_time.sleep(delay)
|
||||
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
||||
|
||||
|
||||
async def fetch_bzzoiro_events(
|
||||
|
||||
Reference in New Issue
Block a user