feat: 数据采集健壮性改进
- 新增 src/core/retry.py: 指数退避 + 抖动的重试装饰器 - bzzoiro: 改进重试逻辑 - 429/5xx/网络错误均触发指数退避重试 - 4xx 直接抛(不重试) - understat: 新增重试(原无重试) - injuries: 新增重试 + 文件缓存 7 天过期 - sources.py: 延迟导入修复循环导入
This commit is contained in:
+20
-3
@@ -4,8 +4,10 @@
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
|
||||
from src.core.http_client import get_client
|
||||
@@ -41,9 +43,24 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
|
||||
"Referer": f"https://understat.com/league/{understat_league}/{season}",
|
||||
}
|
||||
|
||||
client = get_client()
|
||||
resp = await client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
# 重试:网络错误 / 5xx / 429
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
client = get_client()
|
||||
resp = await client.get(url, headers=headers, timeout=30)
|
||||
resp.raise_for_status()
|
||||
break
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
if attempt == 2:
|
||||
raise
|
||||
delay = min(2 ** attempt, 8) + random.uniform(0, 1)
|
||||
logger.warning("understat fetch failed, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||
await asyncio.sleep(delay)
|
||||
else:
|
||||
raise RuntimeError(f"understat fetch failed: {last_exc}")
|
||||
|
||||
# understat 返回 JS 对象,需要提取 JSON
|
||||
text = resp.text
|
||||
# 匹配 var datesData = JSON.parse('...');
|
||||
|
||||
Reference in New Issue
Block a user