feat: 数据采集健壮性改进
- 新增 src/core/retry.py: 指数退避 + 抖动的重试装饰器 - bzzoiro: 改进重试逻辑 - 429/5xx/网络错误均触发指数退避重试 - 4xx 直接抛(不重试) - understat: 新增重试(原无重试) - injuries: 新增重试 + 文件缓存 7 天过期 - sources.py: 延迟导入修复循环导入
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
"""重试工具:带指数退避的瞬态错误重试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from typing import Callable, Iterable, TypeVar
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def with_retry(
|
||||
*,
|
||||
max_retries: int = 3,
|
||||
base_delay: float = 1.0,
|
||||
max_delay: float = 30.0,
|
||||
retryable_exceptions: Iterable[type[BaseException]] = (Exception,),
|
||||
on_retry: Callable[[Exception, int], None] | None = None,
|
||||
) -> Callable:
|
||||
"""重试装饰器(同步/异步通用,指数退避 + 抖动)。
|
||||
|
||||
Args:
|
||||
max_retries: 最大重试次数
|
||||
base_delay: 基础延迟(秒)
|
||||
max_delay: 最大延迟(秒)
|
||||
retryable_exceptions: 触发重试的异常类型
|
||||
on_retry: 重试回调(exception, attempt)
|
||||
"""
|
||||
retryable = tuple(retryable_exceptions)
|
||||
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@functools.wraps(func)
|
||||
async def async_wrapper(*args, **kwargs):
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
return await func(*args, **kwargs)
|
||||
except retryable as e:
|
||||
last_exc = e
|
||||
if attempt == max_retries:
|
||||
break
|
||||
delay = min(base_delay * (2 ** attempt), max_delay)
|
||||
delay += random.uniform(0, delay * 0.1) # 抖动
|
||||
logger.warning(
|
||||
"%s failed (attempt %d/%d), retry in %.1fs: %s",
|
||||
func.__name__, attempt + 1, max_retries, delay, e,
|
||||
)
|
||||
if on_retry:
|
||||
on_retry(e, attempt + 1)
|
||||
await asyncio.sleep(delay)
|
||||
raise last_exc # type: ignore[misc]
|
||||
|
||||
@functools.wraps(func)
|
||||
def sync_wrapper(*args, **kwargs):
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except retryable as e:
|
||||
last_exc = e
|
||||
if attempt == max_retries:
|
||||
break
|
||||
delay = min(base_delay * (2 ** attempt), max_delay)
|
||||
delay += random.uniform(0, delay * 0.1)
|
||||
logger.warning(
|
||||
"%s failed (attempt %d/%d), retry in %.1fs: %s",
|
||||
func.__name__, attempt + 1, max_retries, delay, e,
|
||||
)
|
||||
if on_retry:
|
||||
on_retry(e, attempt + 1)
|
||||
time.sleep(delay)
|
||||
raise last_exc # type: ignore[misc]
|
||||
|
||||
if asyncio.iscoroutinefunction(func):
|
||||
return async_wrapper
|
||||
return sync_wrapper
|
||||
|
||||
return decorator
|
||||
Reference in New Issue
Block a user