feat: 核心模块增强 — 加密 + 运行时配置 + 日志缓冲
- crypto.py: API Key 加密/解密工具 - runtime_config.py: 运行时动态配置管理 - log_buffer.py: 内存日志缓冲区 - config.py: 新增加密配置项 - http_client.py: 增强重试和错误处理
This commit is contained in:
+20
-9
@@ -14,10 +14,13 @@ from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.core.config import settings
|
||||
import httpx
|
||||
|
||||
from src.core.runtime_config import get_runtime_value
|
||||
from src.core.http_client import get_client
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.team_names_zh import zh_name
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats, Team
|
||||
|
||||
@@ -46,9 +49,9 @@ def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, i
|
||||
|
||||
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。"""
|
||||
base = settings.BZZOIRO_BASE.rstrip("/")
|
||||
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
||||
url = f"{base}/{path.lstrip('/')}"
|
||||
key = settings.BZZOIRO_KEY
|
||||
key = await get_runtime_value("BZZOIRO_KEY")
|
||||
if not key:
|
||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||
|
||||
@@ -61,7 +64,14 @@ async def _fetch_json_async(path: str, params: dict | None = None, max_retries:
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
client = get_client()
|
||||
resp = await client.get(url, headers=headers, params=params, timeout=30)
|
||||
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
|
||||
resp = await asyncio.wait_for(
|
||||
client.get(
|
||||
url, headers=headers, params=params,
|
||||
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
|
||||
),
|
||||
timeout=60.0,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
@@ -199,7 +209,8 @@ class BzzoiroSource:
|
||||
# 避免加载联赛全部历史比赛到内存(多赛季采集时内存溢出)
|
||||
if normalized_matches:
|
||||
from datetime import timedelta
|
||||
dates = [nm.date for nm in normalized_matches if nm.date is not None]
|
||||
# normalized_matches 存的是 (nm, raw) 元组,遍历需解包
|
||||
dates = [nm.date for nm, _raw in normalized_matches if nm.date is not None]
|
||||
if dates:
|
||||
min_dt = min(dates) - timedelta(days=30)
|
||||
max_dt = max(dates) + timedelta(days=30)
|
||||
@@ -219,7 +230,7 @@ class BzzoiroSource:
|
||||
# 球队: 内存查找 + 按需创建
|
||||
home_team_id = team_name_to_id.get(nm.home_team)
|
||||
if home_team_id is None:
|
||||
home = Team(name=nm.home_team)
|
||||
home = Team(name=nm.home_team, name_zh=zh_name(nm.home_team))
|
||||
db.add(home)
|
||||
await db.flush()
|
||||
home_team_id = home.id
|
||||
@@ -227,7 +238,7 @@ class BzzoiroSource:
|
||||
|
||||
away_team_id = team_name_to_id.get(nm.away_team)
|
||||
if away_team_id is None:
|
||||
away = Team(name=nm.away_team)
|
||||
away = Team(name=nm.away_team, name_zh=zh_name(nm.away_team))
|
||||
db.add(away)
|
||||
await db.flush()
|
||||
away_team_id = away.id
|
||||
@@ -273,7 +284,7 @@ class BzzoiroSource:
|
||||
home_red_cards=nm.home_red_cards,
|
||||
away_red_cards=nm.away_red_cards,
|
||||
source="bzzoiro",
|
||||
source_event_id=str(raw.get("id", "")),
|
||||
source_record_id=str(raw.get("id", "")),
|
||||
retrieved_at=now,
|
||||
available_at=now,
|
||||
)
|
||||
@@ -299,7 +310,7 @@ class BzzoiroSource:
|
||||
existing_match.stats = MatchStats(
|
||||
match_id=existing_match.id,
|
||||
source="bzzoiro",
|
||||
source_event_id=str(raw.get("id", "")),
|
||||
source_record_id=str(raw.get("id", "")),
|
||||
retrieved_at=now,
|
||||
available_at=now,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user