feat: 核心模块增强 — 加密 + 运行时配置 + 日志缓冲

- crypto.py: API Key 加密/解密工具
- runtime_config.py: 运行时动态配置管理
- log_buffer.py: 内存日志缓冲区
- config.py: 新增加密配置项
- http_client.py: 增强重试和错误处理
This commit is contained in:
shangfangjian
2026-09-19 11:58:03 +08:00
parent b3e2c52b49
commit 786f10aa11
57 changed files with 3178 additions and 488 deletions
+20 -9
View File
@@ -14,10 +14,13 @@ from datetime import datetime, timezone
from sqlalchemy import select
from src.core.config import settings
import httpx
from src.core.runtime_config import get_runtime_value
from src.core.http_client import get_client
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
from src.data.normalize import normalize_bzzoiro
from src.data.team_names_zh import zh_name
from src.data.sources import register
from src.db.models import League, Match, MatchStats, Team
@@ -46,9 +49,9 @@ def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, i
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。"""
base = settings.BZZOIRO_BASE.rstrip("/")
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
url = f"{base}/{path.lstrip('/')}"
key = settings.BZZOIRO_KEY
key = await get_runtime_value("BZZOIRO_KEY")
if not key:
raise RuntimeError("BZZOIRO_KEY 未设置")
@@ -61,7 +64,14 @@ async def _fetch_json_async(path: str, params: dict | None = None, max_retries:
for attempt in range(max_retries):
try:
client = get_client()
resp = await client.get(url, headers=headers, params=params, timeout=30)
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
resp = await asyncio.wait_for(
client.get(
url, headers=headers, params=params,
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
),
timeout=60.0,
)
resp.raise_for_status()
return resp.json()
except Exception as e:
@@ -199,7 +209,8 @@ class BzzoiroSource:
# 避免加载联赛全部历史比赛到内存(多赛季采集时内存溢出)
if normalized_matches:
from datetime import timedelta
dates = [nm.date for nm in normalized_matches if nm.date is not None]
# normalized_matches 存的是 (nm, raw) 元组,遍历需解包
dates = [nm.date for nm, _raw in normalized_matches if nm.date is not None]
if dates:
min_dt = min(dates) - timedelta(days=30)
max_dt = max(dates) + timedelta(days=30)
@@ -219,7 +230,7 @@ class BzzoiroSource:
# 球队: 内存查找 + 按需创建
home_team_id = team_name_to_id.get(nm.home_team)
if home_team_id is None:
home = Team(name=nm.home_team)
home = Team(name=nm.home_team, name_zh=zh_name(nm.home_team))
db.add(home)
await db.flush()
home_team_id = home.id
@@ -227,7 +238,7 @@ class BzzoiroSource:
away_team_id = team_name_to_id.get(nm.away_team)
if away_team_id is None:
away = Team(name=nm.away_team)
away = Team(name=nm.away_team, name_zh=zh_name(nm.away_team))
db.add(away)
await db.flush()
away_team_id = away.id
@@ -273,7 +284,7 @@ class BzzoiroSource:
home_red_cards=nm.home_red_cards,
away_red_cards=nm.away_red_cards,
source="bzzoiro",
source_event_id=str(raw.get("id", "")),
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
)
@@ -299,7 +310,7 @@ class BzzoiroSource:
existing_match.stats = MatchStats(
match_id=existing_match.id,
source="bzzoiro",
source_event_id=str(raw.get("id", "")),
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
)