refactor: fix P0/P1/P2 security and performance issues
P0 fixes: - CORS: replace wildcard methods/headers with configurable lists - deps.py: remove unsafe global _warned_unset variable P1 fixes: - http_client: read default timeout from Settings - bzzoiro: replace sync urllib with async httpx - bzzoiro: normalize validation failures use warning level only - db pool: read pool config from Settings (default 5+10) - backtest: add asyncio.Semaphore(8) for concurrent execution - predict/context_builder: add backtest parameter for cutoff buffer P2 improvements: - injuries: enforce int conversion for player_id/fixture_id - injuries: use system temp dir for cache - utils.py: extract shared actual_1x2/is_correct_1x2 - validation: downgrade 1x2 mismatch log to debug - docker-compose: use env vars for all credentials - .env.example: add POSTGRES_USER/PASSWORD/PORT, API_PORT
This commit is contained in:
shangfangjian
+30
-33
@@ -9,16 +9,13 @@ import asyncio
|
||||
import json as _json
|
||||
import logging
|
||||
import random
|
||||
import time as _time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.core.config import settings
|
||||
from src.core.http_client import get_client
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.sources import register
|
||||
@@ -47,43 +44,46 @@ def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, i
|
||||
return (home_team_id, away_team_id, d.isoformat() if d is not None else "")
|
||||
|
||||
|
||||
def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||
"""同步 HTTP(bzzoiro 客户端保持同步,在 async 函数里 run_in_executor)。"""
|
||||
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。"""
|
||||
base = settings.BZZOIRO_BASE.rstrip("/")
|
||||
url = f"{base}/{path.lstrip('/')}"
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
key = settings.BZZOIRO_KEY
|
||||
if not key:
|
||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Token {key}",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", f"Token {key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return _json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
client = get_client()
|
||||
resp = await client.get(url, headers=headers, params=params, timeout=30)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
last_exc = e
|
||||
if e.code == 429:
|
||||
# 指数退避: 429 通常意味着限速
|
||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||
if status == 429:
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
||||
_time.sleep(delay)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
if 500 <= e.code < 600:
|
||||
if 500 <= (status or 0) < 600:
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro %d, retry %d in %.1fs", e.code, attempt + 1, delay)
|
||||
_time.sleep(delay)
|
||||
logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
raise # 4xx 直接抛
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
||||
last_exc = e
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||
_time.sleep(delay)
|
||||
# 网络错误(连接失败/超时)也退避重试
|
||||
if isinstance(e, (TimeoutError, ConnectionError, OSError)):
|
||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||
await asyncio.sleep(delay)
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
||||
|
||||
|
||||
@@ -95,15 +95,13 @@ async def fetch_bzzoiro_events(
|
||||
date_to: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> list[dict]:
|
||||
"""抓取 bzzoiro 原始事件(异步包装)。"""
|
||||
"""抓取 bzzoiro 原始事件(纯异步,无需 run_in_executor)。"""
|
||||
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
||||
if league_id is None:
|
||||
raise ValueError(f"未知联赛代码: {league_code}")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
rows: list[dict] = []
|
||||
offset = 0
|
||||
payload: dict | list = {}
|
||||
while True:
|
||||
params: dict = {
|
||||
"league_id": league_id,
|
||||
@@ -115,8 +113,7 @@ async def fetch_bzzoiro_events(
|
||||
params["date_from"] = str(date_from)[:10]
|
||||
if date_to:
|
||||
params["date_to"] = str(date_to)[:10]
|
||||
# 显式位置参数,避免 lambda 闭包捕获循环变量
|
||||
payload = await loop.run_in_executor(None, _fetch_json_sync, "/events/", params)
|
||||
payload = await _fetch_json_async("/events/", params)
|
||||
batch = payload.get("results") or []
|
||||
if not batch:
|
||||
break
|
||||
@@ -186,8 +183,8 @@ class BzzoiroSource:
|
||||
try:
|
||||
nm.validate()
|
||||
except Exception as e:
|
||||
logger.debug("normalize skip: %s", e)
|
||||
league_r["errors"].append(f"normalize: {e}")
|
||||
# P1-3: 统一使用 warning,不追加到 errors(仅运行时错误入 errors)
|
||||
logger.warning("normalize skip: %s", e)
|
||||
continue
|
||||
normalized_matches.append((nm, raw))
|
||||
all_team_names.add(nm.home_team)
|
||||
|
||||
Reference in New Issue
Block a user