fix: 修复剩余审查问题
- ingest.py: 异常不再暴露客户端,改为通用错误消息+日志记录 - bzzoiro.py: 修复 normalize 重复调用,改为一次遍历缓存结果 - understat.py: 使用 Repository 替代原始 SQL
This commit is contained in:
@@ -1,6 +1,8 @@
|
|||||||
"""采集路由。"""
|
"""采集路由。"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, HTTPException
|
||||||
|
|
||||||
from src.api.schemas import IngestBzzoiroRequest, IngestResponse, IngestUnderstatRequest, IngestInjuriesRequest, IngestSimpleResponse
|
from src.api.schemas import IngestBzzoiroRequest, IngestResponse, IngestUnderstatRequest, IngestInjuriesRequest, IngestSimpleResponse
|
||||||
@@ -8,6 +10,8 @@ from src.data.sources import get_source
|
|||||||
from src.data.injuries import ingest_injuries
|
from src.data.injuries import ingest_injuries
|
||||||
from src.db.unit_of_work import get_uow
|
from src.db.unit_of_work import get_uow
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1", tags=["ingest"])
|
router = APIRouter(prefix="/api/v1", tags=["ingest"])
|
||||||
|
|
||||||
|
|
||||||
@@ -26,7 +30,8 @@ async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
|||||||
)
|
)
|
||||||
return IngestResponse(**result)
|
return IngestResponse(**result)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(500, str(e))
|
logger.exception("bzzoiro ingest failed")
|
||||||
|
raise HTTPException(500, "数据采集失败,请查看服务器日志")
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ingest/understat", response_model=IngestSimpleResponse)
|
@router.post("/ingest/understat", response_model=IngestSimpleResponse)
|
||||||
@@ -38,7 +43,8 @@ async def ingest_understat_route(req: IngestUnderstatRequest):
|
|||||||
result = await source.ingest(session, league=req.league, season=req.season)
|
result = await source.ingest(session, league=req.league, season=req.season)
|
||||||
return IngestSimpleResponse(**result)
|
return IngestSimpleResponse(**result)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(500, str(e))
|
logger.exception("understat ingest failed")
|
||||||
|
raise HTTPException(500, "xG 回填失败,请查看服务器日志")
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ingest/injuries", response_model=IngestSimpleResponse)
|
@router.post("/ingest/injuries", response_model=IngestSimpleResponse)
|
||||||
@@ -49,4 +55,5 @@ async def ingest_injuries_route(req: IngestInjuriesRequest):
|
|||||||
result = await ingest_injuries(session, date=req.date)
|
result = await ingest_injuries(session, date=req.date)
|
||||||
return IngestSimpleResponse(**result)
|
return IngestSimpleResponse(**result)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(500, str(e))
|
logger.exception("injuries ingest failed")
|
||||||
|
raise HTTPException(500, "伤停采集失败,请查看服务器日志")
|
||||||
|
|||||||
+12
-13
@@ -152,12 +152,21 @@ class BzzoiroSource:
|
|||||||
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
||||||
team_name_to_id: dict[str, int] = {}
|
team_name_to_id: dict[str, int] = {}
|
||||||
existing_match_keys: set[tuple[int, int, str]] = set()
|
existing_match_keys: set[tuple[int, int, str]] = set()
|
||||||
|
normalized_matches: list = [] # 缓存规范化结果,避免重复调用
|
||||||
|
|
||||||
if raw_events:
|
if raw_events:
|
||||||
# 预加载所有涉及的球队名
|
# 一次遍历: 收集球队名 + 规范化
|
||||||
all_team_names = set()
|
all_team_names = set()
|
||||||
for raw in raw_events:
|
for raw in raw_events:
|
||||||
nm = normalize_bzzoiro(raw, code)
|
nm = normalize_bzzoiro(raw, code)
|
||||||
if nm:
|
if nm is not None:
|
||||||
|
try:
|
||||||
|
nm.validate()
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("normalize skip: %s", e)
|
||||||
|
league_r["errors"].append(f"normalize: {e}")
|
||||||
|
continue
|
||||||
|
normalized_matches.append(nm)
|
||||||
all_team_names.add(nm.home_team)
|
all_team_names.add(nm.home_team)
|
||||||
all_team_names.add(nm.away_team)
|
all_team_names.add(nm.away_team)
|
||||||
|
|
||||||
@@ -176,17 +185,7 @@ class BzzoiroSource:
|
|||||||
for row in rows:
|
for row in rows:
|
||||||
existing_match_keys.add((row.home_team_id, row.away_team_id, str(row.d)))
|
existing_match_keys.add((row.home_team_id, row.away_team_id, str(row.d)))
|
||||||
|
|
||||||
for raw in raw_events:
|
for nm in normalized_matches:
|
||||||
try:
|
|
||||||
nm = normalize_bzzoiro(raw, code)
|
|
||||||
if nm is None:
|
|
||||||
continue
|
|
||||||
nm.validate()
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug("normalize skip: %s", e)
|
|
||||||
league_r["errors"].append(f"normalize: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
# 球队: 内存查找 + 按需创建
|
# 球队: 内存查找 + 按需创建
|
||||||
home_team_id = team_name_to_id.get(nm.home_team)
|
home_team_id = team_name_to_id.get(nm.home_team)
|
||||||
if home_team_id is None:
|
if home_team_id is None:
|
||||||
|
|||||||
+13
-13
@@ -86,6 +86,8 @@ class UnderstatSource:
|
|||||||
|
|
||||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||||
"""
|
"""
|
||||||
|
from src.db.repositories import LeagueRepository, MatchRepository, TeamRepository
|
||||||
|
|
||||||
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -95,9 +97,13 @@ class UnderstatSource:
|
|||||||
result["errors"].append(f"fetch failed: {e}")
|
result["errors"].append(f"fetch failed: {e}")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
# 使用 Repository
|
||||||
|
league_repo = LeagueRepository(db)
|
||||||
|
team_repo = TeamRepository(db)
|
||||||
|
match_repo = MatchRepository(db)
|
||||||
|
|
||||||
# 查联赛
|
# 查联赛
|
||||||
stmt = select(League).where(League.code == league)
|
league_obj = await league_repo.get_by_code(league)
|
||||||
league_obj = (await db.execute(stmt)).scalar_one_or_none()
|
|
||||||
if league_obj is None:
|
if league_obj is None:
|
||||||
result["errors"].append(f"league {league} not found in DB")
|
result["errors"].append(f"league {league} not found in DB")
|
||||||
return result
|
return result
|
||||||
@@ -114,22 +120,16 @@ class UnderstatSource:
|
|||||||
result["errors"].append(f"normalize: {e}")
|
result["errors"].append(f"normalize: {e}")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# 匹配已有 Match(天级) - 直接查询
|
# 匹配已有 Match(天级) - 使用 Repository
|
||||||
home_team = (await db.execute(select(Team).where(Team.name == nm.home_team))).scalar_one_or_none()
|
home_team = await team_repo.get_by_name(nm.home_team)
|
||||||
away_team = (await db.execute(select(Team).where(Team.name == nm.away_team))).scalar_one_or_none()
|
away_team = await team_repo.get_by_name(nm.away_team)
|
||||||
if home_team is None or away_team is None:
|
if home_team is None or away_team is None:
|
||||||
result["unmatched"] += 1
|
result["unmatched"] += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
date_only = nm.date.date() if hasattr(nm.date, "date") else nm.date
|
existing = await match_repo.find_by_teams_and_date(
|
||||||
stmt = (
|
league_obj.id, home_team.id, away_team.id, nm.date
|
||||||
select(Match)
|
|
||||||
.where(Match.league_id == league_obj.id)
|
|
||||||
.where(Match.home_team_id == home_team.id)
|
|
||||||
.where(Match.away_team_id == away_team.id)
|
|
||||||
.where(func.date(Match.match_date) == date_only)
|
|
||||||
)
|
)
|
||||||
existing = (await db.execute(stmt)).scalar_one_or_none()
|
|
||||||
if existing is None:
|
if existing is None:
|
||||||
result["unmatched"] += 1
|
result["unmatched"] += 1
|
||||||
continue
|
continue
|
||||||
|
|||||||
Reference in New Issue
Block a user