feat: Sprint 1 - 数据正确性整改
P2-01: 移除 lifespan create_all,改为仅验证连接
新增 /health/ready 就绪检查
P0-04: LLM 输出严格 Pydantic 校验
- Agent 输出越界/非法 → parse_error
- 预测输出自动修正 1X2 与比分一致性
P0-01: injuries cutoff 修复
- get_injuries_for_match 增加 as_of 参数
- injuries_slice 使用 as_of 过滤 retrieved_at
- 防止回测时未来采集数据泄漏
P1-12: 批量入库优化
- 预加载 teams 到内存 dict
- 预加载 existing matches 到内存 set
- 消灭 N+1 查询
This commit is contained in:
shangfangjian
+87
-24
@@ -146,6 +146,43 @@ class BzzoiroSource:
|
||||
db.add(league)
|
||||
await db.flush()
|
||||
|
||||
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
||||
team_name_to_id: dict[str, int] = {}
|
||||
existing_match_keys: set[tuple[int, int, str]] = set()
|
||||
if raw_events:
|
||||
# 预加载所有涉及的球队名
|
||||
all_team_names = set()
|
||||
for raw in raw_events:
|
||||
nm = normalize_bzzoiro(raw, code)
|
||||
if nm:
|
||||
all_team_names.add(nm.home_team)
|
||||
all_team_names.add(nm.away_team)
|
||||
|
||||
if all_team_names:
|
||||
from sqlalchemy import select
|
||||
from src.db.models import Team
|
||||
stmt = select(Team).where(Team.name.in_(all_team_names))
|
||||
teams = (await db.execute(stmt)).scalars().all()
|
||||
team_name_to_id = {t.name: t.id for t in teams}
|
||||
|
||||
# 预加载已有比赛 (league_id + home_id + away_id + date)
|
||||
# 需要先获取球队 ID,所以分批处理
|
||||
date_strs = set()
|
||||
for raw in raw_events:
|
||||
nm = normalize_bzzoiro(raw, code)
|
||||
if nm and nm.date:
|
||||
date_strs.add(nm.date.date().isoformat() if hasattr(nm.date, "date") else str(nm.date))
|
||||
|
||||
if date_strs:
|
||||
from sqlalchemy import func
|
||||
stmt = (
|
||||
select(Match.home_team_id, Match.away_team_id, func.date(Match.match_date).label("d"))
|
||||
.where(Match.league_id == league.id)
|
||||
)
|
||||
rows = (await db.execute(stmt)).all()
|
||||
for row in rows:
|
||||
existing_match_keys.add((row.home_team_id, row.away_team_id, str(row.d)))
|
||||
|
||||
for raw in raw_events:
|
||||
try:
|
||||
nm = normalize_bzzoiro(raw, code)
|
||||
@@ -157,19 +194,34 @@ class BzzoiroSource:
|
||||
league_r["errors"].append(f"normalize: {e}")
|
||||
continue
|
||||
|
||||
# 球队
|
||||
home_team = await get_or_create_team(db, nm.home_team)
|
||||
away_team = await get_or_create_team(db, nm.away_team)
|
||||
# 球队: 内存查找 + 按需创建
|
||||
home_team_id = team_name_to_id.get(nm.home_team)
|
||||
if home_team_id is None:
|
||||
home = Team(name=nm.home_team)
|
||||
db.add(home)
|
||||
await db.flush()
|
||||
home_team_id = home.id
|
||||
team_name_to_id[nm.home_team] = home_team_id
|
||||
|
||||
# 查找已有比赛(天级匹配)
|
||||
existing = await find_existing_match(db, league.id, nm.home_team, nm.away_team, nm.date)
|
||||
away_team_id = team_name_to_id.get(nm.away_team)
|
||||
if away_team_id is None:
|
||||
away = Team(name=nm.away_team)
|
||||
db.add(away)
|
||||
await db.flush()
|
||||
away_team_id = away.id
|
||||
team_name_to_id[nm.away_team] = away_team_id
|
||||
|
||||
# 查找已有比赛: 内存查找
|
||||
date_key = nm.date.date().isoformat() if hasattr(nm.date, "date") else str(nm.date)
|
||||
match_key = (home_team_id, away_team_id, date_key)
|
||||
existing = None if match_key not in existing_match_keys else "exists"
|
||||
|
||||
if existing is None:
|
||||
m = Match(
|
||||
league_id=league.id,
|
||||
season=nm.season_label or None,
|
||||
home_team_id=home_team.id,
|
||||
away_team_id=away_team.id,
|
||||
home_team_id=home_team_id,
|
||||
away_team_id=away_team_id,
|
||||
match_date=nm.date,
|
||||
match_date_date=nm.date.date() if hasattr(nm.date, "date") else nm.date,
|
||||
match_status=nm.match_status,
|
||||
@@ -181,6 +233,7 @@ class BzzoiroSource:
|
||||
)
|
||||
db.add(m)
|
||||
await db.flush()
|
||||
existing_match_keys.add(match_key) # 防止同批重复
|
||||
if nm.home_xg is not None or nm.away_xg is not None:
|
||||
stats = MatchStats(
|
||||
match_id=m.id,
|
||||
@@ -201,35 +254,45 @@ class BzzoiroSource:
|
||||
db.add(stats)
|
||||
league_r["inserted"] += 1
|
||||
else:
|
||||
# 更新(只补空 / 状态升级)
|
||||
# 已有比赛: 需要查询对象来更新
|
||||
# 注意: 这里为了简化仍查询一次,但只在"已有"时触发
|
||||
from sqlalchemy import func
|
||||
stmt = (
|
||||
select(Match)
|
||||
.where(Match.league_id == league.id)
|
||||
.where(Match.home_team_id == home_team_id)
|
||||
.where(Match.away_team_id == away_team_id)
|
||||
.where(func.date(Match.match_date) == date_key)
|
||||
)
|
||||
existing_match = (await db.execute(stmt)).scalar_one()
|
||||
|
||||
changed = False
|
||||
if existing.match_status != nm.match_status and nm.match_status == "finished":
|
||||
existing.match_status = nm.match_status
|
||||
if existing_match.match_status != nm.match_status and nm.match_status == "finished":
|
||||
existing_match.match_status = nm.match_status
|
||||
changed = True
|
||||
if existing.home_goals is None and nm.home_goals is not None:
|
||||
existing.home_goals = nm.home_goals
|
||||
existing.away_goals = nm.away_goals
|
||||
existing.home_ht_goals = nm.home_ht_goals
|
||||
existing.away_ht_goals = nm.away_ht_goals
|
||||
if existing_match.home_goals is None and nm.home_goals is not None:
|
||||
existing_match.home_goals = nm.home_goals
|
||||
existing_match.away_goals = nm.away_goals
|
||||
existing_match.home_ht_goals = nm.home_ht_goals
|
||||
existing_match.away_ht_goals = nm.away_ht_goals
|
||||
changed = True
|
||||
if existing.match_stage is None and nm.match_stage:
|
||||
existing.match_stage = nm.match_stage
|
||||
if existing_match.match_stage is None and nm.match_stage:
|
||||
existing_match.match_stage = nm.match_stage
|
||||
changed = True
|
||||
# stats 只补空
|
||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
existing.stats = MatchStats(match_id=existing.id)
|
||||
db.add(existing.stats)
|
||||
if existing_match.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
existing_match.stats = MatchStats(match_id=existing_match.id)
|
||||
db.add(existing_match.stats)
|
||||
await db.flush()
|
||||
if existing.stats is not None:
|
||||
if existing_match.stats is not None:
|
||||
for fld in ("home_xg", "away_xg", "home_shots", "away_shots",
|
||||
"home_shots_on_target", "away_shots_on_target",
|
||||
"home_corners", "away_corners", "home_possession",
|
||||
"home_yellow_cards", "away_yellow_cards",
|
||||
"home_red_cards", "away_red_cards"):
|
||||
if getattr(existing.stats, fld, None) is None:
|
||||
if getattr(existing_match.stats, fld, None) is None:
|
||||
v = getattr(nm, fld, None)
|
||||
if v is not None:
|
||||
setattr(existing.stats, fld, v)
|
||||
setattr(existing_match.stats, fld, v)
|
||||
changed = True
|
||||
if changed:
|
||||
league_r["updated"] += 1
|
||||
|
||||
+19
-3
@@ -178,8 +178,16 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
return result
|
||||
|
||||
|
||||
async def get_injuries_for_match(db, team_id: int, match_date) -> list[Injury]:
|
||||
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。"""
|
||||
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> list[Injury]:
|
||||
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。
|
||||
|
||||
Args:
|
||||
db: 数据库 session
|
||||
team_id: 球队 ID
|
||||
match_date: 比赛日期
|
||||
as_of: 截止时间(cutoff)。只返回 retrieved_at <= as_of 的记录。
|
||||
用于回测时防止"未来采集的数据"泄漏到历史预测。
|
||||
"""
|
||||
from sqlalchemy import and_, or_, select
|
||||
|
||||
from src.db.models import Injury
|
||||
@@ -192,7 +200,15 @@ async def get_injuries_for_match(db, team_id: int, match_date) -> list[Injury]:
|
||||
.where(Injury.team_id == team_id)
|
||||
.where(Injury.injury_date <= match_date)
|
||||
.where(or_(Injury.return_date.is_(None), Injury.return_date >= match_date))
|
||||
.order_by(Injury.injury_date.desc())
|
||||
)
|
||||
|
||||
# 回测防泄漏: 只使用 as_of 时间点之前已采集的数据
|
||||
if as_of is not None:
|
||||
if hasattr(as_of, "date"):
|
||||
as_of = as_of.date()
|
||||
stmt = stmt.where(Injury.retrieved_at.is_not(None))
|
||||
stmt = stmt.where(Injury.retrieved_at <= as_of)
|
||||
|
||||
stmt = stmt.order_by(Injury.injury_date.desc())
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
Reference in New Issue
Block a user