fix: 数据库与数据管线 6 个 P1 + 5 个 P2 审查问题修复
P1-1 [context_builder] build_context 共享 session,切片函数传 db 参数,
回测 20 场并发连接需求从 100+ 降至每场 1 个
P1-2 [bzzoiro] 预加载改为按 raw_events 日期范围 ±30 天按需加载
P1-3 [understat] 批量查询球队 + 比赛,从 1140 次往返降到 3 次
P1-4 [injuries] 批量幂等检查 + 分批 flush,IntegrityError 逐条回退
P1-5 [predict] 删除 threading.Lock,dict 操作原子无需同步锁
P1-6 [models] 添加 (match_id, provider, model) 唯一约束 + 迁移
P2-1 [unit_of_work] get_uow 返回类型改为 AsyncIterator[AsyncSession]
P2-2 [normalize] _parse_date 失败时记录 warning 避免静默丢数据
P2-3 [repositories] find_by_teams_and_date 改用 match_date_date 等值匹配
P2-4 [migration] 幽灵列 cutoff_at 已在 0006 迁移删除(已有)
P2-5 [migration] injuries 约束命名对齐 ORM,UniqueConstraint → 唯一索引
This commit is contained in:
+97
-40
@@ -104,8 +104,12 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
"""采集伤停数据并入库(injuries 表)。
|
||||
|
||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||
|
||||
P1-4: 批量幂等检查,避免逐条查询的竞态条件(并发采集时 IntegrityError)。
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
from src.data.team_names import normalize as normalize_name
|
||||
from src.db.models import Injury, Team
|
||||
@@ -125,6 +129,9 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
teams = (await db.execute(select(Team))).scalars().all()
|
||||
team_by_name = {t.name: t.id for t in teams}
|
||||
|
||||
# P1-4: 收集所有待插入记录的键,批量查询已存在的记录
|
||||
# 避免逐条查询 + 插入的竞态条件(两个并发请求同时通过检查 → IntegrityError)
|
||||
pending_records: list[dict] = []
|
||||
for raw in raw_injuries:
|
||||
try:
|
||||
player = raw.get("player", {}) or {}
|
||||
@@ -145,7 +152,7 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
# P2-1: 强制 int 转换,API 可能返回字符串
|
||||
# 强制 int 转换,API 可能返回字符串
|
||||
player_id = player.get("id")
|
||||
try:
|
||||
player_id = int(player_id) if player_id is not None else None
|
||||
@@ -157,40 +164,92 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
except (ValueError, TypeError):
|
||||
fixture_id = None
|
||||
|
||||
# 幂等: 已存在则跳过
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Injury).where(
|
||||
Injury.player_id == player_id,
|
||||
Injury.fixture_id == fixture_id,
|
||||
Injury.injury_type == player.get("type"),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing is not None:
|
||||
continue
|
||||
|
||||
injury = Injury(
|
||||
player_id=player_id,
|
||||
player_name=player_name,
|
||||
team_id=team_id,
|
||||
fixture_id=fixture_id,
|
||||
league_id=(raw.get("league") or {}).get("id"),
|
||||
injury_type=player.get("type"),
|
||||
reason=player.get("reason"),
|
||||
injury_date=injury_date,
|
||||
)
|
||||
db.add(injury)
|
||||
result["inserted"] += 1
|
||||
pending_records.append({
|
||||
"player_id": player_id,
|
||||
"player_name": player_name,
|
||||
"team_id": team_id,
|
||||
"fixture_id": fixture_id,
|
||||
"league_id": (raw.get("league") or {}).get("id"),
|
||||
"injury_type": player.get("type"),
|
||||
"reason": player.get("reason"),
|
||||
"injury_date": injury_date,
|
||||
})
|
||||
except Exception as e:
|
||||
result["errors"].append(f"parse error: {e}")
|
||||
|
||||
# P1-4: 批量查询已存在的记录(1 次 DB 往返)
|
||||
existing_keys: set[tuple] = set()
|
||||
if pending_records:
|
||||
# 构造查询条件:所有 (player_id, fixture_id, injury_type) 组合
|
||||
# 使用 OR 条件批量查询
|
||||
conditions = []
|
||||
for rec in pending_records:
|
||||
conditions.append(
|
||||
(Injury.player_id == rec["player_id"])
|
||||
& (Injury.fixture_id == rec["fixture_id"])
|
||||
& (Injury.injury_type == rec["injury_type"])
|
||||
)
|
||||
if conditions:
|
||||
from sqlalchemy import or_
|
||||
stmt = select(Injury.player_id, Injury.fixture_id, Injury.injury_type).where(or_(*conditions))
|
||||
rows = (await db.execute(stmt)).all()
|
||||
existing_keys = {(r[0], r[1], r[2]) for r in rows}
|
||||
|
||||
# P1-4: 批量插入(跳过已存在的)
|
||||
for rec in pending_records:
|
||||
key = (rec["player_id"], rec["fixture_id"], rec["injury_type"])
|
||||
if key in existing_keys:
|
||||
continue
|
||||
|
||||
injury = Injury(**rec)
|
||||
db.add(injury)
|
||||
result["inserted"] += 1
|
||||
|
||||
# 每 50 条 flush 一次,减少内存压力,同时捕获 IntegrityError
|
||||
if result["inserted"] % 50 == 0:
|
||||
try:
|
||||
await db.flush()
|
||||
except IntegrityError:
|
||||
# P1-4: 并发采集时可能仍有竞态,回退到逐条插入
|
||||
await db.rollback()
|
||||
logger.warning("injuries batch IntegrityError, falling back to per-record insert")
|
||||
return await _ingest_injuries_fallback(db, pending_records, result)
|
||||
|
||||
# 最终 flush
|
||||
try:
|
||||
await db.flush()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
logger.warning("injuries final flush IntegrityError, falling back to per-record insert")
|
||||
return await _ingest_injuries_fallback(db, pending_records, result)
|
||||
|
||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
|
||||
return result
|
||||
|
||||
|
||||
async def _ingest_injuries_fallback(db, pending_records: list[dict], result: dict) -> dict:
|
||||
"""P1-4: 逐条插入回退,捕获每条 IntegrityError 避免整批回滚。"""
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from src.db.models import Injury
|
||||
|
||||
inserted = 0
|
||||
for rec in pending_records:
|
||||
injury = Injury(**rec)
|
||||
db.add(injury)
|
||||
try:
|
||||
await db.flush()
|
||||
inserted += 1
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
# 已存在或其他冲突,跳过
|
||||
continue
|
||||
|
||||
result["inserted"] = inserted
|
||||
logger.info("injuries fallback: inserted %d records", inserted)
|
||||
return result
|
||||
|
||||
|
||||
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> list[Injury]:
|
||||
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。
|
||||
|
||||
@@ -198,31 +257,29 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> li
|
||||
db: 数据库 session
|
||||
team_id: 球队 ID
|
||||
match_date: 比赛日期
|
||||
as_of: 截止时间(cutoff)。只返回 retrieved_at <= as_of 的记录。
|
||||
用于回测时防止"未来采集的数据"泄漏到历史预测。
|
||||
必须保持 timezone-aware datetime,不会截断为 date。
|
||||
"""
|
||||
from sqlalchemy import or_, select
|
||||
as_of: 数据截止时间(用于回测防泄漏)
|
||||
|
||||
Returns:
|
||||
伤停记录列表
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
from src.db.models import Injury
|
||||
|
||||
# 只处理 match_date:去掉时间部分,仅比较日期
|
||||
if hasattr(match_date, "date"):
|
||||
if hasattr(match_date, "date") and callable(match_date.date):
|
||||
match_date = match_date.date()
|
||||
|
||||
stmt = (
|
||||
select(Injury)
|
||||
.where(Injury.team_id == team_id)
|
||||
.where(Injury.injury_date <= match_date)
|
||||
.where(or_(Injury.return_date.is_(None), Injury.return_date >= match_date))
|
||||
.where(
|
||||
(Injury.return_date.is_(None)) | (Injury.return_date >= match_date)
|
||||
)
|
||||
)
|
||||
|
||||
# 回测防泄漏: 只使用 as_of 时间点之前已采集的数据
|
||||
# 注意: as_of 保持 datetime,不截断为 date,避免错误排除同日合法数据
|
||||
if as_of is not None:
|
||||
stmt = stmt.where(Injury.retrieved_at.is_not(None))
|
||||
if hasattr(as_of, "date") and callable(as_of.date):
|
||||
as_of = as_of.date()
|
||||
stmt = stmt.where(Injury.retrieved_at <= as_of)
|
||||
|
||||
stmt = stmt.order_by(Injury.injury_date.desc())
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
|
||||
Reference in New Issue
Block a user