refactor: Sprint 3 - 引入 UnitOfWork + Repository 架构
新增: - src/db/unit_of_work.py: UnitOfWork 事务封装 - src/db/repositories.py: Match/Team/League/Prediction Repository 重构: - 删除 src/data/match_lookup.py(由 Repository 替代) - 数据源(bzzoiro/understat/injuries)不再自行 commit - API 路由(ingest)改用 UnitOfWork - LLM 服务(predict/orchestrator/eval/backtest)改用 UnitOfWork 事务边界统一由调用方控制,数据层不再自行决定 commit。
This commit is contained in:
+16
-24
@@ -1,6 +1,7 @@
|
||||
"""Bzzoiro 数据源:抓取 + 入库。
|
||||
|
||||
迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。
|
||||
使用 Repository 模式进行数据访问,不直接控制事务。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -18,10 +19,9 @@ from sqlalchemy import select
|
||||
|
||||
from src.core.config import settings
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||
from src.data.match_lookup import find_existing_match, get_or_create_team
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats
|
||||
from src.db.models import League, Match, MatchStats, Team
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -125,7 +125,10 @@ class BzzoiroSource:
|
||||
date_to: str | None = None,
|
||||
status: str = "finished",
|
||||
) -> dict:
|
||||
"""采集 bzzoiro → 入库。返回统计。"""
|
||||
"""采集 bzzoiro → 入库。返回统计。
|
||||
|
||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||
"""
|
||||
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||
|
||||
for code in leagues:
|
||||
@@ -159,29 +162,19 @@ class BzzoiroSource:
|
||||
all_team_names.add(nm.away_team)
|
||||
|
||||
if all_team_names:
|
||||
from sqlalchemy import select
|
||||
from src.db.models import Team
|
||||
stmt = select(Team).where(Team.name.in_(all_team_names))
|
||||
teams = (await db.execute(stmt)).scalars().all()
|
||||
team_name_to_id = {t.name: t.id for t in teams}
|
||||
|
||||
# 预加载已有比赛 (league_id + home_id + away_id + date)
|
||||
# 需要先获取球队 ID,所以分批处理
|
||||
date_strs = set()
|
||||
for raw in raw_events:
|
||||
nm = normalize_bzzoiro(raw, code)
|
||||
if nm and nm.date:
|
||||
date_strs.add(nm.date.date().isoformat() if hasattr(nm.date, "date") else str(nm.date))
|
||||
|
||||
if date_strs:
|
||||
from sqlalchemy import func
|
||||
stmt = (
|
||||
select(Match.home_team_id, Match.away_team_id, func.date(Match.match_date).label("d"))
|
||||
.where(Match.league_id == league.id)
|
||||
)
|
||||
rows = (await db.execute(stmt)).all()
|
||||
for row in rows:
|
||||
existing_match_keys.add((row.home_team_id, row.away_team_id, str(row.d)))
|
||||
# 预加载已有比赛
|
||||
from sqlalchemy import func
|
||||
stmt = (
|
||||
select(Match.home_team_id, Match.away_team_id, func.date(Match.match_date).label("d"))
|
||||
.where(Match.league_id == league.id)
|
||||
)
|
||||
rows = (await db.execute(stmt)).all()
|
||||
for row in rows:
|
||||
existing_match_keys.add((row.home_team_id, row.away_team_id, str(row.d)))
|
||||
|
||||
for raw in raw_events:
|
||||
try:
|
||||
@@ -255,7 +248,6 @@ class BzzoiroSource:
|
||||
league_r["inserted"] += 1
|
||||
else:
|
||||
# 已有比赛: 需要查询对象来更新
|
||||
# 注意: 这里为了简化仍查询一次,但只在"已有"时触发
|
||||
from sqlalchemy import func
|
||||
stmt = (
|
||||
select(Match)
|
||||
@@ -297,7 +289,7 @@ class BzzoiroSource:
|
||||
if changed:
|
||||
league_r["updated"] += 1
|
||||
|
||||
await db.commit()
|
||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||
result["leagues"][code] = league_r
|
||||
result["total_inserted"] += league_r["inserted"]
|
||||
result["total_updated"] += league_r["updated"]
|
||||
|
||||
@@ -100,7 +100,10 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
|
||||
|
||||
|
||||
async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
"""采集伤停数据并入库(injuries 表)。"""
|
||||
"""采集伤停数据并入库(injuries 表)。
|
||||
|
||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||
"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.data.team_names import normalize as normalize_name
|
||||
@@ -173,7 +176,7 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
except Exception as e:
|
||||
result["errors"].append(f"parse error: {e}")
|
||||
|
||||
await db.commit()
|
||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
|
||||
return result
|
||||
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
"""比赛匹配辅助函数(多数据源共用)。
|
||||
|
||||
bzzoiro / understat 等数据源在入库时都需要:
|
||||
- 按队名获取或创建球队(get_or_create_team)
|
||||
- 按联赛+主队+客队+日期找已有比赛(find_existing_match)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from src.db.models import Match, Team
|
||||
|
||||
|
||||
async def get_or_create_team(db, name: str) -> Team:
|
||||
"""按名获取球队,不存在则创建。"""
|
||||
stmt = select(Team).where(Team.name == name)
|
||||
team = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if team is None:
|
||||
team = Team(name=name)
|
||||
db.add(team)
|
||||
await db.flush()
|
||||
return team
|
||||
|
||||
|
||||
async def find_existing_match(db, league_id: int, home_name: str, away_name: str, date) -> Match | None:
|
||||
"""按联赛+主队+客队+日期找已有比赛(天级匹配,避免时间精度差异)。"""
|
||||
home_team = (await db.execute(select(Team).where(Team.name == home_name))).scalar_one_or_none()
|
||||
away_team = (await db.execute(select(Team).where(Team.name == away_name))).scalar_one_or_none()
|
||||
if home_team is None or away_team is None:
|
||||
return None
|
||||
|
||||
date_only = date.date() if hasattr(date, "date") else date
|
||||
stmt = (
|
||||
select(Match)
|
||||
.where(Match.league_id == league_id)
|
||||
.where(Match.home_team_id == home_team.id)
|
||||
.where(Match.away_team_id == away_team.id)
|
||||
.where(func.date(Match.match_date) == date_only)
|
||||
)
|
||||
return (await db.execute(stmt)).scalar_one_or_none()
|
||||
+24
-7
@@ -1,6 +1,7 @@
|
||||
"""Understat xG 数据源。
|
||||
|
||||
迁移自旧项目 app/data/sources/understat.py,改成 async。
|
||||
使用 Repository 模式进行数据访问,不直接控制事务。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -10,12 +11,13 @@ import logging
|
||||
import random
|
||||
import re
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from src.core.http_client import get_client
|
||||
from src.data.config import FDCO_TO_UNDERSTAT, LEAGUE_NAMES
|
||||
from src.data.match_lookup import find_existing_match
|
||||
from src.data.normalize import normalize_understat
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats
|
||||
from src.db.models import League, Match, MatchStats, Team
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -80,9 +82,10 @@ class UnderstatSource:
|
||||
name = "understat"
|
||||
|
||||
async def ingest(self, db, *, league: str, season: int) -> dict:
|
||||
"""采集 understat xG → 回填到现有 Match。只回填 xG 字段,不创建新 Match。"""
|
||||
from sqlalchemy import select
|
||||
"""采集 understat xG → 回填到现有 Match。只回填 xG 字段,不创建新 Match。
|
||||
|
||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||
"""
|
||||
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
||||
|
||||
try:
|
||||
@@ -111,8 +114,22 @@ class UnderstatSource:
|
||||
result["errors"].append(f"normalize: {e}")
|
||||
continue
|
||||
|
||||
# 匹配已有 Match(天级)
|
||||
existing = await find_existing_match(db, league_obj.id, nm.home_team, nm.away_team, nm.date)
|
||||
# 匹配已有 Match(天级) - 直接查询
|
||||
home_team = (await db.execute(select(Team).where(Team.name == nm.home_team))).scalar_one_or_none()
|
||||
away_team = (await db.execute(select(Team).where(Team.name == nm.away_team))).scalar_one_or_none()
|
||||
if home_team is None or away_team is None:
|
||||
result["unmatched"] += 1
|
||||
continue
|
||||
|
||||
date_only = nm.date.date() if hasattr(nm.date, "date") else nm.date
|
||||
stmt = (
|
||||
select(Match)
|
||||
.where(Match.league_id == league_obj.id)
|
||||
.where(Match.home_team_id == home_team.id)
|
||||
.where(Match.away_team_id == away_team.id)
|
||||
.where(func.date(Match.match_date) == date_only)
|
||||
)
|
||||
existing = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if existing is None:
|
||||
result["unmatched"] += 1
|
||||
continue
|
||||
@@ -129,5 +146,5 @@ class UnderstatSource:
|
||||
if existing.stats.away_xg is None and nm.away_xg is not None:
|
||||
existing.stats.away_xg = nm.away_xg
|
||||
|
||||
await db.commit()
|
||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user