"""回测框架:在历史数据上运行预测并评估 LLM 预测质量。 核心机制: - build_context 已内置 before=match_date,天然防未来信息泄漏 - 对历史比赛跑预测 → 用实际比分 settle → 统计准确率 - 并发控制: asyncio.Semaphore 限制同时 LLM 调用数 """ from __future__ import annotations import asyncio import logging from dataclasses import dataclass, field from datetime import datetime, timezone from sqlalchemy import select from sqlalchemy.orm import selectinload from src.db.models import Match from src.db.unit_of_work import get_uow from src.llm.eval import settle_prediction from src.llm.predict import predict_match from src.llm.utils import actual_1x2 from src.data.team_names_zh import zh_name logger = logging.getLogger(__name__) @dataclass class BacktestMatchResult: """单场回测结果。""" match_id: int league_code: str | None home_team: str away_team: str home_team_zh: str | None away_team_zh: str | None match_date: str actual_home: int actual_away: int actual_1x2: str pred_home: float | None pred_away: float | None pred_1x2: str | None subjective_confidence: float | None correct_1x2: bool prediction_id: int @dataclass class BacktestCandidate: """回测候选比赛(字段快照,不持有 ORM 对象)。 session 关闭后仍可安全读取:所有需要的关系字段已在查询时物化为普通值, 避免在 session 之外访问惰性加载的关系属性(会抛 MissingGreenlet)。 """ match_id: int league_code: str | None home_team: str away_team: str home_team_zh: str | None away_team_zh: str | None match_date: datetime home_goals: int away_goals: int @dataclass class BacktestSummary: """回测汇总统计。""" total: int scored: int success: int = 0 # status=success 的预测数(有完整比分+1x2) degraded: int = 0 # status=degraded 的预测数(专家失败/无有效数据) accuracy_1x2: float | None = None avg_score_rmse: float | None = None avg_subjective_confidence: float | None = None calibration: list[dict] = field(default_factory=list) results: list[BacktestMatchResult] = field(default_factory=list) def _parse_date_bound(value, *, end_of_day: bool) -> datetime | None: """把日期入参解析成可与 timestamptz 列比较的 aware datetime。 支持 "YYYY-MM-DD"、完整 ISO 串(可带偏移)以及 datetime 对象;None 原样返回。 裸日期按 UTC 锚定 —— Match.match_date 是 timestamptz,naive datetime 与之 比较会因时区不同而偏移;start 取当天 00:00,end 取当天 23:59:59.999999 (闭区间,否则最后一天会被静默排除)。 解析失败抛 ValueError(不静默吞掉):fromisoformat 对非法输入统一抛 ValueError, 这里包一层以带上原始值,便于定位是哪个参数写错了。 """ if value is None: return None if isinstance(value, datetime): dt = value else: try: dt = datetime.fromisoformat(str(value)) except ValueError as e: raise ValueError(f"无法解析日期: {value!r}(应为 YYYY-MM-DD 或 ISO 格式)") from e if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) # 闭区间上界:日期串解析出来是 00:00,取当天末刻才能让最后一天参与回测 if end_of_day: dt = dt.replace(hour=23, minute=59, second=59, microsecond=999999) return dt async def _get_historical_matches( db, *, league_id: int | None = None, date_from: str | None = None, date_to: str | None = None, limit: int = 50, ) -> list[BacktestCandidate]: """查询已完赛且有比分的比赛(回测候选)。 返回普通值快照而非 ORM 对象:调用方在 session 关闭后仍需使用这些字段, 而 league / home_team / away_team 是惰性加载关系,在 async 下于 session 之外访问会抛 MissingGreenlet。这里用 selectinload 预加载后立即物化。 """ stmt = ( select(Match) .options( selectinload(Match.league), selectinload(Match.home_team), selectinload(Match.away_team), ) .where(Match.match_status == "finished") .where(Match.home_goals.is_not(None)) .where(Match.away_goals.is_not(None)) ) if league_id is not None: stmt = stmt.where(Match.league_id == league_id) dt_from = _parse_date_bound(date_from, end_of_day=False) if dt_from is not None: stmt = stmt.where(Match.match_date >= dt_from) dt_to = _parse_date_bound(date_to, end_of_day=True) if dt_to is not None: stmt = stmt.where(Match.match_date <= dt_to) stmt = stmt.order_by(Match.match_date.desc()).limit(limit) result = await db.execute(stmt) # 在 session 内物化为纯数据,切断与 ORM 会话的耦合 return [ BacktestCandidate( match_id=m.id, league_code=m.league.code if m.league else None, home_team=m.home_team.name if m.home_team else "?", away_team=m.away_team.name if m.away_team else "?", home_team_zh=zh_name(m.home_team.name) if m.home_team else None, away_team_zh=zh_name(m.away_team.name) if m.away_team else None, match_date=m.match_date, home_goals=m.home_goals, away_goals=m.away_goals, ) for m in result.scalars().all() ] async def run_backtest( *, league_id: int | None = None, date_from: str | None = None, date_to: str | None = None, mode: str = "single", limit: int = 50, model: str | None = None, ) -> BacktestSummary: """运行回测。 Args: league_id: 联赛 ID date_from: 起始日期 (YYYY-MM-DD) date_to: 结束日期 (YYYY-MM-DD) mode: 预测模式 (single/multi) limit: 最大回测场数 model: 指定模型 (None=默认) Returns: BacktestSummary 含逐场结果 + 汇总统计 """ async with get_uow() as session: candidates = await _get_historical_matches( session, league_id=league_id, date_from=date_from, date_to=date_to, limit=limit ) summary = BacktestSummary(total=len(candidates), scored=0) # P1-6: 并发控制,同时最多 8 场预测(避免 LLM API 限流) sem = asyncio.Semaphore(8) async def _one(c: BacktestCandidate) -> BacktestMatchResult | None: async with sem: try: result = await predict_match(c.match_id, mode=mode, model=model, use_cache=False, backtest=True) await settle_prediction(result.prediction_id, c.home_goals, c.away_goals) actual = actual_1x2(c.home_goals, c.away_goals) return BacktestMatchResult( match_id=c.match_id, league_code=c.league_code, home_team=c.home_team, away_team=c.away_team, home_team_zh=c.home_team_zh, away_team_zh=c.away_team_zh, match_date=c.match_date.strftime("%Y-%m-%d") if c.match_date else "?", actual_home=c.home_goals, actual_away=c.away_goals, actual_1x2=actual, pred_home=result.pred_home_goals, pred_away=result.pred_away_goals, pred_1x2=result.pred_1x2, subjective_confidence=result.subjective_confidence, correct_1x2=result.pred_1x2 == actual, prediction_id=result.prediction_id, ) except Exception: logger.exception("backtest match %s failed", c.match_id) return None # 并行执行,保持结果顺序 results = await asyncio.gather(*[_one(c) for c in candidates]) for r in results: if r is not None: summary.results.append(r) summary.scored += 1 # success:有完整预测比分+1x2;degraded:多专家模式无有效结论 if r.pred_1x2 is not None and r.pred_home is not None and r.pred_away is not None: summary.success += 1 else: summary.degraded += 1 logger.info( "回测汇总 mode=%s total=%d scored=%d success=%d accuracy=%s%%", mode, summary.total, summary.scored, summary.success, f"{(sum(1 for r in summary.results if r.correct_1x2) / summary.scored * 100):.1f}" if summary.scored else "n/a", ) # 汇总统计 if summary.scored > 0: correct_count = sum(1 for r in summary.results if r.correct_1x2) summary.accuracy_1x2 = round(correct_count / summary.scored * 100, 1) # 比分 RMSE errors = [] for r in summary.results: if r.pred_home is not None and r.pred_away is not None: err = ((r.pred_home - r.actual_home) ** 2 + (r.pred_away - r.actual_away) ** 2) ** 0.5 errors.append(err) if errors: summary.avg_score_rmse = round(sum(errors) / len(errors), 2) # 平均主观置信度 confs = [r.subjective_confidence for r in summary.results if r.subjective_confidence is not None] if confs: summary.avg_subjective_confidence = round(sum(confs) / len(confs), 2) # 校准:按置信度分桶,看实际准确率是否匹配 summary.calibration = _compute_calibration(summary.results) return summary def _compute_calibration(results: list[BacktestMatchResult]) -> list[dict]: """置信度校准:分桶统计实际准确率。""" buckets: dict[str, dict] = { "0.9-1.0": {"range": (0.9, 1.0), "total": 0, "correct": 0}, "0.7-0.9": {"range": (0.7, 0.9), "total": 0, "correct": 0}, "0.5-0.7": {"range": (0.5, 0.7), "total": 0, "correct": 0}, "0.3-0.5": {"range": (0.3, 0.5), "total": 0, "correct": 0}, "0.0-0.3": {"range": (0.0, 0.3), "total": 0, "correct": 0}, } for r in results: if r.subjective_confidence is None: continue for key, b in buckets.items(): lo, hi = b["range"] if lo <= r.subjective_confidence <= hi: b["total"] += 1 if r.correct_1x2: b["correct"] += 1 break return [ { "bucket": key, "total": b["total"], "accuracy": round(b["correct"] / b["total"] * 100, 1) if b["total"] else None, } for key, b in buckets.items() if b["total"] > 0 ]