"""评估:赛后回填 + 统计。""" from __future__ import annotations import logging from sqlalchemy import func, or_, select from src.db.models import Prediction, Match, League from src.db.unit_of_work import get_uow logger = logging.getLogger(__name__) async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int) -> Prediction: """回填实际结果。 拒绝结算 status 为 degraded/failed 的预测(无有效预测数据)。 """ async with get_uow() as session: pred = await session.get(Prediction, prediction_id) if pred is None: raise ValueError(f"prediction {prediction_id} not found") if pred.status in ("degraded", "failed"): raise ValueError(f"无法结算 status={pred.status} 的预测(无有效预测数据)") pred.actual_home_goals = home_goals pred.actual_away_goals = away_goals pred.settled = True logger.info( "结算完成 prediction_id=%s match=%s actual=%s:%s mode=%s", prediction_id, pred.match_id, home_goals, away_goals, pred.mode or "single", ) return pred def _actual_1x2(home: int, away: int) -> str: """根据实际比分返胜平负。""" if home > away: return "1" if home == away: return "X" return "2" def _build_filters( provider: str | None = None, model: str | None = None, prompt_version: str | None = None, mode: str | None = None, league_code: str | None = None, ) -> list: """构建评估筛选条件(参数化列明,防拼接注入)。""" filters = [Prediction.settled == True] if provider: filters.append(Prediction.provider == provider) if model: filters.append(Prediction.model == model) if prompt_version: filters.append(Prediction.prompt_version == prompt_version) if mode: filters.append(Prediction.mode == mode) if league_code: league_subq = select(League.id).where(League.code == league_code).scalar_subquery() filters.append(Prediction.match_id.in_( select(Match.id).where(Match.league_id.in_(league_subq)) )) return filters async def get_eval_summary( limit: int = 1000, *, provider: str | None = None, model: str | None = None, prompt_version: str | None = None, mode: str | None = None, league_code: str | None = None, ) -> dict: """按 provider × 模型聚合评估。 P3-4: 默认限制评估最近 1000 条已结算预测,避免全表加载导致内存压力。 只统计有效预测: - settled == True - status == "success" - 预测比分字段齐全 degraded 或无比分的预测不计入准确率。 """ filters = _build_filters(provider, model, prompt_version, mode, league_code) async with get_uow() as session: total_settled = (await session.execute( select(func.count()).where(Prediction.settled == True) )).scalar_one() filtered_settled = (await session.execute( select(func.count()).where(*filters) )).scalar_one() skipped_degraded = (await session.execute( select(func.count()).where( Prediction.settled == True, or_(Prediction.status != "success", Prediction.status.is_(None)), ) )).scalar_one() # 有效评估行: settled + status=success + 筛选条件 stmt = ( select(Prediction) .where(Prediction.settled == True, Prediction.status == "success", *filters) .order_by(Prediction.id.desc()) .limit(limit) ) rows = list((await session.execute(stmt)).scalars().all()) from collections import defaultdict buckets: dict[tuple[str, str, str], dict] = defaultdict(lambda: { "total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0, # 置信度校准分桶(仅 settled 且 pred 完整者计入) "conf_buckets": { "low(0-0.5)": {"total": 0, "correct": 0}, "medium(0.5-0.7)": {"total": 0, "correct": 0}, "high(0.7-1)": {"total": 0, "correct": 0}, }, }) evaluated = 0 skipped_incomplete = 0 for p in rows: if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None): skipped_incomplete += 1 continue key = (p.provider, p.model, p.prompt_version or "") b = buckets[key] b["total"] += 1 evaluated += 1 correct = False if p.actual_home_goals is not None and p.actual_away_goals is not None: actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals) if p.pred_1x2 == actual: b["correct_1x2"] += 1 correct = True if ( p.pred_home_goals is not None and p.pred_away_goals is not None and p.actual_home_goals is not None and p.actual_away_goals is not None ): err = ((p.pred_home_goals - p.actual_home_goals) ** 2 + (p.pred_away_goals - p.actual_away_goals) ** 2) ** 0.5 b["score_errors"].append(err) if p.subjective_confidence is not None: b["conf_sum"] += p.subjective_confidence b["conf_count"] += 1 # 仅当有实际结果可用于校准时,才落入置信度分桶 if p.actual_home_goals is not None and p.actual_away_goals is not None: conf = p.subjective_confidence if conf < 0.5: bucket = "low(0-0.5)" elif conf < 0.7: bucket = "medium(0.5-0.7)" else: bucket = "high(0.7-1)" b["conf_buckets"][bucket]["total"] += 1 if correct: b["conf_buckets"][bucket]["correct"] += 1 summary = [] for (prov, model, ver), b in sorted(buckets.items()): acc = (b["correct_1x2"] / b["total"] * 100) if b["total"] else 0 avg_err = (sum(b["score_errors"]) / len(b["score_errors"])) if b["score_errors"] else None avg_conf = (b["conf_sum"] / b["conf_count"]) if b["conf_count"] else None # 校准分桶 → 命中率 calibration = {} for name, cb in b["conf_buckets"].items(): hit_rate = round(cb["correct"] / cb["total"] * 100, 1) if cb["total"] else None calibration[name] = {"total": cb["total"], "hit_rate": hit_rate} summary.append({ "provider": prov, "model": model, "prompt_version": ver or None, "total": b["total"], "accuracy_1x2": round(acc, 1), "avg_score_rmse": round(avg_err, 2) if avg_err is not None else None, "avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None, "calibration": calibration, }) return { "summary": summary, "total_settled": total_settled, "filtered_settled": filtered_settled, "evaluated": evaluated, "skipped_degraded": skipped_degraded, "skipped_incomplete": skipped_incomplete, }