shangfangjian
7d2eabf750
删除未引用/未调用符号: - PredictionRepository(无引用) - is_correct_1x2(无调用者) - LeagueOut(路由用 list[dict]) - IngestResponse(IngestBzzoiroResponse 已替代) - SecurityCheckError(从未 raise,assert_security_on_startup 用 sys.exit) - short_write(仅自引用,全仓库无外部调用) - fetchIngestJobs(列表函数无页面使用,单数 fetchIngestJob 仍保留) - clear_prompt_cache(无入口) 去重: - eval._actual_1x2 改为委托 utils.actual_1x2(单一权威源) 全量测试 270 通过,业务行为不变。
190 lines
7.4 KiB
Python
190 lines
7.4 KiB
Python
"""评估:赛后回填 + 统计。"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
|
||
from sqlalchemy import func, or_, select
|
||
|
||
from src.db.models import Prediction, Match, League
|
||
from src.db.unit_of_work import get_uow
|
||
from src.llm.utils import actual_1x2
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int) -> Prediction:
|
||
"""回填实际结果。
|
||
|
||
拒绝结算 status 为 degraded/failed 的预测(无有效预测数据)。
|
||
"""
|
||
async with get_uow() as session:
|
||
pred = await session.get(Prediction, prediction_id)
|
||
if pred is None:
|
||
raise ValueError(f"prediction {prediction_id} not found")
|
||
if pred.status in ("degraded", "failed"):
|
||
raise ValueError(f"无法结算 status={pred.status} 的预测(无有效预测数据)")
|
||
pred.actual_home_goals = home_goals
|
||
pred.actual_away_goals = away_goals
|
||
pred.settled = True
|
||
logger.info(
|
||
"结算完成 prediction_id=%s match=%s actual=%s:%s mode=%s",
|
||
prediction_id, pred.match_id, home_goals, away_goals, pred.mode or "single",
|
||
)
|
||
return pred
|
||
|
||
|
||
def _actual_1x2(home: int, away: int) -> str:
|
||
"""根据实际比分返胜平负(委托 utils.actual_1x2 单一权威源)。"""
|
||
return actual_1x2(home, away)
|
||
|
||
|
||
def _build_filters(
|
||
provider: str | None = None,
|
||
model: str | None = None,
|
||
prompt_version: str | None = None,
|
||
mode: str | None = None,
|
||
league_code: str | None = None,
|
||
) -> list:
|
||
"""构建评估筛选条件(参数化列明,防拼接注入)。"""
|
||
filters = [Prediction.settled == True]
|
||
if provider:
|
||
filters.append(Prediction.provider == provider)
|
||
if model:
|
||
filters.append(Prediction.model == model)
|
||
if prompt_version:
|
||
filters.append(Prediction.prompt_version == prompt_version)
|
||
if mode:
|
||
filters.append(Prediction.mode == mode)
|
||
if league_code:
|
||
league_subq = select(League.id).where(League.code == league_code).scalar_subquery()
|
||
filters.append(Prediction.match_id.in_(
|
||
select(Match.id).where(Match.league_id.in_(league_subq))
|
||
))
|
||
return filters
|
||
|
||
|
||
async def get_eval_summary(
|
||
limit: int = 1000,
|
||
*,
|
||
provider: str | None = None,
|
||
model: str | None = None,
|
||
prompt_version: str | None = None,
|
||
mode: str | None = None,
|
||
league_code: str | None = None,
|
||
) -> dict:
|
||
"""按 provider × 模型聚合评估。
|
||
|
||
P3-4: 默认限制评估最近 1000 条已结算预测,避免全表加载导致内存压力。
|
||
|
||
只统计有效预测:
|
||
- settled == True
|
||
- status == "success"
|
||
- 预测比分字段齐全
|
||
degraded 或无比分的预测不计入准确率。
|
||
"""
|
||
filters = _build_filters(provider, model, prompt_version, mode, league_code)
|
||
|
||
async with get_uow() as session:
|
||
total_settled = (await session.execute(
|
||
select(func.count()).where(Prediction.settled == True)
|
||
)).scalar_one()
|
||
|
||
filtered_settled = (await session.execute(
|
||
select(func.count()).where(*filters)
|
||
)).scalar_one()
|
||
|
||
skipped_degraded = (await session.execute(
|
||
select(func.count()).where(
|
||
Prediction.settled == True,
|
||
or_(Prediction.status != "success", Prediction.status.is_(None)),
|
||
)
|
||
)).scalar_one()
|
||
|
||
# 有效评估行: settled + status=success + 筛选条件
|
||
stmt = (
|
||
select(Prediction)
|
||
.where(Prediction.settled == True, Prediction.status == "success", *filters)
|
||
.order_by(Prediction.id.desc())
|
||
.limit(limit)
|
||
)
|
||
rows = list((await session.execute(stmt)).scalars().all())
|
||
|
||
from collections import defaultdict
|
||
buckets: dict[tuple[str, str, str], dict] = defaultdict(lambda: {
|
||
"total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0,
|
||
# 置信度校准分桶(仅 settled 且 pred 完整者计入)
|
||
"conf_buckets": {
|
||
"low(0-0.5)": {"total": 0, "correct": 0},
|
||
"medium(0.5-0.7)": {"total": 0, "correct": 0},
|
||
"high(0.7-1)": {"total": 0, "correct": 0},
|
||
},
|
||
})
|
||
evaluated = 0
|
||
skipped_incomplete = 0
|
||
for p in rows:
|
||
if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None):
|
||
skipped_incomplete += 1
|
||
continue
|
||
key = (p.provider, p.model, p.prompt_version or "")
|
||
b = buckets[key]
|
||
b["total"] += 1
|
||
evaluated += 1
|
||
|
||
correct = False
|
||
if p.actual_home_goals is not None and p.actual_away_goals is not None:
|
||
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
|
||
if p.pred_1x2 == actual:
|
||
b["correct_1x2"] += 1
|
||
correct = True
|
||
if (
|
||
p.pred_home_goals is not None and p.pred_away_goals is not None
|
||
and p.actual_home_goals is not None and p.actual_away_goals is not None
|
||
):
|
||
err = ((p.pred_home_goals - p.actual_home_goals) ** 2 +
|
||
(p.pred_away_goals - p.actual_away_goals) ** 2) ** 0.5
|
||
b["score_errors"].append(err)
|
||
if p.subjective_confidence is not None:
|
||
b["conf_sum"] += p.subjective_confidence
|
||
b["conf_count"] += 1
|
||
# 仅当有实际结果可用于校准时,才落入置信度分桶
|
||
if p.actual_home_goals is not None and p.actual_away_goals is not None:
|
||
conf = p.subjective_confidence
|
||
if conf < 0.5:
|
||
bucket = "low(0-0.5)"
|
||
elif conf < 0.7:
|
||
bucket = "medium(0.5-0.7)"
|
||
else:
|
||
bucket = "high(0.7-1)"
|
||
b["conf_buckets"][bucket]["total"] += 1
|
||
if correct:
|
||
b["conf_buckets"][bucket]["correct"] += 1
|
||
|
||
summary = []
|
||
for (prov, model, ver), b in sorted(buckets.items()):
|
||
acc = (b["correct_1x2"] / b["total"] * 100) if b["total"] else 0
|
||
avg_err = (sum(b["score_errors"]) / len(b["score_errors"])) if b["score_errors"] else None
|
||
avg_conf = (b["conf_sum"] / b["conf_count"]) if b["conf_count"] else None
|
||
# 校准分桶 → 命中率
|
||
calibration = {}
|
||
for name, cb in b["conf_buckets"].items():
|
||
hit_rate = round(cb["correct"] / cb["total"] * 100, 1) if cb["total"] else None
|
||
calibration[name] = {"total": cb["total"], "hit_rate": hit_rate}
|
||
summary.append({
|
||
"provider": prov,
|
||
"model": model,
|
||
"prompt_version": ver or None,
|
||
"total": b["total"],
|
||
"accuracy_1x2": round(acc, 1),
|
||
"avg_score_rmse": round(avg_err, 2) if avg_err is not None else None,
|
||
"avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None,
|
||
"calibration": calibration,
|
||
})
|
||
return {
|
||
"summary": summary,
|
||
"total_settled": total_settled,
|
||
"filtered_settled": filtered_settled,
|
||
"evaluated": evaluated,
|
||
"skipped_degraded": skipped_degraded,
|
||
"skipped_incomplete": skipped_incomplete,
|
||
}
|