完善评估能力:筛选参数 + degraded 排除 + 前端评估页

后端:
- settle_prediction 拒绝 degraded/failed(明确错误信息)
- get_eval_summary 支持 provider/model/prompt_version/mode 筛选
- 返回 filtered_settled/evaluated/skipped_degraded 等计数
- matches 游标分页方向修复(scheduled ASC 用 > 条件)
- available_at 加 2h 缓冲(近似完赛时间)
- bzzziro 统计字段映射注释(待真实响应验证)
- injuries 区分 no_local_data 与 success 空名单

前端:
- 新增 EvalPage(筛选控件 + 汇总卡片 + 准确率表格)
- 挂载 /admin/eval 路由与导航

测试:
- test_matches_cursor.py:游标方向
- test_available_at.py:2h 缓冲与回测防泄漏
- test_bzzoirot_stats.py:统计字段映射
- test_injuries_no_local_data.py:no_local_data vs success
- test_injuries_inserted_count.py:失败批不计入
- test_eval_excludes_degraded.py:degraded 排除准确率
This commit is contained in:
Profeto Agent
2026-09-19 09:40:24 +00:00
parent c2c4752856
commit 835d7217d0
21 changed files with 888 additions and 67 deletions
+79 -10
View File
@@ -3,20 +3,25 @@ from __future__ import annotations
import logging
from sqlalchemy import func, select
from sqlalchemy import func, or_, select
from src.db.models import Prediction
from src.db.models import Prediction, Match, League
from src.db.unit_of_work import get_uow
logger = logging.getLogger(__name__)
async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int) -> Prediction:
"""回填实际结果。"""
"""回填实际结果。
拒绝结算 status 为 degraded/failed 的预测(无有效预测数据)。
"""
async with get_uow() as session:
pred = await session.get(Prediction, prediction_id)
if pred is None:
raise ValueError(f"prediction {prediction_id} not found")
if pred.status in ("degraded", "failed"):
raise ValueError(f"无法结算 status={pred.status} 的预测(无有效预测数据)")
pred.actual_home_goals = home_goals
pred.actual_away_goals = away_goals
pred.settled = True
@@ -32,34 +37,91 @@ def _actual_1x2(home: int, away: int) -> str:
return "2"
async def get_eval_summary(limit: int = 1000) -> dict:
def _build_filters(
provider: str | None = None,
model: str | None = None,
prompt_version: str | None = None,
mode: str | None = None,
league_code: str | None = None,
) -> list:
"""构建评估筛选条件(参数化列明,防拼接注入)。"""
filters = [Prediction.settled == True]
if provider:
filters.append(Prediction.provider == provider)
if model:
filters.append(Prediction.model == model)
if prompt_version:
filters.append(Prediction.prompt_version == prompt_version)
if mode:
filters.append(Prediction.mode == mode)
if league_code:
league_subq = select(League.id).where(League.code == league_code).scalar_subquery()
filters.append(Prediction.match_id.in_(
select(Match.id).where(Match.league_id.in_(league_subq))
))
return filters
async def get_eval_summary(
limit: int = 1000,
*,
provider: str | None = None,
model: str | None = None,
prompt_version: str | None = None,
mode: str | None = None,
league_code: str | None = None,
) -> dict:
"""按 provider × 模型聚合评估。
P3-4: 默认限制评估最近 1000 条已结算预测,避免全表加载导致内存压力。
可通过 eval 路由的 query 参数调整。
只统计有效预测:
- settled == True
- status == "success"
- 预测比分字段齐全
degraded 或无比分的预测不计入准确率。
"""
filters = _build_filters(provider, model, prompt_version, mode, league_code)
async with get_uow() as session:
# 先统计全量已结算数,用于前端展示"共 X 条,评估 Y 条"
total_settled = (await session.execute(
select(func.count()).where(Prediction.settled == True)
)).scalar_one()
filtered_settled = (await session.execute(
select(func.count()).where(*filters)
)).scalar_one()
skipped_degraded = (await session.execute(
select(func.count()).where(
Prediction.settled == True,
or_(Prediction.status != "success", Prediction.status.is_(None)),
)
)).scalar_one()
# 有效评估行: settled + status=success + 筛选条件
stmt = (
select(Prediction)
.where(Prediction.settled == True)
.where(Prediction.settled == True, Prediction.status == "success", *filters)
.order_by(Prediction.id.desc())
.limit(limit)
)
result = await session.execute(stmt)
rows = list(result.scalars().all())
rows = list((await session.execute(stmt)).scalars().all())
from collections import defaultdict
buckets: dict[tuple[str, str], dict] = defaultdict(lambda: {
"total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0,
})
evaluated = 0
skipped_incomplete = 0
for p in rows:
if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None):
skipped_incomplete += 1
continue
key = (p.provider, p.model)
b = buckets[key]
b["total"] += 1
evaluated += 1
if p.actual_home_goals is None or p.actual_away_goals is None:
continue
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
@@ -86,4 +148,11 @@ async def get_eval_summary(limit: int = 1000) -> dict:
"avg_score_rmse": round(avg_err, 2) if avg_err is not None else None,
"avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None,
})
return {"summary": summary, "total_settled": total_settled, "evaluated": len(rows)}
return {
"summary": summary,
"total_settled": total_settled,
"filtered_settled": filtered_settled,
"evaluated": evaluated,
"skipped_degraded": skipped_degraded,
"skipped_incomplete": skipped_incomplete,
}