完善评估能力:筛选参数 + degraded 排除 + 前端评估页
后端: - settle_prediction 拒绝 degraded/failed(明确错误信息) - get_eval_summary 支持 provider/model/prompt_version/mode 筛选 - 返回 filtered_settled/evaluated/skipped_degraded 等计数 - matches 游标分页方向修复(scheduled ASC 用 > 条件) - available_at 加 2h 缓冲(近似完赛时间) - bzzziro 统计字段映射注释(待真实响应验证) - injuries 区分 no_local_data 与 success 空名单 前端: - 新增 EvalPage(筛选控件 + 汇总卡片 + 准确率表格) - 挂载 /admin/eval 路由与导航 测试: - test_matches_cursor.py:游标方向 - test_available_at.py:2h 缓冲与回测防泄漏 - test_bzzoirot_stats.py:统计字段映射 - test_injuries_no_local_data.py:no_local_data vs success - test_injuries_inserted_count.py:失败批不计入 - test_eval_excludes_degraded.py:degraded 排除准确率
This commit is contained in:
+79
-10
@@ -3,20 +3,25 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select
|
||||
|
||||
from src.db.models import Prediction
|
||||
from src.db.models import Prediction, Match, League
|
||||
from src.db.unit_of_work import get_uow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int) -> Prediction:
|
||||
"""回填实际结果。"""
|
||||
"""回填实际结果。
|
||||
|
||||
拒绝结算 status 为 degraded/failed 的预测(无有效预测数据)。
|
||||
"""
|
||||
async with get_uow() as session:
|
||||
pred = await session.get(Prediction, prediction_id)
|
||||
if pred is None:
|
||||
raise ValueError(f"prediction {prediction_id} not found")
|
||||
if pred.status in ("degraded", "failed"):
|
||||
raise ValueError(f"无法结算 status={pred.status} 的预测(无有效预测数据)")
|
||||
pred.actual_home_goals = home_goals
|
||||
pred.actual_away_goals = away_goals
|
||||
pred.settled = True
|
||||
@@ -32,34 +37,91 @@ def _actual_1x2(home: int, away: int) -> str:
|
||||
return "2"
|
||||
|
||||
|
||||
async def get_eval_summary(limit: int = 1000) -> dict:
|
||||
def _build_filters(
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
prompt_version: str | None = None,
|
||||
mode: str | None = None,
|
||||
league_code: str | None = None,
|
||||
) -> list:
|
||||
"""构建评估筛选条件(参数化列明,防拼接注入)。"""
|
||||
filters = [Prediction.settled == True]
|
||||
if provider:
|
||||
filters.append(Prediction.provider == provider)
|
||||
if model:
|
||||
filters.append(Prediction.model == model)
|
||||
if prompt_version:
|
||||
filters.append(Prediction.prompt_version == prompt_version)
|
||||
if mode:
|
||||
filters.append(Prediction.mode == mode)
|
||||
if league_code:
|
||||
league_subq = select(League.id).where(League.code == league_code).scalar_subquery()
|
||||
filters.append(Prediction.match_id.in_(
|
||||
select(Match.id).where(Match.league_id.in_(league_subq))
|
||||
))
|
||||
return filters
|
||||
|
||||
|
||||
async def get_eval_summary(
|
||||
limit: int = 1000,
|
||||
*,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
prompt_version: str | None = None,
|
||||
mode: str | None = None,
|
||||
league_code: str | None = None,
|
||||
) -> dict:
|
||||
"""按 provider × 模型聚合评估。
|
||||
|
||||
P3-4: 默认限制评估最近 1000 条已结算预测,避免全表加载导致内存压力。
|
||||
可通过 eval 路由的 query 参数调整。
|
||||
|
||||
只统计有效预测:
|
||||
- settled == True
|
||||
- status == "success"
|
||||
- 预测比分字段齐全
|
||||
degraded 或无比分的预测不计入准确率。
|
||||
"""
|
||||
filters = _build_filters(provider, model, prompt_version, mode, league_code)
|
||||
|
||||
async with get_uow() as session:
|
||||
# 先统计全量已结算数,用于前端展示"共 X 条,评估 Y 条"
|
||||
total_settled = (await session.execute(
|
||||
select(func.count()).where(Prediction.settled == True)
|
||||
)).scalar_one()
|
||||
|
||||
filtered_settled = (await session.execute(
|
||||
select(func.count()).where(*filters)
|
||||
)).scalar_one()
|
||||
|
||||
skipped_degraded = (await session.execute(
|
||||
select(func.count()).where(
|
||||
Prediction.settled == True,
|
||||
or_(Prediction.status != "success", Prediction.status.is_(None)),
|
||||
)
|
||||
)).scalar_one()
|
||||
|
||||
# 有效评估行: settled + status=success + 筛选条件
|
||||
stmt = (
|
||||
select(Prediction)
|
||||
.where(Prediction.settled == True)
|
||||
.where(Prediction.settled == True, Prediction.status == "success", *filters)
|
||||
.order_by(Prediction.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
rows = list(result.scalars().all())
|
||||
rows = list((await session.execute(stmt)).scalars().all())
|
||||
|
||||
from collections import defaultdict
|
||||
buckets: dict[tuple[str, str], dict] = defaultdict(lambda: {
|
||||
"total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0,
|
||||
})
|
||||
evaluated = 0
|
||||
skipped_incomplete = 0
|
||||
for p in rows:
|
||||
if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None):
|
||||
skipped_incomplete += 1
|
||||
continue
|
||||
key = (p.provider, p.model)
|
||||
b = buckets[key]
|
||||
b["total"] += 1
|
||||
evaluated += 1
|
||||
if p.actual_home_goals is None or p.actual_away_goals is None:
|
||||
continue
|
||||
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
|
||||
@@ -86,4 +148,11 @@ async def get_eval_summary(limit: int = 1000) -> dict:
|
||||
"avg_score_rmse": round(avg_err, 2) if avg_err is not None else None,
|
||||
"avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None,
|
||||
})
|
||||
return {"summary": summary, "total_settled": total_settled, "evaluated": len(rows)}
|
||||
return {
|
||||
"summary": summary,
|
||||
"total_settled": total_settled,
|
||||
"filtered_settled": filtered_settled,
|
||||
"evaluated": evaluated,
|
||||
"skipped_degraded": skipped_degraded,
|
||||
"skipped_incomplete": skipped_incomplete,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user