完善评估能力:筛选参数 + degraded 排除 + 前端评估页

后端:
- settle_prediction 拒绝 degraded/failed(明确错误信息)
- get_eval_summary 支持 provider/model/prompt_version/mode 筛选
- 返回 filtered_settled/evaluated/skipped_degraded 等计数
- matches 游标分页方向修复(scheduled ASC 用 > 条件)
- available_at 加 2h 缓冲(近似完赛时间)
- bzzziro 统计字段映射注释(待真实响应验证)
- injuries 区分 no_local_data 与 success 空名单

前端:
- 新增 EvalPage(筛选控件 + 汇总卡片 + 准确率表格)
- 挂载 /admin/eval 路由与导航

测试:
- test_matches_cursor.py:游标方向
- test_available_at.py:2h 缓冲与回测防泄漏
- test_bzzoirot_stats.py:统计字段映射
- test_injuries_no_local_data.py:no_local_data vs success
- test_injuries_inserted_count.py:失败批不计入
- test_eval_excludes_degraded.py:degraded 排除准确率
This commit is contained in:
Profeto Agent
2026-09-19 09:40:24 +00:00
parent c2c4752856
commit 835d7217d0
21 changed files with 888 additions and 67 deletions
+27 -4
View File
@@ -17,7 +17,10 @@ router = APIRouter(prefix="/api/v1", tags=["eval"])
@router.post("/eval/settle", dependencies=[Depends(require_admin)])
async def settle(req: SettleRequest, db: AsyncSession = Depends(get_db)):
"""回填实际结果。"""
"""回填实际结果。
status 为 degraded/failed 的预测无法结算。
"""
try:
pred = await settle_prediction(req.prediction_id, req.home_goals, req.away_goals)
return {"id": pred.id, "settled": pred.settled}
@@ -30,6 +33,26 @@ async def settle(req: SettleRequest, db: AsyncSession = Depends(get_db)):
@router.get("/eval/summary", response_model=EvalSummaryOut, dependencies=[Depends(require_admin)])
async def eval_summary(limit: int = Query(1000, ge=1, le=10000, description="最大评估条数")):
"""提供商/模型准确率对比。P3-4: 默认评估最近 1000 条,可通过 limit 调整。"""
return await get_eval_summary(limit=limit)
async def eval_summary(
limit: int = Query(1000, ge=1, le=10000, description="最大评估条数"),
provider: str | None = Query(None, description="按提供商筛选"),
model: str | None = Query(None, description="按模型筛选"),
prompt_version: str | None = Query(None, description="按 prompt 版本筛选"),
mode: str | None = Query(None, description="按模式筛选(single/multi)"),
league_code: str | None = Query(None, description="按联赛代码筛选(如 E0/SP1)"),
db: AsyncSession = Depends(get_db_read),
):
"""提供商/模型准确率对比。
P3-4: 默认评估最近 1000 条,可通过 limit 调整。
支持按 provider / model / prompt_version / mode / league_code 筛选。
只统计 status=success 且预测比分齐全的已结算预测,degraded 不计入。
"""
return await get_eval_summary(
limit=limit,
provider=provider,
model=model,
prompt_version=prompt_version,
mode=mode,
league_code=league_code,
)
+13 -4
View File
@@ -41,10 +41,19 @@ async def list_matches(
last_date_str, last_id_str = cursor.split("|", 1)
last_date = datetime.fromisoformat(last_date_str)
last_id = int(last_id_str)
q = q.where(
(Match.match_date < last_date) |
((Match.match_date == last_date) & (Match.id < last_id))
)
# 游标方向必须与排序方向一致:
# - scheduled(ASC):取「更大」的未开赛场次
# - 其它(DESC):取「更小」的已赛场次
if status == "scheduled":
q = q.where(
(Match.match_date > last_date) |
((Match.match_date == last_date) & (Match.id > last_id))
)
else:
q = q.where(
(Match.match_date < last_date) |
((Match.match_date == last_date) & (Match.id < last_id))
)
except (ValueError, AttributeError):
pass
+5
View File
@@ -124,3 +124,8 @@ class SettleRequest(BaseModel):
class EvalSummaryOut(BaseModel):
summary: list[dict[str, Any]]
total_settled: int
filtered_settled: int
evaluated: int
skipped_degraded: int
skipped_incomplete: int = 0
+6 -5
View File
@@ -10,7 +10,7 @@ import json as _json
import logging
import random
from collections.abc import Iterable
from datetime import datetime, timezone
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
@@ -270,8 +270,9 @@ class BzzoiroSource:
if any(getattr(nm, f) is not None for f in ['home_xg', 'away_xg', 'home_shots', 'away_shots', 'home_shots_on_target', 'away_shots_on_target', 'home_corners', 'away_corners', 'home_possession', 'home_yellow_cards', 'away_yellow_cards', 'home_red_cards', 'away_red_cards']):
now = datetime.now(timezone.utc)
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
available_at = nm.date if nm.date else now
# 近似:开球 + 2 小时(实际完赛时间约为 +2 小时,非官方公布时间)
# 回测 cutoff 若贴着开球,不会把「完赛后才有的统计」误标为赛前可用
available_at = nm.date + timedelta(hours=2) if nm.date else now
stats = MatchStats(
match_id=m.id,
home_xg=nm.home_xg,
@@ -318,8 +319,8 @@ class BzzoiroSource:
):
now = datetime.now(timezone.utc)
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
available_at = nm.date if nm.date else now
# 近似:开球 + 2 小时(实际完赛时间约为 +2 小时,非官方公布时间)
available_at = nm.date + timedelta(hours=2) if nm.date else now
existing_match.stats = MatchStats(
match_id=existing_match.id,
source="bzzoiro",
+20 -7
View File
@@ -233,14 +233,16 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
batch: list[Injury] = []
async def _flush_batch():
"""使用 savepoint flush 一批记录;失败只回滚本批。"""
"""使用 savepoint flush 一批记录;失败只回滚本批。返回实际写入条数。"""
if not batch:
return
return 0
count = len(batch)
async with db.begin_nested():
for obj in batch:
db.add(obj)
await db.flush()
batch.clear()
return count
for i, rec in enumerate(pending_records):
key = (rec["player_id"], rec["fixture_id"], rec["injury_type"])
@@ -248,12 +250,11 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
continue
batch.append(Injury(**rec))
result["inserted"] += 1
# 每 BATCH_SIZE 条 flush 一次
if len(batch) >= BATCH_SIZE:
try:
await _flush_batch()
result["inserted"] += await _flush_batch()
except IntegrityError:
logger.warning(
"injuries batch IntegrityError at record %d, "
@@ -266,7 +267,7 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
# 最终 flush(剩余不足一批的记录)
try:
await _flush_batch()
result["inserted"] += await _flush_batch()
except IntegrityError:
logger.warning(
"injuries final flush IntegrityError, "
@@ -293,10 +294,12 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> In
- query_status="success": 查询成功(即使结果也为空)
- query_status="source_not_configured": API_FOOTBALL_KEY 未配置
- query_status="query_error": 查询异常
- query_status="no_local_data": Key 已配置,但该队 injuries 表无任何历史记录
语义区分:
- 成功查询 + 空结果 → has_data=True(明确知道「无人伤停」)
- 源未配置 / 查询失败 → has_data=False(无法判断,跳过 LLM)
- success + 空结果 → has_data=True(明确知道「无人伤停」)
- no_local_data → has_data=False(本地尚未采集,需先 ingest)
- source_not_configured / query_error → has_data=False(无法判断)
"""
from sqlalchemy import select, func
from src.db.models import Injury
@@ -328,6 +331,16 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> In
result = await db.execute(stmt)
records = list(result.scalars().all())
# 判定「无本地数据」:该队从未有伤停记录
# 规则:该 team_id 在 injuries 表中 count==0
if not records:
count_stmt = select(func.count()).where(Injury.team_id == team_id)
team_count = (await db.execute(count_stmt)).scalar_one() or 0
if team_count == 0:
logger.debug("API Key 已配置但本地无伤停数据 team=%s,标记 no_local_data", team_id)
return InjuryQueryResult(records=[], query_status="no_local_data")
return InjuryQueryResult(records=records, query_status="success")
except Exception as e:
logger.exception("伤停查询异常 team=%s: %s", team_id, e)
+17 -1
View File
@@ -145,7 +145,23 @@ def _to_float(v) -> float | None:
def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
"""bzzoiro event → NormalizedMatch。"""
"""bzzoiro event → NormalizedMatch。
统计字段映射说明:
当前字段名基于常见足球 API 模式推测(home_shots/away_shots 等),
未经真实 bzzoiro 响应校验。若真实字段不同,映射结果将为 None。
⚠️ 待用真实响应核对的字段清单(请提供一份 event 样例验证):
- 射门: home_shots / away_shots(或 shots_home / shots_away)
- 射正: home_shots_on_target / away_shots_on_target(或 sot_home / sot_away)
- 角球: home_corners / away_corners(或 corners_home / corners_away)
- 控球: home_possession(或 possession,仅主队值)
- xG: home_xg / away_xg(或 xg_home / xg_away / expected_goals_home / expected_goals_away)
- 黄牌: home_yellow_cards / away_yellow_cards(或 yellow_cards_home / yellow_cards_away)
- 红牌: home_red_cards / away_red_cards(或 red_cards_home / red_cards_away)
映射策略:优先查主字段名,回退到别名。所有字段缺失时保持 None,不伪造。
"""
from src.data.team_names import normalize as normalize_name
date = _parse_date(raw.get("event_date"))
+2 -2
View File
@@ -200,9 +200,9 @@ class UnderstatSource:
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
now = datetime.now(timezone.utc)
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
# 近似:开球 + 2 小时(实际完赛时间约为 +2 小时,非官方公布时间)
match_date = existing.match_date if existing.match_date else now
available_at = match_date
available_at = match_date + timedelta(hours=2)
existing.stats = MatchStats(
match_id=existing.id,
source="understat",
+4 -1
View File
@@ -378,6 +378,9 @@ async def injuries_slice(header: MatchHeader, *, before=None, db: AsyncSession |
lines.append(f" {label}: 伤停源未配置")
elif result.query_status == "query_error":
lines.append(f" {label}: 查询异常")
elif result.query_status == "no_local_data":
# API Key 已配置但本地无伤停记录
lines.append(f" {label}: 本地尚无伤停数据,请先采集")
elif result.records:
n_records += len(result.records)
lines.append(f" {label}伤停({len(result.records)}人):")
@@ -387,7 +390,7 @@ async def injuries_slice(header: MatchHeader, *, before=None, db: AsyncSession |
if len(result.records) > 8:
lines.append(f" ...及其他 {len(result.records) - 8}")
else:
# 查询成功但无人伤停
# success + 空列表 → 明确无伤停
lines.append(f" {label}: 当前无伤停记录")
# 决定 has_data:
+79 -10
View File
@@ -3,20 +3,25 @@ from __future__ import annotations
import logging
from sqlalchemy import func, select
from sqlalchemy import func, or_, select
from src.db.models import Prediction
from src.db.models import Prediction, Match, League
from src.db.unit_of_work import get_uow
logger = logging.getLogger(__name__)
async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int) -> Prediction:
"""回填实际结果。"""
"""回填实际结果。
拒绝结算 status 为 degraded/failed 的预测(无有效预测数据)。
"""
async with get_uow() as session:
pred = await session.get(Prediction, prediction_id)
if pred is None:
raise ValueError(f"prediction {prediction_id} not found")
if pred.status in ("degraded", "failed"):
raise ValueError(f"无法结算 status={pred.status} 的预测(无有效预测数据)")
pred.actual_home_goals = home_goals
pred.actual_away_goals = away_goals
pred.settled = True
@@ -32,34 +37,91 @@ def _actual_1x2(home: int, away: int) -> str:
return "2"
async def get_eval_summary(limit: int = 1000) -> dict:
def _build_filters(
provider: str | None = None,
model: str | None = None,
prompt_version: str | None = None,
mode: str | None = None,
league_code: str | None = None,
) -> list:
"""构建评估筛选条件(参数化列明,防拼接注入)。"""
filters = [Prediction.settled == True]
if provider:
filters.append(Prediction.provider == provider)
if model:
filters.append(Prediction.model == model)
if prompt_version:
filters.append(Prediction.prompt_version == prompt_version)
if mode:
filters.append(Prediction.mode == mode)
if league_code:
league_subq = select(League.id).where(League.code == league_code).scalar_subquery()
filters.append(Prediction.match_id.in_(
select(Match.id).where(Match.league_id.in_(league_subq))
))
return filters
async def get_eval_summary(
limit: int = 1000,
*,
provider: str | None = None,
model: str | None = None,
prompt_version: str | None = None,
mode: str | None = None,
league_code: str | None = None,
) -> dict:
"""按 provider × 模型聚合评估。
P3-4: 默认限制评估最近 1000 条已结算预测,避免全表加载导致内存压力。
可通过 eval 路由的 query 参数调整。
只统计有效预测:
- settled == True
- status == "success"
- 预测比分字段齐全
degraded 或无比分的预测不计入准确率。
"""
filters = _build_filters(provider, model, prompt_version, mode, league_code)
async with get_uow() as session:
# 先统计全量已结算数,用于前端展示"共 X 条,评估 Y 条"
total_settled = (await session.execute(
select(func.count()).where(Prediction.settled == True)
)).scalar_one()
filtered_settled = (await session.execute(
select(func.count()).where(*filters)
)).scalar_one()
skipped_degraded = (await session.execute(
select(func.count()).where(
Prediction.settled == True,
or_(Prediction.status != "success", Prediction.status.is_(None)),
)
)).scalar_one()
# 有效评估行: settled + status=success + 筛选条件
stmt = (
select(Prediction)
.where(Prediction.settled == True)
.where(Prediction.settled == True, Prediction.status == "success", *filters)
.order_by(Prediction.id.desc())
.limit(limit)
)
result = await session.execute(stmt)
rows = list(result.scalars().all())
rows = list((await session.execute(stmt)).scalars().all())
from collections import defaultdict
buckets: dict[tuple[str, str], dict] = defaultdict(lambda: {
"total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0,
})
evaluated = 0
skipped_incomplete = 0
for p in rows:
if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None):
skipped_incomplete += 1
continue
key = (p.provider, p.model)
b = buckets[key]
b["total"] += 1
evaluated += 1
if p.actual_home_goals is None or p.actual_away_goals is None:
continue
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
@@ -86,4 +148,11 @@ async def get_eval_summary(limit: int = 1000) -> dict:
"avg_score_rmse": round(avg_err, 2) if avg_err is not None else None,
"avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None,
})
return {"summary": summary, "total_settled": total_settled, "evaluated": len(rows)}
return {
"summary": summary,
"total_settled": total_settled,
"filtered_settled": filtered_settled,
"evaluated": evaluated,
"skipped_degraded": skipped_degraded,
"skipped_incomplete": skipped_incomplete,
}