完善评估能力:筛选参数 + degraded 排除 + 前端评估页
后端: - settle_prediction 拒绝 degraded/failed(明确错误信息) - get_eval_summary 支持 provider/model/prompt_version/mode 筛选 - 返回 filtered_settled/evaluated/skipped_degraded 等计数 - matches 游标分页方向修复(scheduled ASC 用 > 条件) - available_at 加 2h 缓冲(近似完赛时间) - bzzziro 统计字段映射注释(待真实响应验证) - injuries 区分 no_local_data 与 success 空名单 前端: - 新增 EvalPage(筛选控件 + 汇总卡片 + 准确率表格) - 挂载 /admin/eval 路由与导航 测试: - test_matches_cursor.py:游标方向 - test_available_at.py:2h 缓冲与回测防泄漏 - test_bzzoirot_stats.py:统计字段映射 - test_injuries_no_local_data.py:no_local_data vs success - test_injuries_inserted_count.py:失败批不计入 - test_eval_excludes_degraded.py:degraded 排除准确率
This commit is contained in:
+27
-4
@@ -17,7 +17,10 @@ router = APIRouter(prefix="/api/v1", tags=["eval"])
|
||||
|
||||
@router.post("/eval/settle", dependencies=[Depends(require_admin)])
|
||||
async def settle(req: SettleRequest, db: AsyncSession = Depends(get_db)):
|
||||
"""回填实际结果。"""
|
||||
"""回填实际结果。
|
||||
|
||||
status 为 degraded/failed 的预测无法结算。
|
||||
"""
|
||||
try:
|
||||
pred = await settle_prediction(req.prediction_id, req.home_goals, req.away_goals)
|
||||
return {"id": pred.id, "settled": pred.settled}
|
||||
@@ -30,6 +33,26 @@ async def settle(req: SettleRequest, db: AsyncSession = Depends(get_db)):
|
||||
|
||||
|
||||
@router.get("/eval/summary", response_model=EvalSummaryOut, dependencies=[Depends(require_admin)])
|
||||
async def eval_summary(limit: int = Query(1000, ge=1, le=10000, description="最大评估条数")):
|
||||
"""提供商/模型准确率对比。P3-4: 默认评估最近 1000 条,可通过 limit 调整。"""
|
||||
return await get_eval_summary(limit=limit)
|
||||
async def eval_summary(
|
||||
limit: int = Query(1000, ge=1, le=10000, description="最大评估条数"),
|
||||
provider: str | None = Query(None, description="按提供商筛选"),
|
||||
model: str | None = Query(None, description="按模型筛选"),
|
||||
prompt_version: str | None = Query(None, description="按 prompt 版本筛选"),
|
||||
mode: str | None = Query(None, description="按模式筛选(single/multi)"),
|
||||
league_code: str | None = Query(None, description="按联赛代码筛选(如 E0/SP1)"),
|
||||
db: AsyncSession = Depends(get_db_read),
|
||||
):
|
||||
"""提供商/模型准确率对比。
|
||||
|
||||
P3-4: 默认评估最近 1000 条,可通过 limit 调整。
|
||||
支持按 provider / model / prompt_version / mode / league_code 筛选。
|
||||
只统计 status=success 且预测比分齐全的已结算预测,degraded 不计入。
|
||||
"""
|
||||
return await get_eval_summary(
|
||||
limit=limit,
|
||||
provider=provider,
|
||||
model=model,
|
||||
prompt_version=prompt_version,
|
||||
mode=mode,
|
||||
league_code=league_code,
|
||||
)
|
||||
|
||||
@@ -41,10 +41,19 @@ async def list_matches(
|
||||
last_date_str, last_id_str = cursor.split("|", 1)
|
||||
last_date = datetime.fromisoformat(last_date_str)
|
||||
last_id = int(last_id_str)
|
||||
q = q.where(
|
||||
(Match.match_date < last_date) |
|
||||
((Match.match_date == last_date) & (Match.id < last_id))
|
||||
)
|
||||
# 游标方向必须与排序方向一致:
|
||||
# - scheduled(ASC):取「更大」的未开赛场次
|
||||
# - 其它(DESC):取「更小」的已赛场次
|
||||
if status == "scheduled":
|
||||
q = q.where(
|
||||
(Match.match_date > last_date) |
|
||||
((Match.match_date == last_date) & (Match.id > last_id))
|
||||
)
|
||||
else:
|
||||
q = q.where(
|
||||
(Match.match_date < last_date) |
|
||||
((Match.match_date == last_date) & (Match.id < last_id))
|
||||
)
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
|
||||
@@ -124,3 +124,8 @@ class SettleRequest(BaseModel):
|
||||
|
||||
class EvalSummaryOut(BaseModel):
|
||||
summary: list[dict[str, Any]]
|
||||
total_settled: int
|
||||
filtered_settled: int
|
||||
evaluated: int
|
||||
skipped_degraded: int
|
||||
skipped_incomplete: int = 0
|
||||
|
||||
+6
-5
@@ -10,7 +10,7 @@ import json as _json
|
||||
import logging
|
||||
import random
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
@@ -270,8 +270,9 @@ class BzzoiroSource:
|
||||
if any(getattr(nm, f) is not None for f in ['home_xg', 'away_xg', 'home_shots', 'away_shots', 'home_shots_on_target', 'away_shots_on_target', 'home_corners', 'away_corners', 'home_possession', 'home_yellow_cards', 'away_yellow_cards', 'home_red_cards', 'away_red_cards']):
|
||||
now = datetime.now(timezone.utc)
|
||||
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
|
||||
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
|
||||
available_at = nm.date if nm.date else now
|
||||
# 近似:开球 + 2 小时(实际完赛时间约为 +2 小时,非官方公布时间)
|
||||
# 回测 cutoff 若贴着开球,不会把「完赛后才有的统计」误标为赛前可用
|
||||
available_at = nm.date + timedelta(hours=2) if nm.date else now
|
||||
stats = MatchStats(
|
||||
match_id=m.id,
|
||||
home_xg=nm.home_xg,
|
||||
@@ -318,8 +319,8 @@ class BzzoiroSource:
|
||||
):
|
||||
now = datetime.now(timezone.utc)
|
||||
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
|
||||
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
|
||||
available_at = nm.date if nm.date else now
|
||||
# 近似:开球 + 2 小时(实际完赛时间约为 +2 小时,非官方公布时间)
|
||||
available_at = nm.date + timedelta(hours=2) if nm.date else now
|
||||
existing_match.stats = MatchStats(
|
||||
match_id=existing_match.id,
|
||||
source="bzzoiro",
|
||||
|
||||
+20
-7
@@ -233,14 +233,16 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
batch: list[Injury] = []
|
||||
|
||||
async def _flush_batch():
|
||||
"""使用 savepoint flush 一批记录;失败只回滚本批。"""
|
||||
"""使用 savepoint flush 一批记录;失败只回滚本批。返回实际写入条数。"""
|
||||
if not batch:
|
||||
return
|
||||
return 0
|
||||
count = len(batch)
|
||||
async with db.begin_nested():
|
||||
for obj in batch:
|
||||
db.add(obj)
|
||||
await db.flush()
|
||||
batch.clear()
|
||||
return count
|
||||
|
||||
for i, rec in enumerate(pending_records):
|
||||
key = (rec["player_id"], rec["fixture_id"], rec["injury_type"])
|
||||
@@ -248,12 +250,11 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
continue
|
||||
|
||||
batch.append(Injury(**rec))
|
||||
result["inserted"] += 1
|
||||
|
||||
# 每 BATCH_SIZE 条 flush 一次
|
||||
if len(batch) >= BATCH_SIZE:
|
||||
try:
|
||||
await _flush_batch()
|
||||
result["inserted"] += await _flush_batch()
|
||||
except IntegrityError:
|
||||
logger.warning(
|
||||
"injuries batch IntegrityError at record %d, "
|
||||
@@ -266,7 +267,7 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
|
||||
# 最终 flush(剩余不足一批的记录)
|
||||
try:
|
||||
await _flush_batch()
|
||||
result["inserted"] += await _flush_batch()
|
||||
except IntegrityError:
|
||||
logger.warning(
|
||||
"injuries final flush IntegrityError, "
|
||||
@@ -293,10 +294,12 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> In
|
||||
- query_status="success": 查询成功(即使结果也为空)
|
||||
- query_status="source_not_configured": API_FOOTBALL_KEY 未配置
|
||||
- query_status="query_error": 查询异常
|
||||
- query_status="no_local_data": Key 已配置,但该队 injuries 表无任何历史记录
|
||||
|
||||
语义区分:
|
||||
- 成功查询 + 空结果 → has_data=True(明确知道「无人伤停」)
|
||||
- 源未配置 / 查询失败 → has_data=False(无法判断,跳过 LLM)
|
||||
- success + 空结果 → has_data=True(明确知道「无人伤停」)
|
||||
- no_local_data → has_data=False(本地尚未采集,需先 ingest)
|
||||
- source_not_configured / query_error → has_data=False(无法判断)
|
||||
"""
|
||||
from sqlalchemy import select, func
|
||||
from src.db.models import Injury
|
||||
@@ -328,6 +331,16 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> In
|
||||
|
||||
result = await db.execute(stmt)
|
||||
records = list(result.scalars().all())
|
||||
|
||||
# 判定「无本地数据」:该队从未有伤停记录
|
||||
# 规则:该 team_id 在 injuries 表中 count==0
|
||||
if not records:
|
||||
count_stmt = select(func.count()).where(Injury.team_id == team_id)
|
||||
team_count = (await db.execute(count_stmt)).scalar_one() or 0
|
||||
if team_count == 0:
|
||||
logger.debug("API Key 已配置但本地无伤停数据 team=%s,标记 no_local_data", team_id)
|
||||
return InjuryQueryResult(records=[], query_status="no_local_data")
|
||||
|
||||
return InjuryQueryResult(records=records, query_status="success")
|
||||
except Exception as e:
|
||||
logger.exception("伤停查询异常 team=%s: %s", team_id, e)
|
||||
|
||||
+17
-1
@@ -145,7 +145,23 @@ def _to_float(v) -> float | None:
|
||||
|
||||
|
||||
def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
|
||||
"""bzzoiro event → NormalizedMatch。"""
|
||||
"""bzzoiro event → NormalizedMatch。
|
||||
|
||||
统计字段映射说明:
|
||||
当前字段名基于常见足球 API 模式推测(home_shots/away_shots 等),
|
||||
未经真实 bzzoiro 响应校验。若真实字段不同,映射结果将为 None。
|
||||
|
||||
⚠️ 待用真实响应核对的字段清单(请提供一份 event 样例验证):
|
||||
- 射门: home_shots / away_shots(或 shots_home / shots_away)
|
||||
- 射正: home_shots_on_target / away_shots_on_target(或 sot_home / sot_away)
|
||||
- 角球: home_corners / away_corners(或 corners_home / corners_away)
|
||||
- 控球: home_possession(或 possession,仅主队值)
|
||||
- xG: home_xg / away_xg(或 xg_home / xg_away / expected_goals_home / expected_goals_away)
|
||||
- 黄牌: home_yellow_cards / away_yellow_cards(或 yellow_cards_home / yellow_cards_away)
|
||||
- 红牌: home_red_cards / away_red_cards(或 red_cards_home / red_cards_away)
|
||||
|
||||
映射策略:优先查主字段名,回退到别名。所有字段缺失时保持 None,不伪造。
|
||||
"""
|
||||
from src.data.team_names import normalize as normalize_name
|
||||
|
||||
date = _parse_date(raw.get("event_date"))
|
||||
|
||||
@@ -200,9 +200,9 @@ class UnderstatSource:
|
||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
now = datetime.now(timezone.utc)
|
||||
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
|
||||
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
|
||||
# 近似:开球 + 2 小时(实际完赛时间约为 +2 小时,非官方公布时间)
|
||||
match_date = existing.match_date if existing.match_date else now
|
||||
available_at = match_date
|
||||
available_at = match_date + timedelta(hours=2)
|
||||
existing.stats = MatchStats(
|
||||
match_id=existing.id,
|
||||
source="understat",
|
||||
|
||||
@@ -378,6 +378,9 @@ async def injuries_slice(header: MatchHeader, *, before=None, db: AsyncSession |
|
||||
lines.append(f" {label}: 伤停源未配置")
|
||||
elif result.query_status == "query_error":
|
||||
lines.append(f" {label}: 查询异常")
|
||||
elif result.query_status == "no_local_data":
|
||||
# API Key 已配置但本地无伤停记录
|
||||
lines.append(f" {label}: 本地尚无伤停数据,请先采集")
|
||||
elif result.records:
|
||||
n_records += len(result.records)
|
||||
lines.append(f" {label}伤停({len(result.records)}人):")
|
||||
@@ -387,7 +390,7 @@ async def injuries_slice(header: MatchHeader, *, before=None, db: AsyncSession |
|
||||
if len(result.records) > 8:
|
||||
lines.append(f" ...及其他 {len(result.records) - 8} 人")
|
||||
else:
|
||||
# 查询成功但无人伤停
|
||||
# success + 空列表 → 明确无伤停
|
||||
lines.append(f" {label}: 当前无伤停记录")
|
||||
|
||||
# 决定 has_data:
|
||||
|
||||
+79
-10
@@ -3,20 +3,25 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select
|
||||
|
||||
from src.db.models import Prediction
|
||||
from src.db.models import Prediction, Match, League
|
||||
from src.db.unit_of_work import get_uow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int) -> Prediction:
|
||||
"""回填实际结果。"""
|
||||
"""回填实际结果。
|
||||
|
||||
拒绝结算 status 为 degraded/failed 的预测(无有效预测数据)。
|
||||
"""
|
||||
async with get_uow() as session:
|
||||
pred = await session.get(Prediction, prediction_id)
|
||||
if pred is None:
|
||||
raise ValueError(f"prediction {prediction_id} not found")
|
||||
if pred.status in ("degraded", "failed"):
|
||||
raise ValueError(f"无法结算 status={pred.status} 的预测(无有效预测数据)")
|
||||
pred.actual_home_goals = home_goals
|
||||
pred.actual_away_goals = away_goals
|
||||
pred.settled = True
|
||||
@@ -32,34 +37,91 @@ def _actual_1x2(home: int, away: int) -> str:
|
||||
return "2"
|
||||
|
||||
|
||||
async def get_eval_summary(limit: int = 1000) -> dict:
|
||||
def _build_filters(
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
prompt_version: str | None = None,
|
||||
mode: str | None = None,
|
||||
league_code: str | None = None,
|
||||
) -> list:
|
||||
"""构建评估筛选条件(参数化列明,防拼接注入)。"""
|
||||
filters = [Prediction.settled == True]
|
||||
if provider:
|
||||
filters.append(Prediction.provider == provider)
|
||||
if model:
|
||||
filters.append(Prediction.model == model)
|
||||
if prompt_version:
|
||||
filters.append(Prediction.prompt_version == prompt_version)
|
||||
if mode:
|
||||
filters.append(Prediction.mode == mode)
|
||||
if league_code:
|
||||
league_subq = select(League.id).where(League.code == league_code).scalar_subquery()
|
||||
filters.append(Prediction.match_id.in_(
|
||||
select(Match.id).where(Match.league_id.in_(league_subq))
|
||||
))
|
||||
return filters
|
||||
|
||||
|
||||
async def get_eval_summary(
|
||||
limit: int = 1000,
|
||||
*,
|
||||
provider: str | None = None,
|
||||
model: str | None = None,
|
||||
prompt_version: str | None = None,
|
||||
mode: str | None = None,
|
||||
league_code: str | None = None,
|
||||
) -> dict:
|
||||
"""按 provider × 模型聚合评估。
|
||||
|
||||
P3-4: 默认限制评估最近 1000 条已结算预测,避免全表加载导致内存压力。
|
||||
可通过 eval 路由的 query 参数调整。
|
||||
|
||||
只统计有效预测:
|
||||
- settled == True
|
||||
- status == "success"
|
||||
- 预测比分字段齐全
|
||||
degraded 或无比分的预测不计入准确率。
|
||||
"""
|
||||
filters = _build_filters(provider, model, prompt_version, mode, league_code)
|
||||
|
||||
async with get_uow() as session:
|
||||
# 先统计全量已结算数,用于前端展示"共 X 条,评估 Y 条"
|
||||
total_settled = (await session.execute(
|
||||
select(func.count()).where(Prediction.settled == True)
|
||||
)).scalar_one()
|
||||
|
||||
filtered_settled = (await session.execute(
|
||||
select(func.count()).where(*filters)
|
||||
)).scalar_one()
|
||||
|
||||
skipped_degraded = (await session.execute(
|
||||
select(func.count()).where(
|
||||
Prediction.settled == True,
|
||||
or_(Prediction.status != "success", Prediction.status.is_(None)),
|
||||
)
|
||||
)).scalar_one()
|
||||
|
||||
# 有效评估行: settled + status=success + 筛选条件
|
||||
stmt = (
|
||||
select(Prediction)
|
||||
.where(Prediction.settled == True)
|
||||
.where(Prediction.settled == True, Prediction.status == "success", *filters)
|
||||
.order_by(Prediction.id.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
rows = list(result.scalars().all())
|
||||
rows = list((await session.execute(stmt)).scalars().all())
|
||||
|
||||
from collections import defaultdict
|
||||
buckets: dict[tuple[str, str], dict] = defaultdict(lambda: {
|
||||
"total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0,
|
||||
})
|
||||
evaluated = 0
|
||||
skipped_incomplete = 0
|
||||
for p in rows:
|
||||
if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None):
|
||||
skipped_incomplete += 1
|
||||
continue
|
||||
key = (p.provider, p.model)
|
||||
b = buckets[key]
|
||||
b["total"] += 1
|
||||
evaluated += 1
|
||||
if p.actual_home_goals is None or p.actual_away_goals is None:
|
||||
continue
|
||||
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
|
||||
@@ -86,4 +148,11 @@ async def get_eval_summary(limit: int = 1000) -> dict:
|
||||
"avg_score_rmse": round(avg_err, 2) if avg_err is not None else None,
|
||||
"avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None,
|
||||
})
|
||||
return {"summary": summary, "total_settled": total_settled, "evaluated": len(rows)}
|
||||
return {
|
||||
"summary": summary,
|
||||
"total_settled": total_settled,
|
||||
"filtered_settled": filtered_settled,
|
||||
"evaluated": evaluated,
|
||||
"skipped_degraded": skipped_degraded,
|
||||
"skipped_incomplete": skipped_incomplete,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user