shangfangjian
11da4d3631
R1 429 key 轮换路径调用不存在的 _km → NameError(且位于凭证脱敏日志行): 改用 src/data/key_ring._mask,全树不再有 _km 调用。 [注: 该改动已随另一工作流提交9ccda4b一并入库] R2 ingest_bzzoiro_standings 被截断(return 前无 upsert 逻辑,standings 表 永不写入、total_upserted 恒为 0):从f05dc1a移植完整实现,按 (league_id, season, team_id) upsert,保留逐联赛错误隔离,不加 db.commit()。 [注: 同上,已随9ccda4b入库] R3 src/llm/agents/orchestrator.py 完成日志把 list 喂给 %d → logging TypeError: 改为 len(ok_reports),与同文件降级日志行写法一致。 R4 mode=multi 静默丢弃调用方传入的 model:predict_match_multi 新增 model 关键字参数,经 _ACTIVE_MODEL_OVERRIDE 下传至 _agent_provider, 显式 model 优先级最高;override 生效时跳过 provider 缓存读写以免串味, 并在 predict.py 派发点透传。 R5 回测把字符串日期直接与 timestamptz 列比较:新增 _parse_date_bound 助手, 支持 YYYY-MM-DD / 完整 ISO / datetime / None,裸日期按 UTC 锚定, 结束日取当天末刻(闭区间,避免最后一天被静默排除),非法输入抛 ValueError。 新增 tests/test_review_required_fixes.py 覆盖 R1-R5(R2/R4 为行为测试), 20 项全通过。
296 lines
10 KiB
Python
296 lines
10 KiB
Python
"""回测框架:在历史数据上运行预测并评估 LLM 预测质量。
|
|
|
|
核心机制:
|
|
- build_context 已内置 before=match_date,天然防未来信息泄漏
|
|
- 对历史比赛跑预测 → 用实际比分 settle → 统计准确率
|
|
- 并发控制: asyncio.Semaphore 限制同时 LLM 调用数
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import selectinload
|
|
|
|
from src.db.models import Match
|
|
from src.db.unit_of_work import get_uow
|
|
from src.llm.eval import settle_prediction
|
|
from src.llm.predict import predict_match
|
|
from src.llm.utils import actual_1x2
|
|
from src.data.team_names_zh import zh_name
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass
|
|
class BacktestMatchResult:
|
|
"""单场回测结果。"""
|
|
match_id: int
|
|
league_code: str | None
|
|
home_team: str
|
|
away_team: str
|
|
home_team_zh: str | None
|
|
away_team_zh: str | None
|
|
match_date: str
|
|
actual_home: int
|
|
actual_away: int
|
|
actual_1x2: str
|
|
pred_home: float | None
|
|
pred_away: float | None
|
|
pred_1x2: str | None
|
|
subjective_confidence: float | None
|
|
correct_1x2: bool
|
|
prediction_id: int
|
|
|
|
|
|
@dataclass
|
|
class BacktestCandidate:
|
|
"""回测候选比赛(字段快照,不持有 ORM 对象)。
|
|
|
|
session 关闭后仍可安全读取:所有需要的关系字段已在查询时物化为普通值,
|
|
避免在 session 之外访问惰性加载的关系属性(会抛 MissingGreenlet)。
|
|
"""
|
|
match_id: int
|
|
league_code: str | None
|
|
home_team: str
|
|
away_team: str
|
|
home_team_zh: str | None
|
|
away_team_zh: str | None
|
|
match_date: datetime
|
|
home_goals: int
|
|
away_goals: int
|
|
|
|
|
|
|
|
@dataclass
|
|
class BacktestSummary:
|
|
"""回测汇总统计。"""
|
|
total: int
|
|
scored: int
|
|
success: int = 0 # status=success 的预测数(有完整比分+1x2)
|
|
degraded: int = 0 # status=degraded 的预测数(专家失败/无有效数据)
|
|
accuracy_1x2: float | None = None
|
|
avg_score_rmse: float | None = None
|
|
avg_subjective_confidence: float | None = None
|
|
calibration: list[dict] = field(default_factory=list)
|
|
results: list[BacktestMatchResult] = field(default_factory=list)
|
|
|
|
|
|
def _parse_date_bound(value, *, end_of_day: bool) -> datetime | None:
|
|
"""把日期入参解析成可与 timestamptz 列比较的 aware datetime。
|
|
|
|
支持 "YYYY-MM-DD"、完整 ISO 串(可带偏移)以及 datetime 对象;None 原样返回。
|
|
裸日期按 UTC 锚定 —— Match.match_date 是 timestamptz,naive datetime 与之
|
|
比较会因时区不同而偏移;start 取当天 00:00,end 取当天 23:59:59.999999
|
|
(闭区间,否则最后一天会被静默排除)。
|
|
|
|
解析失败抛 ValueError(不静默吞掉):fromisoformat 对非法输入统一抛 ValueError,
|
|
这里包一层以带上原始值,便于定位是哪个参数写错了。
|
|
"""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, datetime):
|
|
dt = value
|
|
else:
|
|
try:
|
|
dt = datetime.fromisoformat(str(value))
|
|
except ValueError as e:
|
|
raise ValueError(f"无法解析日期: {value!r}(应为 YYYY-MM-DD 或 ISO 格式)") from e
|
|
|
|
if dt.tzinfo is None:
|
|
dt = dt.replace(tzinfo=timezone.utc)
|
|
# 闭区间上界:日期串解析出来是 00:00,取当天末刻才能让最后一天参与回测
|
|
if end_of_day:
|
|
dt = dt.replace(hour=23, minute=59, second=59, microsecond=999999)
|
|
return dt
|
|
|
|
|
|
async def _get_historical_matches(
|
|
db,
|
|
*,
|
|
league_id: int | None = None,
|
|
date_from: str | None = None,
|
|
date_to: str | None = None,
|
|
limit: int = 50,
|
|
) -> list[BacktestCandidate]:
|
|
"""查询已完赛且有比分的比赛(回测候选)。
|
|
|
|
返回普通值快照而非 ORM 对象:调用方在 session 关闭后仍需使用这些字段,
|
|
而 league / home_team / away_team 是惰性加载关系,在 async 下于 session
|
|
之外访问会抛 MissingGreenlet。这里用 selectinload 预加载后立即物化。
|
|
"""
|
|
stmt = (
|
|
select(Match)
|
|
.options(
|
|
selectinload(Match.league),
|
|
selectinload(Match.home_team),
|
|
selectinload(Match.away_team),
|
|
)
|
|
.where(Match.match_status == "finished")
|
|
.where(Match.home_goals.is_not(None))
|
|
.where(Match.away_goals.is_not(None))
|
|
)
|
|
if league_id is not None:
|
|
stmt = stmt.where(Match.league_id == league_id)
|
|
dt_from = _parse_date_bound(date_from, end_of_day=False)
|
|
if dt_from is not None:
|
|
stmt = stmt.where(Match.match_date >= dt_from)
|
|
dt_to = _parse_date_bound(date_to, end_of_day=True)
|
|
if dt_to is not None:
|
|
stmt = stmt.where(Match.match_date <= dt_to)
|
|
|
|
stmt = stmt.order_by(Match.match_date.desc()).limit(limit)
|
|
result = await db.execute(stmt)
|
|
# 在 session 内物化为纯数据,切断与 ORM 会话的耦合
|
|
return [
|
|
BacktestCandidate(
|
|
match_id=m.id,
|
|
league_code=m.league.code if m.league else None,
|
|
home_team=m.home_team.name if m.home_team else "?",
|
|
away_team=m.away_team.name if m.away_team else "?",
|
|
home_team_zh=zh_name(m.home_team.name) if m.home_team else None,
|
|
away_team_zh=zh_name(m.away_team.name) if m.away_team else None,
|
|
match_date=m.match_date,
|
|
home_goals=m.home_goals,
|
|
away_goals=m.away_goals,
|
|
)
|
|
for m in result.scalars().all()
|
|
]
|
|
|
|
|
|
async def run_backtest(
|
|
*,
|
|
league_id: int | None = None,
|
|
date_from: str | None = None,
|
|
date_to: str | None = None,
|
|
mode: str = "single",
|
|
limit: int = 50,
|
|
model: str | None = None,
|
|
) -> BacktestSummary:
|
|
"""运行回测。
|
|
|
|
Args:
|
|
league_id: 联赛 ID
|
|
date_from: 起始日期 (YYYY-MM-DD)
|
|
date_to: 结束日期 (YYYY-MM-DD)
|
|
mode: 预测模式 (single/multi)
|
|
limit: 最大回测场数
|
|
model: 指定模型 (None=默认)
|
|
|
|
Returns:
|
|
BacktestSummary 含逐场结果 + 汇总统计
|
|
"""
|
|
async with get_uow() as session:
|
|
candidates = await _get_historical_matches(
|
|
session, league_id=league_id, date_from=date_from, date_to=date_to, limit=limit
|
|
)
|
|
|
|
summary = BacktestSummary(total=len(candidates), scored=0)
|
|
|
|
# P1-6: 并发控制,同时最多 8 场预测(避免 LLM API 限流)
|
|
sem = asyncio.Semaphore(8)
|
|
|
|
async def _one(c: BacktestCandidate) -> BacktestMatchResult | None:
|
|
async with sem:
|
|
try:
|
|
result = await predict_match(c.match_id, mode=mode, model=model, use_cache=False, backtest=True)
|
|
await settle_prediction(result.prediction_id, c.home_goals, c.away_goals)
|
|
actual = actual_1x2(c.home_goals, c.away_goals)
|
|
return BacktestMatchResult(
|
|
match_id=c.match_id,
|
|
league_code=c.league_code,
|
|
home_team=c.home_team,
|
|
away_team=c.away_team,
|
|
home_team_zh=c.home_team_zh,
|
|
away_team_zh=c.away_team_zh,
|
|
match_date=c.match_date.strftime("%Y-%m-%d") if c.match_date else "?",
|
|
actual_home=c.home_goals,
|
|
actual_away=c.away_goals,
|
|
actual_1x2=actual,
|
|
pred_home=result.pred_home_goals,
|
|
pred_away=result.pred_away_goals,
|
|
pred_1x2=result.pred_1x2,
|
|
subjective_confidence=result.subjective_confidence,
|
|
correct_1x2=result.pred_1x2 == actual,
|
|
prediction_id=result.prediction_id,
|
|
)
|
|
except Exception:
|
|
logger.exception("backtest match %s failed", c.match_id)
|
|
return None
|
|
|
|
# 并行执行,保持结果顺序
|
|
results = await asyncio.gather(*[_one(c) for c in candidates])
|
|
for r in results:
|
|
if r is not None:
|
|
summary.results.append(r)
|
|
summary.scored += 1
|
|
# success:有完整预测比分+1x2;degraded:多专家模式无有效结论
|
|
if r.pred_1x2 is not None and r.pred_home is not None and r.pred_away is not None:
|
|
summary.success += 1
|
|
else:
|
|
summary.degraded += 1
|
|
|
|
logger.info(
|
|
"回测汇总 mode=%s total=%d scored=%d success=%d accuracy=%s%%",
|
|
mode, summary.total, summary.scored, summary.success,
|
|
f"{(sum(1 for r in summary.results if r.correct_1x2) / summary.scored * 100):.1f}" if summary.scored else "n/a",
|
|
)
|
|
|
|
# 汇总统计
|
|
if summary.scored > 0:
|
|
correct_count = sum(1 for r in summary.results if r.correct_1x2)
|
|
summary.accuracy_1x2 = round(correct_count / summary.scored * 100, 1)
|
|
|
|
# 比分 RMSE
|
|
errors = []
|
|
for r in summary.results:
|
|
if r.pred_home is not None and r.pred_away is not None:
|
|
err = ((r.pred_home - r.actual_home) ** 2 + (r.pred_away - r.actual_away) ** 2) ** 0.5
|
|
errors.append(err)
|
|
if errors:
|
|
summary.avg_score_rmse = round(sum(errors) / len(errors), 2)
|
|
|
|
# 平均主观置信度
|
|
confs = [r.subjective_confidence for r in summary.results if r.subjective_confidence is not None]
|
|
if confs:
|
|
summary.avg_subjective_confidence = round(sum(confs) / len(confs), 2)
|
|
|
|
# 校准:按置信度分桶,看实际准确率是否匹配
|
|
summary.calibration = _compute_calibration(summary.results)
|
|
|
|
return summary
|
|
|
|
|
|
def _compute_calibration(results: list[BacktestMatchResult]) -> list[dict]:
|
|
"""置信度校准:分桶统计实际准确率。"""
|
|
buckets: dict[str, dict] = {
|
|
"0.9-1.0": {"range": (0.9, 1.0), "total": 0, "correct": 0},
|
|
"0.7-0.9": {"range": (0.7, 0.9), "total": 0, "correct": 0},
|
|
"0.5-0.7": {"range": (0.5, 0.7), "total": 0, "correct": 0},
|
|
"0.3-0.5": {"range": (0.3, 0.5), "total": 0, "correct": 0},
|
|
"0.0-0.3": {"range": (0.0, 0.3), "total": 0, "correct": 0},
|
|
}
|
|
for r in results:
|
|
if r.subjective_confidence is None:
|
|
continue
|
|
for key, b in buckets.items():
|
|
lo, hi = b["range"]
|
|
if lo <= r.subjective_confidence <= hi:
|
|
b["total"] += 1
|
|
if r.correct_1x2:
|
|
b["correct"] += 1
|
|
break
|
|
|
|
return [
|
|
{
|
|
"bucket": key,
|
|
"total": b["total"],
|
|
"accuracy": round(b["correct"] / b["total"] * 100, 1) if b["total"] else None,
|
|
}
|
|
for key, b in buckets.items()
|
|
if b["total"] > 0
|
|
]
|