feat: Sprint 2 - 概率语义 + 轻量快照 + 数据库约束
P0-05: confidence → subjective_confidence 改名(15 文件)
LLM 主观置信度与概率分离
P0-02: predictions 增加 cutoff_at + input_hash(轻量快照)
MatchContext 暴露 match_dt
单/多 Agent 路径均记录快照元数据
P2-05: 数据库 CHECK 约束
- pred_home_goals >= 0
- pred_away_goals >= 0
- subjective_confidence 0~1
- pred_1x2 IN (1,X,2)
- mode IN (single,multi)
P2-06: limit 分页约束(ge=1, le=200)
P2-01: 新增 /health/ready 就绪检查
This commit is contained in:
@@ -46,7 +46,7 @@ async def backtest(req: BacktestRequest):
|
||||
"scored": summary.scored,
|
||||
"accuracy_1x2": summary.accuracy_1x2,
|
||||
"avg_score_rmse": summary.avg_score_rmse,
|
||||
"avg_confidence": summary.avg_confidence,
|
||||
"avg_subjective_confidence": summary.avg_confidence,
|
||||
"calibration": summary.calibration,
|
||||
},
|
||||
"results": [
|
||||
@@ -61,7 +61,7 @@ async def backtest(req: BacktestRequest):
|
||||
"pred_home": r.pred_home,
|
||||
"pred_away": r.pred_away,
|
||||
"pred_1x2": r.pred_1x2,
|
||||
"confidence": r.confidence,
|
||||
"subjective_confidence": r.subjective_confidence,
|
||||
"correct_1x2": r.correct_1x2,
|
||||
}
|
||||
for r in summary.results
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""预测路由。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import selectinload
|
||||
|
||||
@@ -38,7 +38,7 @@ async def predict(req: PredictRequest, db: AsyncSession = Depends(get_db)):
|
||||
pred_home_goals=result.pred_home_goals,
|
||||
pred_away_goals=result.pred_away_goals,
|
||||
pred_1x2=result.pred_1x2,
|
||||
confidence=result.confidence,
|
||||
subjective_confidence=result.subjective_confidence,
|
||||
reasoning=result.reasoning,
|
||||
agent_outputs=getattr(result, "agent_outputs", None),
|
||||
agent_weights=getattr(result, "agent_weights", None),
|
||||
@@ -50,7 +50,7 @@ async def predict(req: PredictRequest, db: AsyncSession = Depends(get_db)):
|
||||
@router.get("/predictions", response_model=list[PredictionOut])
|
||||
async def list_predictions(
|
||||
match_id: int | None = None,
|
||||
limit: int = 50,
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
db: AsyncSession = Depends(get_db_read),
|
||||
):
|
||||
stmt = select(Prediction).options(selectinload(Prediction.match))
|
||||
@@ -69,7 +69,7 @@ async def list_predictions(
|
||||
pred_home_goals=p.pred_home_goals,
|
||||
pred_away_goals=p.pred_away_goals,
|
||||
pred_1x2=p.pred_1x2,
|
||||
confidence=p.confidence,
|
||||
subjective_confidence=p.subjective_confidence,
|
||||
reasoning=p.reasoning,
|
||||
agent_outputs=p.agent_outputs,
|
||||
created_at=p.created_at,
|
||||
@@ -96,7 +96,7 @@ async def get_prediction(prediction_id: int, db: AsyncSession = Depends(get_db_r
|
||||
pred_home_goals=p.pred_home_goals,
|
||||
pred_away_goals=p.pred_away_goals,
|
||||
pred_1x2=p.pred_1x2,
|
||||
confidence=p.confidence,
|
||||
subjective_confidence=p.subjective_confidence,
|
||||
reasoning=p.reasoning,
|
||||
agent_outputs=p.agent_outputs,
|
||||
created_at=p.created_at,
|
||||
|
||||
+2
-2
@@ -54,7 +54,7 @@ class PredictOut(BaseModel):
|
||||
pred_home_goals: float | None
|
||||
pred_away_goals: float | None
|
||||
pred_1x2: str | None
|
||||
confidence: float | None
|
||||
subjective_confidence: float | None
|
||||
reasoning: str | None
|
||||
agent_outputs: list[dict] | None = None
|
||||
agent_weights: dict | None = None
|
||||
@@ -72,7 +72,7 @@ class PredictionOut(BaseModel):
|
||||
pred_home_goals: float | None
|
||||
pred_away_goals: float | None
|
||||
pred_1x2: str | None
|
||||
confidence: float | None
|
||||
subjective_confidence: float | None
|
||||
reasoning: str | None
|
||||
agent_outputs: list[dict] | None = None
|
||||
created_at: datetime
|
||||
|
||||
+11
-1
@@ -5,6 +5,7 @@ from datetime import date, datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Date,
|
||||
DateTime,
|
||||
Float,
|
||||
@@ -156,12 +157,15 @@ class Prediction(Base):
|
||||
pred_home_goals: Mapped[float | None] = mapped_column(Float)
|
||||
pred_away_goals: Mapped[float | None] = mapped_column(Float)
|
||||
pred_1x2: Mapped[str | None] = mapped_column(String(3))
|
||||
confidence: Mapped[float | None] = mapped_column(Float)
|
||||
subjective_confidence: Mapped[float | None] = mapped_column(Float) # LLM 主观置信度,非概率
|
||||
reasoning: Mapped[str | None] = mapped_column(Text)
|
||||
raw_response: Mapped[dict | None] = mapped_column(JSONB)
|
||||
# multi-agent 模式: 各专家报告
|
||||
mode: Mapped[str] = mapped_column(String(20), nullable=False, default="single")
|
||||
agent_outputs: Mapped[dict | None] = mapped_column(JSONB)
|
||||
# 回测/可复现性
|
||||
cutoff_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
input_hash: Mapped[str | None] = mapped_column(String(64))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
||||
actual_home_goals: Mapped[int | None] = mapped_column(Integer)
|
||||
actual_away_goals: Mapped[int | None] = mapped_column(Integer)
|
||||
@@ -172,4 +176,10 @@ class Prediction(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_predictions_match", "match_id"),
|
||||
Index("ix_predictions_provider_model", "provider", "model"),
|
||||
# 数据库级约束:最后一道防线
|
||||
CheckConstraint("pred_home_goals >= 0", name="ck_pred_home_goals_nonneg"),
|
||||
CheckConstraint("pred_away_goals >= 0", name="ck_pred_away_goals_nonneg"),
|
||||
CheckConstraint("subjective_confidence >= 0 AND subjective_confidence <= 1", name="ck_confidence_range"),
|
||||
CheckConstraint("pred_1x2 IN ('1', 'X', '2')", name="ck_pred_1x2_enum"),
|
||||
CheckConstraint("mode IN ('single', 'multi')", name="ck_mode_enum"),
|
||||
)
|
||||
|
||||
@@ -48,7 +48,7 @@ class AgentReport:
|
||||
data_sufficiency: str = "medium" # high | medium | low | none
|
||||
analysis: str = ""
|
||||
home_edge: float | None = None # -1.0 ~ 1.0, 正=利主队
|
||||
confidence: float | None = None # 0.0 ~ 1.0
|
||||
subjective_confidence: float | None = None # 0.0 ~ 1.0
|
||||
key_evidence: list[str] = field(default_factory=list)
|
||||
# xg agent 专属
|
||||
exp_home_goals: float | None = None
|
||||
@@ -67,7 +67,7 @@ class AgentReport:
|
||||
"data_sufficiency": self.data_sufficiency,
|
||||
"analysis": self.analysis,
|
||||
"home_edge": self.home_edge,
|
||||
"confidence": self.confidence,
|
||||
"subjective_confidence": self.subjective_confidence,
|
||||
"key_evidence": self.key_evidence,
|
||||
"exp_home_goals": self.exp_home_goals,
|
||||
"exp_away_goals": self.exp_away_goals,
|
||||
@@ -122,7 +122,7 @@ def _parse_report(agent: str, parsed: dict, resp: LLMResponse, model: str) -> Ag
|
||||
data_sufficiency=validated.data_sufficiency,
|
||||
analysis=validated.analysis,
|
||||
home_edge=validated.home_edge,
|
||||
confidence=validated.confidence,
|
||||
subjective_confidence=validated.subjective_confidence,
|
||||
key_evidence=validated.key_evidence,
|
||||
exp_home_goals=validated.exp_home_goals,
|
||||
exp_away_goals=validated.exp_away_goals,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
@@ -68,7 +69,7 @@ class MultiPredictResult:
|
||||
pred_home_goals: float | None
|
||||
pred_away_goals: float | None
|
||||
pred_1x2: str | None
|
||||
confidence: float | None
|
||||
subjective_confidence: float | None
|
||||
reasoning: str | None
|
||||
agent_outputs: list[dict]
|
||||
agent_weights: dict | None
|
||||
@@ -164,6 +165,7 @@ async def predict_match_multi(
|
||||
|
||||
# 1. 比赛头(各 agent 共享;不存在则 404)
|
||||
header = await load_match_header(match_id)
|
||||
cutoff_at = header.match_dt
|
||||
|
||||
# 2. 并行专家
|
||||
specialist_provider = _get_specialist_provider()
|
||||
@@ -177,6 +179,11 @@ async def predict_match_multi(
|
||||
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
|
||||
# 3.5 计算输入 hash(基于终裁报告)
|
||||
input_hash = hashlib.sha256(
|
||||
_reports_to_json(reports).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
# 4. 存库
|
||||
async with AsyncSessionLocal() as db:
|
||||
m = await db.get(Match, match_id)
|
||||
@@ -203,10 +210,12 @@ async def predict_match_multi(
|
||||
pred_home_goals=validated.pred_home_goals,
|
||||
pred_away_goals=validated.pred_away_goals,
|
||||
pred_1x2=validated.pred_1x2,
|
||||
confidence=validated.confidence,
|
||||
subjective_confidence=validated.subjective_confidence,
|
||||
reasoning=validated.reasoning,
|
||||
raw_response=final,
|
||||
agent_outputs=[r.to_dict() for r in reports],
|
||||
cutoff_at=cutoff_at,
|
||||
input_hash=input_hash,
|
||||
)
|
||||
db.add(pred)
|
||||
await db.commit()
|
||||
@@ -221,7 +230,7 @@ async def predict_match_multi(
|
||||
pred_home_goals=pred.pred_home_goals,
|
||||
pred_away_goals=pred.pred_away_goals,
|
||||
pred_1x2=pred.pred_1x2,
|
||||
confidence=pred.confidence,
|
||||
subjective_confidence=pred.subjective_confidence,
|
||||
reasoning=pred.reasoning,
|
||||
agent_outputs=pred.agent_outputs,
|
||||
agent_weights=agent_weights,
|
||||
|
||||
+4
-4
@@ -139,7 +139,7 @@ async def run_backtest(
|
||||
pred_home=result.pred_home_goals,
|
||||
pred_away=result.pred_away_goals,
|
||||
pred_1x2=result.pred_1x2,
|
||||
confidence=result.confidence,
|
||||
subjective_confidence=result.subjective_confidence,
|
||||
correct_1x2=correct,
|
||||
prediction_id=result.prediction_id,
|
||||
)
|
||||
@@ -164,7 +164,7 @@ async def run_backtest(
|
||||
summary.avg_score_rmse = round(sum(errors) / len(errors), 2)
|
||||
|
||||
# 平均置信度
|
||||
confs = [r.confidence for r in summary.results if r.confidence is not None]
|
||||
confs = [r.subjective_confidence for r in summary.results if r.subjective_confidence is not None]
|
||||
if confs:
|
||||
summary.avg_confidence = round(sum(confs) / len(confs), 2)
|
||||
|
||||
@@ -184,11 +184,11 @@ def _compute_calibration(results: list[BacktestMatchResult]) -> list[dict]:
|
||||
"0.0-0.3": {"range": (0.0, 0.3), "total": 0, "correct": 0},
|
||||
}
|
||||
for r in results:
|
||||
if r.confidence is None:
|
||||
if r.subjective_confidence is None:
|
||||
continue
|
||||
for key, b in buckets.items():
|
||||
lo, hi = b["range"]
|
||||
if lo <= r.confidence <= hi:
|
||||
if lo <= r.subjective_confidence <= hi:
|
||||
b["total"] += 1
|
||||
if r.correct_1x2:
|
||||
b["correct"] += 1
|
||||
|
||||
@@ -36,6 +36,7 @@ class MatchContext:
|
||||
text: str
|
||||
has_stats: bool
|
||||
has_injuries: bool
|
||||
match_dt: object | None = None # 比赛时间(回测防泄漏 + 快照用)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -291,6 +292,7 @@ async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5)
|
||||
text="\n".join(parts),
|
||||
has_stats=has_stats,
|
||||
has_injuries=has_injuries,
|
||||
match_dt=header.match_dt,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -61,8 +61,8 @@ async def get_eval_summary() -> dict:
|
||||
err = ((p.pred_home_goals - p.actual_home_goals) ** 2 +
|
||||
(p.pred_away_goals - p.actual_away_goals) ** 2) ** 0.5
|
||||
b["score_errors"].append(err)
|
||||
if p.confidence is not None:
|
||||
b["conf_sum"] += p.confidence
|
||||
if p.subjective_confidence is not None:
|
||||
b["conf_sum"] += p.subjective_confidence
|
||||
b["conf_count"] += 1
|
||||
|
||||
summary = []
|
||||
|
||||
+11
-3
@@ -2,9 +2,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
|
||||
@@ -64,7 +66,7 @@ class PredictResult:
|
||||
pred_home_goals: float | None
|
||||
pred_away_goals: float | None
|
||||
pred_1x2: str | None
|
||||
confidence: float | None
|
||||
subjective_confidence: float | None
|
||||
reasoning: str | None
|
||||
context: str
|
||||
latency_ms: int | None
|
||||
@@ -112,6 +114,10 @@ async def _predict_single(
|
||||
# 1. 拼上下文
|
||||
ctx = await build_context(match_id)
|
||||
|
||||
# 1.5 计算快照元数据(用于可复现性)
|
||||
cutoff_at = ctx.match_dt # 比赛时间 = 数据截止时间
|
||||
input_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
|
||||
|
||||
# 2. 拼 prompt(指定版本)
|
||||
template = _load_prompt_template(version)
|
||||
user_prompt = template.replace("{{context}}", ctx.text)
|
||||
@@ -155,9 +161,11 @@ async def _predict_single(
|
||||
pred_home_goals=validated.pred_home_goals,
|
||||
pred_away_goals=validated.pred_away_goals,
|
||||
pred_1x2=validated.pred_1x2,
|
||||
confidence=validated.confidence,
|
||||
subjective_confidence=validated.subjective_confidence,
|
||||
reasoning=validated.reasoning,
|
||||
raw_response=resp.raw,
|
||||
cutoff_at=cutoff_at,
|
||||
input_hash=input_hash,
|
||||
)
|
||||
db.add(pred)
|
||||
await db.commit()
|
||||
@@ -171,7 +179,7 @@ async def _predict_single(
|
||||
pred_home_goals=pred.pred_home_goals,
|
||||
pred_away_goals=pred.pred_away_goals,
|
||||
pred_1x2=pred.pred_1x2,
|
||||
confidence=pred.confidence,
|
||||
subjective_confidence=pred.subjective_confidence,
|
||||
reasoning=pred.reasoning,
|
||||
context=ctx.text,
|
||||
latency_ms=resp.latency_ms,
|
||||
|
||||
@@ -15,7 +15,7 @@ class AgentReportSchema(BaseModel):
|
||||
data_sufficiency: str = "medium"
|
||||
analysis: str = ""
|
||||
home_edge: float | None = Field(None, ge=-1.0, le=1.0)
|
||||
confidence: float | None = Field(None, ge=0.0, le=1.0)
|
||||
subjective_confidence: float | None = Field(None, ge=0.0, le=1.0)
|
||||
key_evidence: list[str] = Field(default_factory=list)
|
||||
exp_home_goals: float | None = Field(None, ge=0.0, le=10.0)
|
||||
exp_away_goals: float | None = Field(None, ge=0.0, le=10.0)
|
||||
@@ -50,7 +50,7 @@ class PredictionOutputSchema(BaseModel):
|
||||
pred_home_goals: float = Field(ge=0.0, le=10.0)
|
||||
pred_away_goals: float = Field(ge=0.0, le=10.0)
|
||||
pred_1x2: str
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
subjective_confidence: float = Field(ge=0.0, le=1.0)
|
||||
reasoning: str = ""
|
||||
|
||||
@field_validator("pred_1x2")
|
||||
@@ -87,7 +87,7 @@ def validate_agent_output(raw: dict) -> AgentReportSchema:
|
||||
data_sufficiency=raw.get("data_sufficiency", "medium"),
|
||||
analysis=raw.get("analysis", ""),
|
||||
home_edge=_safe_float(raw.get("home_edge")),
|
||||
confidence=_safe_float(raw.get("confidence")),
|
||||
subjective_confidence=_safe_float(raw.get("subjective_confidence") or raw.get("confidence")),
|
||||
key_evidence=raw.get("key_evidence", []),
|
||||
exp_home_goals=_safe_float(raw.get("exp_home_goals")),
|
||||
exp_away_goals=_safe_float(raw.get("exp_away_goals")),
|
||||
@@ -101,7 +101,7 @@ def validate_prediction_output(raw: dict) -> PredictionOutputSchema:
|
||||
pred_home_goals=float(raw.get("pred_home_goals", 0)),
|
||||
pred_away_goals=float(raw.get("pred_away_goals", 0)),
|
||||
pred_1x2=raw.get("1x2") or raw.get("pred_1x2", "X"),
|
||||
confidence=float(raw.get("confidence", 0.5)),
|
||||
subjective_confidence=float(raw.get("confidence", 0.5)),
|
||||
reasoning=str(raw.get("reasoning", ""))[:1000],
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user