feat: Sprint 2 - 概率语义 + 轻量快照 + 数据库约束
P0-05: confidence → subjective_confidence 改名(15 文件)
LLM 主观置信度与概率分离
P0-02: predictions 增加 cutoff_at + input_hash(轻量快照)
MatchContext 暴露 match_dt
单/多 Agent 路径均记录快照元数据
P2-05: 数据库 CHECK 约束
- pred_home_goals >= 0
- pred_away_goals >= 0
- subjective_confidence 0~1
- pred_1x2 IN (1,X,2)
- mode IN (single,multi)
P2-06: limit 分页约束(ge=1, le=200)
P2-01: 新增 /health/ready 就绪检查
This commit is contained in:
@@ -48,7 +48,7 @@ class AgentReport:
|
||||
data_sufficiency: str = "medium" # high | medium | low | none
|
||||
analysis: str = ""
|
||||
home_edge: float | None = None # -1.0 ~ 1.0, 正=利主队
|
||||
confidence: float | None = None # 0.0 ~ 1.0
|
||||
subjective_confidence: float | None = None # 0.0 ~ 1.0
|
||||
key_evidence: list[str] = field(default_factory=list)
|
||||
# xg agent 专属
|
||||
exp_home_goals: float | None = None
|
||||
@@ -67,7 +67,7 @@ class AgentReport:
|
||||
"data_sufficiency": self.data_sufficiency,
|
||||
"analysis": self.analysis,
|
||||
"home_edge": self.home_edge,
|
||||
"confidence": self.confidence,
|
||||
"subjective_confidence": self.subjective_confidence,
|
||||
"key_evidence": self.key_evidence,
|
||||
"exp_home_goals": self.exp_home_goals,
|
||||
"exp_away_goals": self.exp_away_goals,
|
||||
@@ -122,7 +122,7 @@ def _parse_report(agent: str, parsed: dict, resp: LLMResponse, model: str) -> Ag
|
||||
data_sufficiency=validated.data_sufficiency,
|
||||
analysis=validated.analysis,
|
||||
home_edge=validated.home_edge,
|
||||
confidence=validated.confidence,
|
||||
subjective_confidence=validated.subjective_confidence,
|
||||
key_evidence=validated.key_evidence,
|
||||
exp_home_goals=validated.exp_home_goals,
|
||||
exp_away_goals=validated.exp_away_goals,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
@@ -68,7 +69,7 @@ class MultiPredictResult:
|
||||
pred_home_goals: float | None
|
||||
pred_away_goals: float | None
|
||||
pred_1x2: str | None
|
||||
confidence: float | None
|
||||
subjective_confidence: float | None
|
||||
reasoning: str | None
|
||||
agent_outputs: list[dict]
|
||||
agent_weights: dict | None
|
||||
@@ -164,6 +165,7 @@ async def predict_match_multi(
|
||||
|
||||
# 1. 比赛头(各 agent 共享;不存在则 404)
|
||||
header = await load_match_header(match_id)
|
||||
cutoff_at = header.match_dt
|
||||
|
||||
# 2. 并行专家
|
||||
specialist_provider = _get_specialist_provider()
|
||||
@@ -177,6 +179,11 @@ async def predict_match_multi(
|
||||
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
|
||||
# 3.5 计算输入 hash(基于终裁报告)
|
||||
input_hash = hashlib.sha256(
|
||||
_reports_to_json(reports).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
# 4. 存库
|
||||
async with AsyncSessionLocal() as db:
|
||||
m = await db.get(Match, match_id)
|
||||
@@ -203,10 +210,12 @@ async def predict_match_multi(
|
||||
pred_home_goals=validated.pred_home_goals,
|
||||
pred_away_goals=validated.pred_away_goals,
|
||||
pred_1x2=validated.pred_1x2,
|
||||
confidence=validated.confidence,
|
||||
subjective_confidence=validated.subjective_confidence,
|
||||
reasoning=validated.reasoning,
|
||||
raw_response=final,
|
||||
agent_outputs=[r.to_dict() for r in reports],
|
||||
cutoff_at=cutoff_at,
|
||||
input_hash=input_hash,
|
||||
)
|
||||
db.add(pred)
|
||||
await db.commit()
|
||||
@@ -221,7 +230,7 @@ async def predict_match_multi(
|
||||
pred_home_goals=pred.pred_home_goals,
|
||||
pred_away_goals=pred.pred_away_goals,
|
||||
pred_1x2=pred.pred_1x2,
|
||||
confidence=pred.confidence,
|
||||
subjective_confidence=pred.subjective_confidence,
|
||||
reasoning=pred.reasoning,
|
||||
agent_outputs=pred.agent_outputs,
|
||||
agent_weights=agent_weights,
|
||||
|
||||
+4
-4
@@ -139,7 +139,7 @@ async def run_backtest(
|
||||
pred_home=result.pred_home_goals,
|
||||
pred_away=result.pred_away_goals,
|
||||
pred_1x2=result.pred_1x2,
|
||||
confidence=result.confidence,
|
||||
subjective_confidence=result.subjective_confidence,
|
||||
correct_1x2=correct,
|
||||
prediction_id=result.prediction_id,
|
||||
)
|
||||
@@ -164,7 +164,7 @@ async def run_backtest(
|
||||
summary.avg_score_rmse = round(sum(errors) / len(errors), 2)
|
||||
|
||||
# 平均置信度
|
||||
confs = [r.confidence for r in summary.results if r.confidence is not None]
|
||||
confs = [r.subjective_confidence for r in summary.results if r.subjective_confidence is not None]
|
||||
if confs:
|
||||
summary.avg_confidence = round(sum(confs) / len(confs), 2)
|
||||
|
||||
@@ -184,11 +184,11 @@ def _compute_calibration(results: list[BacktestMatchResult]) -> list[dict]:
|
||||
"0.0-0.3": {"range": (0.0, 0.3), "total": 0, "correct": 0},
|
||||
}
|
||||
for r in results:
|
||||
if r.confidence is None:
|
||||
if r.subjective_confidence is None:
|
||||
continue
|
||||
for key, b in buckets.items():
|
||||
lo, hi = b["range"]
|
||||
if lo <= r.confidence <= hi:
|
||||
if lo <= r.subjective_confidence <= hi:
|
||||
b["total"] += 1
|
||||
if r.correct_1x2:
|
||||
b["correct"] += 1
|
||||
|
||||
@@ -36,6 +36,7 @@ class MatchContext:
|
||||
text: str
|
||||
has_stats: bool
|
||||
has_injuries: bool
|
||||
match_dt: object | None = None # 比赛时间(回测防泄漏 + 快照用)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -291,6 +292,7 @@ async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5)
|
||||
text="\n".join(parts),
|
||||
has_stats=has_stats,
|
||||
has_injuries=has_injuries,
|
||||
match_dt=header.match_dt,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -61,8 +61,8 @@ async def get_eval_summary() -> dict:
|
||||
err = ((p.pred_home_goals - p.actual_home_goals) ** 2 +
|
||||
(p.pred_away_goals - p.actual_away_goals) ** 2) ** 0.5
|
||||
b["score_errors"].append(err)
|
||||
if p.confidence is not None:
|
||||
b["conf_sum"] += p.confidence
|
||||
if p.subjective_confidence is not None:
|
||||
b["conf_sum"] += p.subjective_confidence
|
||||
b["conf_count"] += 1
|
||||
|
||||
summary = []
|
||||
|
||||
+11
-3
@@ -2,9 +2,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
|
||||
@@ -64,7 +66,7 @@ class PredictResult:
|
||||
pred_home_goals: float | None
|
||||
pred_away_goals: float | None
|
||||
pred_1x2: str | None
|
||||
confidence: float | None
|
||||
subjective_confidence: float | None
|
||||
reasoning: str | None
|
||||
context: str
|
||||
latency_ms: int | None
|
||||
@@ -112,6 +114,10 @@ async def _predict_single(
|
||||
# 1. 拼上下文
|
||||
ctx = await build_context(match_id)
|
||||
|
||||
# 1.5 计算快照元数据(用于可复现性)
|
||||
cutoff_at = ctx.match_dt # 比赛时间 = 数据截止时间
|
||||
input_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
|
||||
|
||||
# 2. 拼 prompt(指定版本)
|
||||
template = _load_prompt_template(version)
|
||||
user_prompt = template.replace("{{context}}", ctx.text)
|
||||
@@ -155,9 +161,11 @@ async def _predict_single(
|
||||
pred_home_goals=validated.pred_home_goals,
|
||||
pred_away_goals=validated.pred_away_goals,
|
||||
pred_1x2=validated.pred_1x2,
|
||||
confidence=validated.confidence,
|
||||
subjective_confidence=validated.subjective_confidence,
|
||||
reasoning=validated.reasoning,
|
||||
raw_response=resp.raw,
|
||||
cutoff_at=cutoff_at,
|
||||
input_hash=input_hash,
|
||||
)
|
||||
db.add(pred)
|
||||
await db.commit()
|
||||
@@ -171,7 +179,7 @@ async def _predict_single(
|
||||
pred_home_goals=pred.pred_home_goals,
|
||||
pred_away_goals=pred.pred_away_goals,
|
||||
pred_1x2=pred.pred_1x2,
|
||||
confidence=pred.confidence,
|
||||
subjective_confidence=pred.subjective_confidence,
|
||||
reasoning=pred.reasoning,
|
||||
context=ctx.text,
|
||||
latency_ms=resp.latency_ms,
|
||||
|
||||
@@ -15,7 +15,7 @@ class AgentReportSchema(BaseModel):
|
||||
data_sufficiency: str = "medium"
|
||||
analysis: str = ""
|
||||
home_edge: float | None = Field(None, ge=-1.0, le=1.0)
|
||||
confidence: float | None = Field(None, ge=0.0, le=1.0)
|
||||
subjective_confidence: float | None = Field(None, ge=0.0, le=1.0)
|
||||
key_evidence: list[str] = Field(default_factory=list)
|
||||
exp_home_goals: float | None = Field(None, ge=0.0, le=10.0)
|
||||
exp_away_goals: float | None = Field(None, ge=0.0, le=10.0)
|
||||
@@ -50,7 +50,7 @@ class PredictionOutputSchema(BaseModel):
|
||||
pred_home_goals: float = Field(ge=0.0, le=10.0)
|
||||
pred_away_goals: float = Field(ge=0.0, le=10.0)
|
||||
pred_1x2: str
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
subjective_confidence: float = Field(ge=0.0, le=1.0)
|
||||
reasoning: str = ""
|
||||
|
||||
@field_validator("pred_1x2")
|
||||
@@ -87,7 +87,7 @@ def validate_agent_output(raw: dict) -> AgentReportSchema:
|
||||
data_sufficiency=raw.get("data_sufficiency", "medium"),
|
||||
analysis=raw.get("analysis", ""),
|
||||
home_edge=_safe_float(raw.get("home_edge")),
|
||||
confidence=_safe_float(raw.get("confidence")),
|
||||
subjective_confidence=_safe_float(raw.get("subjective_confidence") or raw.get("confidence")),
|
||||
key_evidence=raw.get("key_evidence", []),
|
||||
exp_home_goals=_safe_float(raw.get("exp_home_goals")),
|
||||
exp_away_goals=_safe_float(raw.get("exp_away_goals")),
|
||||
@@ -101,7 +101,7 @@ def validate_prediction_output(raw: dict) -> PredictionOutputSchema:
|
||||
pred_home_goals=float(raw.get("pred_home_goals", 0)),
|
||||
pred_away_goals=float(raw.get("pred_away_goals", 0)),
|
||||
pred_1x2=raw.get("1x2") or raw.get("pred_1x2", "X"),
|
||||
confidence=float(raw.get("confidence", 0.5)),
|
||||
subjective_confidence=float(raw.get("confidence", 0.5)),
|
||||
reasoning=str(raw.get("reasoning", ""))[:1000],
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user