feat: Sprint 1 - 数据正确性整改
P2-01: 移除 lifespan create_all,改为仅验证连接
新增 /health/ready 就绪检查
P0-04: LLM 输出严格 Pydantic 校验
- Agent 输出越界/非法 → parse_error
- 预测输出自动修正 1X2 与比分一致性
P0-01: injuries cutoff 修复
- get_injuries_for_match 增加 as_of 参数
- injuries_slice 使用 as_of 过滤 retrieved_at
- 防止回测时未来采集数据泄漏
P1-12: 批量入库优化
- 预加载 teams 到内存 dict
- 预加载 existing matches 到内存 set
- 消灭 N+1 查询
This commit is contained in:
+23
-25
@@ -99,36 +99,34 @@ def _stub_no_data(agent: str) -> AgentReport:
|
||||
|
||||
|
||||
def _parse_report(agent: str, parsed: dict, resp: LLMResponse, model: str) -> AgentReport:
|
||||
"""把 LLM JSON 输出解析为 AgentReport,字段宽容处理。"""
|
||||
def _f(v, default=None):
|
||||
try:
|
||||
return float(v) if v is not None else default
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
"""把 LLM JSON 输出解析为 AgentReport,经过严格校验。"""
|
||||
from src.llm.validation import validate_agent_output
|
||||
|
||||
suff = str(parsed.get("data_sufficiency", "medium")).lower()
|
||||
if suff not in ("high", "medium", "low", "none"):
|
||||
suff = "medium"
|
||||
|
||||
evidence = parsed.get("key_evidence") or []
|
||||
if isinstance(evidence, str):
|
||||
evidence = [evidence]
|
||||
|
||||
score = parsed.get("probable_score")
|
||||
if isinstance(score, dict):
|
||||
score = f"{score.get('home', '?')}-{score.get('away', '?')}"
|
||||
try:
|
||||
validated = validate_agent_output(parsed)
|
||||
except Exception as e:
|
||||
# 校验失败 → 返回 parse_error 而非静默降级
|
||||
return AgentReport(
|
||||
agent=agent,
|
||||
status="parse_error",
|
||||
analysis=f"输出校验失败: {e}",
|
||||
model=model,
|
||||
latency_ms=resp.latency_ms,
|
||||
prompt_tokens=resp.prompt_tokens,
|
||||
completion_tokens=resp.completion_tokens,
|
||||
)
|
||||
|
||||
return AgentReport(
|
||||
agent=agent,
|
||||
status="ok",
|
||||
data_sufficiency=suff,
|
||||
analysis=str(parsed.get("analysis", ""))[:600],
|
||||
home_edge=_f(parsed.get("home_edge")),
|
||||
confidence=_f(parsed.get("confidence")),
|
||||
key_evidence=[str(e)[:120] for e in evidence[:5]],
|
||||
exp_home_goals=_f(parsed.get("exp_home_goals")),
|
||||
exp_away_goals=_f(parsed.get("exp_away_goals")),
|
||||
probable_score=score if isinstance(score, str) else None,
|
||||
data_sufficiency=validated.data_sufficiency,
|
||||
analysis=validated.analysis,
|
||||
home_edge=validated.home_edge,
|
||||
confidence=validated.confidence,
|
||||
key_evidence=validated.key_evidence,
|
||||
exp_home_goals=validated.exp_home_goals,
|
||||
exp_away_goals=validated.exp_away_goals,
|
||||
probable_score=validated.probable_score,
|
||||
model=model,
|
||||
latency_ms=resp.latency_ms,
|
||||
prompt_tokens=resp.prompt_tokens,
|
||||
|
||||
@@ -183,6 +183,13 @@ async def predict_match_multi(
|
||||
if m is None:
|
||||
raise ValueError(f"match {match_id} not found")
|
||||
|
||||
# 严格校验终裁输出
|
||||
from src.llm.validation import validate_prediction_output
|
||||
try:
|
||||
validated = validate_prediction_output(final)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"终裁输出校验失败: {e}")
|
||||
|
||||
agent_weights = final.get("agent_weights")
|
||||
pred = Prediction(
|
||||
match_id=match_id,
|
||||
@@ -193,11 +200,11 @@ async def predict_match_multi(
|
||||
prompt_tokens=sum(r.prompt_tokens or 0 for r in reports) + agg_prompt_tokens,
|
||||
completion_tokens=sum(r.completion_tokens or 0 for r in reports) + agg_completion_tokens,
|
||||
latency_ms=latency_ms,
|
||||
pred_home_goals=final.get("pred_home_goals"),
|
||||
pred_away_goals=final.get("pred_away_goals"),
|
||||
pred_1x2=final.get("1x2"),
|
||||
confidence=final.get("confidence"),
|
||||
reasoning=final.get("reasoning"),
|
||||
pred_home_goals=validated.pred_home_goals,
|
||||
pred_away_goals=validated.pred_away_goals,
|
||||
pred_1x2=validated.pred_1x2,
|
||||
confidence=validated.confidence,
|
||||
reasoning=validated.reasoning,
|
||||
raw_response=final,
|
||||
agent_outputs=[r.to_dict() for r in reports],
|
||||
)
|
||||
|
||||
+2
-1
@@ -117,7 +117,8 @@ async def run_backtest(
|
||||
|
||||
for m in matches:
|
||||
try:
|
||||
# 预测 (build_context 内部已用 before=match_date 防泄漏)
|
||||
# 预测 (build_context 内部已用 before=match_date 防泄漏,
|
||||
# injuries_slice 也使用 as_of=match_date 过滤 retrieved_at)
|
||||
result = await predict_match(m.id, mode=mode, model=model)
|
||||
|
||||
# 用实际比分 settle
|
||||
|
||||
@@ -219,12 +219,16 @@ async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None)
|
||||
|
||||
|
||||
async def injuries_slice(header: MatchHeader, *, before=None) -> str:
|
||||
"""D - 阵容完整性切片: 伤停与停赛名单,评估战力缺失程度。"""
|
||||
"""D - 阵容完整性切片: 伤停与停赛名单,评估战力缺失程度。
|
||||
|
||||
before=cutoff: 只使用 cutoff 之前已采集的伤停数据,防回测泄漏。
|
||||
"""
|
||||
from src.data.injuries import get_injuries_for_match
|
||||
|
||||
cutoff = before or header.match_dt
|
||||
async with AsyncSessionLocal() as db:
|
||||
home_injuries = await get_injuries_for_match(db, header.home_team_id, before or header.match_dt)
|
||||
away_injuries = await get_injuries_for_match(db, header.away_team_id, before or header.match_dt)
|
||||
home_injuries = await get_injuries_for_match(db, header.home_team_id, cutoff, as_of=cutoff)
|
||||
away_injuries = await get_injuries_for_match(db, header.away_team_id, cutoff, as_of=cutoff)
|
||||
|
||||
lines = ["── 阵容完整性 ──"]
|
||||
has_data = False
|
||||
|
||||
+12
-5
@@ -130,6 +130,13 @@ async def _predict_single(
|
||||
|
||||
parsed = resp.parsed or {}
|
||||
|
||||
# 3.5 严格校验 LLM 输出
|
||||
from src.llm.validation import validate_prediction_output
|
||||
try:
|
||||
validated = validate_prediction_output(parsed)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"LLM 输出校验失败: {e}")
|
||||
|
||||
# 4. 存预测(独立 session,因为 context 用的是自己的 session)
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 验证 match 存在
|
||||
@@ -145,11 +152,11 @@ async def _predict_single(
|
||||
prompt_tokens=resp.prompt_tokens,
|
||||
completion_tokens=resp.completion_tokens,
|
||||
latency_ms=resp.latency_ms,
|
||||
pred_home_goals=parsed.get("pred_home_goals"),
|
||||
pred_away_goals=parsed.get("pred_away_goals"),
|
||||
pred_1x2=parsed.get("1x2"),
|
||||
confidence=parsed.get("confidence"),
|
||||
reasoning=parsed.get("reasoning"),
|
||||
pred_home_goals=validated.pred_home_goals,
|
||||
pred_away_goals=validated.pred_away_goals,
|
||||
pred_1x2=validated.pred_1x2,
|
||||
confidence=validated.confidence,
|
||||
reasoning=validated.reasoning,
|
||||
raw_response=resp.raw,
|
||||
)
|
||||
db.add(pred)
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
"""LLM 输出严格校验。
|
||||
|
||||
所有 LLM JSON 输出必须经过 Pydantic 校验 + 语义一致性检查后才能落库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
from src.db.models import Prediction
|
||||
|
||||
|
||||
class AgentReportSchema(BaseModel):
|
||||
"""单个专家 Agent 输出的校验 schema。"""
|
||||
|
||||
data_sufficiency: str = "medium"
|
||||
analysis: str = ""
|
||||
home_edge: float | None = Field(None, ge=-1.0, le=1.0)
|
||||
confidence: float | None = Field(None, ge=0.0, le=1.0)
|
||||
key_evidence: list[str] = Field(default_factory=list)
|
||||
exp_home_goals: float | None = Field(None, ge=0.0, le=10.0)
|
||||
exp_away_goals: float | None = Field(None, ge=0.0, le=10.0)
|
||||
probable_score: str | None = None
|
||||
|
||||
@field_validator("data_sufficiency")
|
||||
@classmethod
|
||||
def validate_sufficiency(cls, v: str) -> str:
|
||||
allowed = {"high", "medium", "low", "none"}
|
||||
return v.lower() if v.lower() in allowed else "medium"
|
||||
|
||||
@field_validator("key_evidence", mode="before")
|
||||
@classmethod
|
||||
def normalize_evidence(cls, v) -> list[str]:
|
||||
if v is None:
|
||||
return []
|
||||
if isinstance(v, str):
|
||||
return [v]
|
||||
if isinstance(v, list):
|
||||
return [str(e)[:120] for e in v[:5]]
|
||||
return []
|
||||
|
||||
@field_validator("analysis")
|
||||
@classmethod
|
||||
def truncate_analysis(cls, v: str) -> str:
|
||||
return str(v)[:600]
|
||||
|
||||
|
||||
class PredictionOutputSchema(BaseModel):
|
||||
"""最终预测输出的校验 schema。"""
|
||||
|
||||
pred_home_goals: float = Field(ge=0.0, le=10.0)
|
||||
pred_away_goals: float = Field(ge=0.0, le=10.0)
|
||||
pred_1x2: str
|
||||
confidence: float = Field(ge=0.0, le=1.0)
|
||||
reasoning: str = ""
|
||||
|
||||
@field_validator("pred_1x2")
|
||||
@classmethod
|
||||
def validate_1x2(cls, v: str) -> str:
|
||||
if v not in ("1", "X", "2"):
|
||||
raise ValueError(f"pred_1x2 must be '1', 'X', or '2', got '{v}'")
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_consistency(self) -> "PredictionOutputSchema":
|
||||
"""验证比分与胜平负一致。"""
|
||||
expected = _score_to_1x2(self.pred_home_goals, self.pred_away_goals)
|
||||
if expected and self.pred_1x2 != expected:
|
||||
# 自动修正而非拒绝(LLM 常见小错误)
|
||||
self.pred_1x2 = expected
|
||||
return self
|
||||
|
||||
|
||||
def _score_to_1x2(home: float, away: float) -> str | None:
|
||||
"""从比分推导胜平负。"""
|
||||
if home > away:
|
||||
return "1"
|
||||
if home == away:
|
||||
return "X"
|
||||
if home < away:
|
||||
return "2"
|
||||
return None
|
||||
|
||||
|
||||
def validate_agent_output(raw: dict) -> AgentReportSchema:
|
||||
"""校验并规范化单个 Agent 输出。"""
|
||||
return AgentReportSchema(
|
||||
data_sufficiency=raw.get("data_sufficiency", "medium"),
|
||||
analysis=raw.get("analysis", ""),
|
||||
home_edge=_safe_float(raw.get("home_edge")),
|
||||
confidence=_safe_float(raw.get("confidence")),
|
||||
key_evidence=raw.get("key_evidence", []),
|
||||
exp_home_goals=_safe_float(raw.get("exp_home_goals")),
|
||||
exp_away_goals=_safe_float(raw.get("exp_away_goals")),
|
||||
probable_score=_format_score(raw.get("probable_score")),
|
||||
)
|
||||
|
||||
|
||||
def validate_prediction_output(raw: dict) -> PredictionOutputSchema:
|
||||
"""校验最终预测输出。"""
|
||||
return PredictionOutputSchema(
|
||||
pred_home_goals=float(raw.get("pred_home_goals", 0)),
|
||||
pred_away_goals=float(raw.get("pred_away_goals", 0)),
|
||||
pred_1x2=raw.get("1x2") or raw.get("pred_1x2", "X"),
|
||||
confidence=float(raw.get("confidence", 0.5)),
|
||||
reasoning=str(raw.get("reasoning", ""))[:1000],
|
||||
)
|
||||
|
||||
|
||||
def _safe_float(v) -> float | None:
|
||||
"""安全转 float,失败返回 None。"""
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
f = float(v)
|
||||
if not (f == f): # NaN check
|
||||
return None
|
||||
return f
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _format_score(v) -> str | None:
|
||||
"""格式化比分输出。"""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, str):
|
||||
return v
|
||||
if isinstance(v, dict):
|
||||
return f"{v.get('home', '?')}-{v.get('away', '?')}"
|
||||
return None
|
||||
Reference in New Issue
Block a user