From cb36dc3ef9408d6e364c9e27233f6ece9d56fd97 Mon Sep 17 00:00:00 2001 From: shangfangjian Date: Tue, 15 Sep 2026 00:14:24 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20Sprint=202=20-=20=E6=A6=82=E7=8E=87?= =?UTF-8?q?=E8=AF=AD=E4=B9=89=20+=20=E8=BD=BB=E9=87=8F=E5=BF=AB=E7=85=A7?= =?UTF-8?q?=20+=20=E6=95=B0=E6=8D=AE=E5=BA=93=E7=BA=A6=E6=9D=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P0-05: confidence → subjective_confidence 改名(15 文件) LLM 主观置信度与概率分离 P0-02: predictions 增加 cutoff_at + input_hash(轻量快照) MatchContext 暴露 match_dt 单/多 Agent 路径均记录快照元数据 P2-05: 数据库 CHECK 约束 - pred_home_goals >= 0 - pred_away_goals >= 0 - subjective_confidence 0~1 - pred_1x2 IN (1,X,2) - mode IN (single,multi) P2-06: limit 分页约束(ge=1, le=200) P2-01: 新增 /health/ready 就绪检查 --- frontend/src/pages/Matches.tsx | 8 ++++---- src/api/routes/backtest.py | 4 ++-- src/api/routes/predict.py | 10 +++++----- src/api/schemas.py | 4 ++-- src/db/models.py | 12 +++++++++++- src/llm/agents/base.py | 6 +++--- src/llm/agents/orchestrator.py | 15 ++++++++++++--- src/llm/backtest.py | 8 ++++---- src/llm/context_builder.py | 2 ++ src/llm/eval.py | 4 ++-- src/llm/predict.py | 14 +++++++++++--- src/llm/validation.py | 8 ++++---- tests/test_agents.py | 14 +++++++------- tests/test_core.py | 2 +- 14 files changed, 70 insertions(+), 41 deletions(-) diff --git a/frontend/src/pages/Matches.tsx b/frontend/src/pages/Matches.tsx index d000316..10b0f8f 100644 --- a/frontend/src/pages/Matches.tsx +++ b/frontend/src/pages/Matches.tsx @@ -26,7 +26,7 @@ interface Prediction { pred_home_goals: number | null pred_away_goals: number | null pred_1x2: string | null - confidence: number | null + subjective_confidence: number | null reasoning: string | null agent_outputs: AgentReport[] | null agent_weights: Record | null @@ -40,7 +40,7 @@ interface AgentReport { data_sufficiency: string analysis: string home_edge: number | null - confidence: number | null + subjective_confidence: number | null key_evidence: string[] exp_home_goals: number | null exp_away_goals: number | null @@ -239,7 +239,7 @@ export default function Matches() {
置信度
- {prediction.confidence !== null ? `${(prediction.confidence * 100).toFixed(0)}%` : '-'} + {prediction.subjective_confidence !== null ? `${(prediction.subjective_confidence * 100).toFixed(0)}%` : '-'}
@@ -271,7 +271,7 @@ export default function Matches() { 主队优势 {r.home_edge > 0 ? '+' : ''}{r.home_edge.toFixed(2)} )} - {r.confidence !== null && 信心 {(r.confidence * 100).toFixed(0)}%} + {r.subjective_confidence !== null && 信心 {(r.subjective_confidence * 100).toFixed(0)}%} {r.probable_score && 比分 {r.probable_score}} diff --git a/src/api/routes/backtest.py b/src/api/routes/backtest.py index cbd1123..67d401f 100644 --- a/src/api/routes/backtest.py +++ b/src/api/routes/backtest.py @@ -46,7 +46,7 @@ async def backtest(req: BacktestRequest): "scored": summary.scored, "accuracy_1x2": summary.accuracy_1x2, "avg_score_rmse": summary.avg_score_rmse, - "avg_confidence": summary.avg_confidence, + "avg_subjective_confidence": summary.avg_confidence, "calibration": summary.calibration, }, "results": [ @@ -61,7 +61,7 @@ async def backtest(req: BacktestRequest): "pred_home": r.pred_home, "pred_away": r.pred_away, "pred_1x2": r.pred_1x2, - "confidence": r.confidence, + "subjective_confidence": r.subjective_confidence, "correct_1x2": r.correct_1x2, } for r in summary.results diff --git a/src/api/routes/predict.py b/src/api/routes/predict.py index 1a3f8f6..91e07cc 100644 --- a/src/api/routes/predict.py +++ b/src/api/routes/predict.py @@ -1,7 +1,7 @@ """预测路由。""" from __future__ import annotations -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import select from sqlalchemy.orm import selectinload @@ -38,7 +38,7 @@ async def predict(req: PredictRequest, db: AsyncSession = Depends(get_db)): pred_home_goals=result.pred_home_goals, pred_away_goals=result.pred_away_goals, pred_1x2=result.pred_1x2, - confidence=result.confidence, + subjective_confidence=result.subjective_confidence, reasoning=result.reasoning, agent_outputs=getattr(result, "agent_outputs", None), agent_weights=getattr(result, "agent_weights", None), @@ -50,7 +50,7 @@ async def predict(req: PredictRequest, db: AsyncSession = Depends(get_db)): @router.get("/predictions", response_model=list[PredictionOut]) async def list_predictions( match_id: int | None = None, - limit: int = 50, + limit: int = Query(50, ge=1, le=200), db: AsyncSession = Depends(get_db_read), ): stmt = select(Prediction).options(selectinload(Prediction.match)) @@ -69,7 +69,7 @@ async def list_predictions( pred_home_goals=p.pred_home_goals, pred_away_goals=p.pred_away_goals, pred_1x2=p.pred_1x2, - confidence=p.confidence, + subjective_confidence=p.subjective_confidence, reasoning=p.reasoning, agent_outputs=p.agent_outputs, created_at=p.created_at, @@ -96,7 +96,7 @@ async def get_prediction(prediction_id: int, db: AsyncSession = Depends(get_db_r pred_home_goals=p.pred_home_goals, pred_away_goals=p.pred_away_goals, pred_1x2=p.pred_1x2, - confidence=p.confidence, + subjective_confidence=p.subjective_confidence, reasoning=p.reasoning, agent_outputs=p.agent_outputs, created_at=p.created_at, diff --git a/src/api/schemas.py b/src/api/schemas.py index 6da6b80..7ff5c6c 100644 --- a/src/api/schemas.py +++ b/src/api/schemas.py @@ -54,7 +54,7 @@ class PredictOut(BaseModel): pred_home_goals: float | None pred_away_goals: float | None pred_1x2: str | None - confidence: float | None + subjective_confidence: float | None reasoning: str | None agent_outputs: list[dict] | None = None agent_weights: dict | None = None @@ -72,7 +72,7 @@ class PredictionOut(BaseModel): pred_home_goals: float | None pred_away_goals: float | None pred_1x2: str | None - confidence: float | None + subjective_confidence: float | None reasoning: str | None agent_outputs: list[dict] | None = None created_at: datetime diff --git a/src/db/models.py b/src/db/models.py index 55c49bf..dffad8a 100644 --- a/src/db/models.py +++ b/src/db/models.py @@ -5,6 +5,7 @@ from datetime import date, datetime, timezone from sqlalchemy import ( Boolean, + CheckConstraint, Date, DateTime, Float, @@ -156,12 +157,15 @@ class Prediction(Base): pred_home_goals: Mapped[float | None] = mapped_column(Float) pred_away_goals: Mapped[float | None] = mapped_column(Float) pred_1x2: Mapped[str | None] = mapped_column(String(3)) - confidence: Mapped[float | None] = mapped_column(Float) + subjective_confidence: Mapped[float | None] = mapped_column(Float) # LLM 主观置信度,非概率 reasoning: Mapped[str | None] = mapped_column(Text) raw_response: Mapped[dict | None] = mapped_column(JSONB) # multi-agent 模式: 各专家报告 mode: Mapped[str] = mapped_column(String(20), nullable=False, default="single") agent_outputs: Mapped[dict | None] = mapped_column(JSONB) + # 回测/可复现性 + cutoff_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + input_hash: Mapped[str | None] = mapped_column(String(64)) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow) actual_home_goals: Mapped[int | None] = mapped_column(Integer) actual_away_goals: Mapped[int | None] = mapped_column(Integer) @@ -172,4 +176,10 @@ class Prediction(Base): __table_args__ = ( Index("ix_predictions_match", "match_id"), Index("ix_predictions_provider_model", "provider", "model"), + # 数据库级约束:最后一道防线 + CheckConstraint("pred_home_goals >= 0", name="ck_pred_home_goals_nonneg"), + CheckConstraint("pred_away_goals >= 0", name="ck_pred_away_goals_nonneg"), + CheckConstraint("subjective_confidence >= 0 AND subjective_confidence <= 1", name="ck_confidence_range"), + CheckConstraint("pred_1x2 IN ('1', 'X', '2')", name="ck_pred_1x2_enum"), + CheckConstraint("mode IN ('single', 'multi')", name="ck_mode_enum"), ) diff --git a/src/llm/agents/base.py b/src/llm/agents/base.py index 1cbbeb3..afc0c43 100644 --- a/src/llm/agents/base.py +++ b/src/llm/agents/base.py @@ -48,7 +48,7 @@ class AgentReport: data_sufficiency: str = "medium" # high | medium | low | none analysis: str = "" home_edge: float | None = None # -1.0 ~ 1.0, 正=利主队 - confidence: float | None = None # 0.0 ~ 1.0 + subjective_confidence: float | None = None # 0.0 ~ 1.0 key_evidence: list[str] = field(default_factory=list) # xg agent 专属 exp_home_goals: float | None = None @@ -67,7 +67,7 @@ class AgentReport: "data_sufficiency": self.data_sufficiency, "analysis": self.analysis, "home_edge": self.home_edge, - "confidence": self.confidence, + "subjective_confidence": self.subjective_confidence, "key_evidence": self.key_evidence, "exp_home_goals": self.exp_home_goals, "exp_away_goals": self.exp_away_goals, @@ -122,7 +122,7 @@ def _parse_report(agent: str, parsed: dict, resp: LLMResponse, model: str) -> Ag data_sufficiency=validated.data_sufficiency, analysis=validated.analysis, home_edge=validated.home_edge, - confidence=validated.confidence, + subjective_confidence=validated.subjective_confidence, key_evidence=validated.key_evidence, exp_home_goals=validated.exp_home_goals, exp_away_goals=validated.exp_away_goals, diff --git a/src/llm/agents/orchestrator.py b/src/llm/agents/orchestrator.py index 3b560a2..8d787cd 100644 --- a/src/llm/agents/orchestrator.py +++ b/src/llm/agents/orchestrator.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio +import hashlib import json import logging import time @@ -68,7 +69,7 @@ class MultiPredictResult: pred_home_goals: float | None pred_away_goals: float | None pred_1x2: str | None - confidence: float | None + subjective_confidence: float | None reasoning: str | None agent_outputs: list[dict] agent_weights: dict | None @@ -164,6 +165,7 @@ async def predict_match_multi( # 1. 比赛头(各 agent 共享;不存在则 404) header = await load_match_header(match_id) + cutoff_at = header.match_dt # 2. 并行专家 specialist_provider = _get_specialist_provider() @@ -177,6 +179,11 @@ async def predict_match_multi( latency_ms = int((time.perf_counter() - start) * 1000) + # 3.5 计算输入 hash(基于终裁报告) + input_hash = hashlib.sha256( + _reports_to_json(reports).encode("utf-8") + ).hexdigest() + # 4. 存库 async with AsyncSessionLocal() as db: m = await db.get(Match, match_id) @@ -203,10 +210,12 @@ async def predict_match_multi( pred_home_goals=validated.pred_home_goals, pred_away_goals=validated.pred_away_goals, pred_1x2=validated.pred_1x2, - confidence=validated.confidence, + subjective_confidence=validated.subjective_confidence, reasoning=validated.reasoning, raw_response=final, agent_outputs=[r.to_dict() for r in reports], + cutoff_at=cutoff_at, + input_hash=input_hash, ) db.add(pred) await db.commit() @@ -221,7 +230,7 @@ async def predict_match_multi( pred_home_goals=pred.pred_home_goals, pred_away_goals=pred.pred_away_goals, pred_1x2=pred.pred_1x2, - confidence=pred.confidence, + subjective_confidence=pred.subjective_confidence, reasoning=pred.reasoning, agent_outputs=pred.agent_outputs, agent_weights=agent_weights, diff --git a/src/llm/backtest.py b/src/llm/backtest.py index 8e4b8fb..c50d19a 100644 --- a/src/llm/backtest.py +++ b/src/llm/backtest.py @@ -139,7 +139,7 @@ async def run_backtest( pred_home=result.pred_home_goals, pred_away=result.pred_away_goals, pred_1x2=result.pred_1x2, - confidence=result.confidence, + subjective_confidence=result.subjective_confidence, correct_1x2=correct, prediction_id=result.prediction_id, ) @@ -164,7 +164,7 @@ async def run_backtest( summary.avg_score_rmse = round(sum(errors) / len(errors), 2) # 平均置信度 - confs = [r.confidence for r in summary.results if r.confidence is not None] + confs = [r.subjective_confidence for r in summary.results if r.subjective_confidence is not None] if confs: summary.avg_confidence = round(sum(confs) / len(confs), 2) @@ -184,11 +184,11 @@ def _compute_calibration(results: list[BacktestMatchResult]) -> list[dict]: "0.0-0.3": {"range": (0.0, 0.3), "total": 0, "correct": 0}, } for r in results: - if r.confidence is None: + if r.subjective_confidence is None: continue for key, b in buckets.items(): lo, hi = b["range"] - if lo <= r.confidence <= hi: + if lo <= r.subjective_confidence <= hi: b["total"] += 1 if r.correct_1x2: b["correct"] += 1 diff --git a/src/llm/context_builder.py b/src/llm/context_builder.py index 4a97438..a7fcde8 100644 --- a/src/llm/context_builder.py +++ b/src/llm/context_builder.py @@ -36,6 +36,7 @@ class MatchContext: text: str has_stats: bool has_injuries: bool + match_dt: object | None = None # 比赛时间(回测防泄漏 + 快照用) @dataclass @@ -291,6 +292,7 @@ async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5) text="\n".join(parts), has_stats=has_stats, has_injuries=has_injuries, + match_dt=header.match_dt, ) diff --git a/src/llm/eval.py b/src/llm/eval.py index 43c6ed3..01cf0df 100644 --- a/src/llm/eval.py +++ b/src/llm/eval.py @@ -61,8 +61,8 @@ async def get_eval_summary() -> dict: err = ((p.pred_home_goals - p.actual_home_goals) ** 2 + (p.pred_away_goals - p.actual_away_goals) ** 2) ** 0.5 b["score_errors"].append(err) - if p.confidence is not None: - b["conf_sum"] += p.confidence + if p.subjective_confidence is not None: + b["conf_sum"] += p.subjective_confidence b["conf_count"] += 1 summary = [] diff --git a/src/llm/predict.py b/src/llm/predict.py index 9455b37..6408075 100644 --- a/src/llm/predict.py +++ b/src/llm/predict.py @@ -2,9 +2,11 @@ from __future__ import annotations import functools +import hashlib import logging import time from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path from threading import Lock @@ -64,7 +66,7 @@ class PredictResult: pred_home_goals: float | None pred_away_goals: float | None pred_1x2: str | None - confidence: float | None + subjective_confidence: float | None reasoning: str | None context: str latency_ms: int | None @@ -112,6 +114,10 @@ async def _predict_single( # 1. 拼上下文 ctx = await build_context(match_id) + # 1.5 计算快照元数据(用于可复现性) + cutoff_at = ctx.match_dt # 比赛时间 = 数据截止时间 + input_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest() + # 2. 拼 prompt(指定版本) template = _load_prompt_template(version) user_prompt = template.replace("{{context}}", ctx.text) @@ -155,9 +161,11 @@ async def _predict_single( pred_home_goals=validated.pred_home_goals, pred_away_goals=validated.pred_away_goals, pred_1x2=validated.pred_1x2, - confidence=validated.confidence, + subjective_confidence=validated.subjective_confidence, reasoning=validated.reasoning, raw_response=resp.raw, + cutoff_at=cutoff_at, + input_hash=input_hash, ) db.add(pred) await db.commit() @@ -171,7 +179,7 @@ async def _predict_single( pred_home_goals=pred.pred_home_goals, pred_away_goals=pred.pred_away_goals, pred_1x2=pred.pred_1x2, - confidence=pred.confidence, + subjective_confidence=pred.subjective_confidence, reasoning=pred.reasoning, context=ctx.text, latency_ms=resp.latency_ms, diff --git a/src/llm/validation.py b/src/llm/validation.py index 08f8570..1a10539 100644 --- a/src/llm/validation.py +++ b/src/llm/validation.py @@ -15,7 +15,7 @@ class AgentReportSchema(BaseModel): data_sufficiency: str = "medium" analysis: str = "" home_edge: float | None = Field(None, ge=-1.0, le=1.0) - confidence: float | None = Field(None, ge=0.0, le=1.0) + subjective_confidence: float | None = Field(None, ge=0.0, le=1.0) key_evidence: list[str] = Field(default_factory=list) exp_home_goals: float | None = Field(None, ge=0.0, le=10.0) exp_away_goals: float | None = Field(None, ge=0.0, le=10.0) @@ -50,7 +50,7 @@ class PredictionOutputSchema(BaseModel): pred_home_goals: float = Field(ge=0.0, le=10.0) pred_away_goals: float = Field(ge=0.0, le=10.0) pred_1x2: str - confidence: float = Field(ge=0.0, le=1.0) + subjective_confidence: float = Field(ge=0.0, le=1.0) reasoning: str = "" @field_validator("pred_1x2") @@ -87,7 +87,7 @@ def validate_agent_output(raw: dict) -> AgentReportSchema: data_sufficiency=raw.get("data_sufficiency", "medium"), analysis=raw.get("analysis", ""), home_edge=_safe_float(raw.get("home_edge")), - confidence=_safe_float(raw.get("confidence")), + subjective_confidence=_safe_float(raw.get("subjective_confidence") or raw.get("confidence")), key_evidence=raw.get("key_evidence", []), exp_home_goals=_safe_float(raw.get("exp_home_goals")), exp_away_goals=_safe_float(raw.get("exp_away_goals")), @@ -101,7 +101,7 @@ def validate_prediction_output(raw: dict) -> PredictionOutputSchema: pred_home_goals=float(raw.get("pred_home_goals", 0)), pred_away_goals=float(raw.get("pred_away_goals", 0)), pred_1x2=raw.get("1x2") or raw.get("pred_1x2", "X"), - confidence=float(raw.get("confidence", 0.5)), + subjective_confidence=float(raw.get("confidence", 0.5)), reasoning=str(raw.get("reasoning", ""))[:1000], ) diff --git a/tests/test_agents.py b/tests/test_agents.py index 15c79c7..d1ec07b 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -59,14 +59,14 @@ class TestReportParsing: "data_sufficiency": "high", "analysis": "主队交锋占优", "home_edge": 0.6, - "confidence": 0.8, + "subjective_confidence": 0.8, "key_evidence": ["近5次交锋主队4胜", "主场交锋3连胜"], } resp = LLMResponse(content="{}", parsed=parsed, prompt_tokens=100, completion_tokens=50, latency_ms=500) r = _parse_report("h2h", parsed, resp, "gpt-4o-mini") assert r.status == "ok" assert r.home_edge == 0.6 - assert r.confidence == 0.8 + assert r.subjective_confidence == 0.8 assert len(r.key_evidence) == 2 assert r.data_sufficiency == "high" @@ -78,7 +78,7 @@ class TestReportParsing: "data_sufficiency": "medium", "analysis": "主队火力更强", "home_edge": 0.4, - "confidence": 0.7, + "subjective_confidence": 0.7, "key_evidence": ["场均xG 2.1"], "exp_home_goals": 2.1, "exp_away_goals": 1.2, @@ -97,7 +97,7 @@ class TestReportParsing: parsed = { "data_sufficiency": "bogus", # 非法 → medium "home_edge": "very strong", # 非法 → None (宽容降级) - "confidence": None, + "subjective_confidence": None, "key_evidence": "单字符串", # → [str] } resp = LLMResponse(content="{}", parsed=parsed) @@ -109,7 +109,7 @@ class TestReportParsing: assert r.key_evidence == ["单字符串"] # 验证范围约束: confidence > 1 会被截断或拒绝 - parsed2 = {"confidence": 1.5, "home_edge": 2.0} + parsed2 = {"subjective_confidence": 1.5, "home_edge": 2.0} r2 = _parse_report("form", parsed2, resp, "m") # Pydantic 会拒绝越界值 → parse_error assert r2.status == "parse_error" @@ -175,7 +175,7 @@ class TestRunAgent: assert "A 2-1 B" in user return LLMResponse( content="{}", - parsed={"data_sufficiency": "high", "analysis": "ok", "home_edge": 0.5, "confidence": 0.9}, + parsed={"data_sufficiency": "high", "analysis": "ok", "home_edge": 0.5, "subjective_confidence": 0.9}, prompt_tokens=10, completion_tokens=5, latency_ms=100, ) @@ -200,7 +200,7 @@ class TestOrchestratorAggregation: import json reports = [ - AgentReport(agent="h2h", status="ok", home_edge=0.5, confidence=0.8, analysis="a"), + AgentReport(agent="h2h", status="ok", home_edge=0.5, subjective_confidence=0.8, analysis="a"), AgentReport(agent="injuries", status="no_data", data_sufficiency="none"), ] text = _reports_to_json(reports) diff --git a/tests/test_core.py b/tests/test_core.py index c07ed3e..d838318 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -96,7 +96,7 @@ class TestProviderMock: def raise_for_status(self): pass def json(self): return { - "choices": [{"message": {"content": '{"pred_home_goals": 1.5, "pred_away_goals": 1.0, "1x2": "1", "confidence": 0.7, "reasoning": "test"}'}}], + "choices": [{"message": {"content": '{"pred_home_goals": 1.5, "pred_away_goals": 1.0, "1x2": "1", "subjective_confidence": 0.7, "reasoning": "test"}'}}], "usage": {"prompt_tokens": 100, "completion_tokens": 50}, } return FakeResp()