shangfangjian/Profeto
1
0
Fork 0
Code Issues Pull Requests 2 Actions Packages Projects Releases Wiki Activity

fix(P0-03): Prediction 幂等指纹——只追加,不覆盖

Browse Source 
_upsert_prediction 改为 _insert_or_find_by_fingerprint:
- 同 input_hash → 返回已有行(绝不 UPDATE pred_/reasoning/agent_outputs)
- 不同 input_hash → INSERT 新行

input_hash 升级为规范 JSON SHA-256,捕获:match_id, cutoff, prompt_version,
prompt_hash, system_prompt_hash, provider, model, mode, run_type, temperature,
context_hash, agent_ids。移除旧 (match, provider, model, mode, run_type) 唯一约束,
改为 partial unique index(WHERE input_hash IS NOT NULL,兼容旧 NULL 数据)。

三条路径(single/multi/baseline)统一传足指纹字段。
迁移 0024 + 测试 test_p0_prediction_fingerprint(10/10);全量 295 通过。
This commit is contained in:
shangfangjian
2026-09-22 03:13:32 +08:00
parent 49d78136a1
commit 64ae8e663a
11 changed files with 405 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llm/agents/orchestrator.py
+19 -12
View File
@@ -12,7 +12,7 @@ from src.core.config import settings
from src.db.base import AsyncSessionLocal
from src.db.models import Match, Prediction
from src.db.unit_of_work import get_uow
from src.llm.predict import PredictResult, _upsert_prediction
from src.llm.predict import PredictResult, _insert_or_find_by_fingerprint
from src.llm.agents.base import AgentReport, AgentSpec, load_agent_prompt
from src.llm.context_builder import (
MatchHeader,
@@ -285,10 +285,13 @@ async def predict_match_multi(
latency_ms = int((time.perf_counter() - start) * 1000)
# 3.5 计算输入 hash(基于终裁报告)
input_hash = hashlib.sha256(
_reports_to_json(reports).encode("utf-8")
).hexdigest()
# P0-03: 指纹输入——终裁报告 hash 作 context_hash,专家列表作 agent_ids
reports_json = _reports_to_json(reports)
context_hash = hashlib.sha256(reports_json.encode("utf-8")).hexdigest()
agent_ids = sorted([r.agent for r in reports]) if reports else []
# 终裁模板 hash(规范:复用 prompt 版本 + 终裁 system prompt)
prompt_hash = hashlib.sha256(f"multi_{version}".encode("utf-8")).hexdigest()
system_prompt_hash = hashlib.sha256(AGGREGATOR_SYSTEM.encode("utf-8")).hexdigest()
# 4. 存库(使用 UnitOfWork)
async with get_uow() as session:
@@ -315,15 +318,20 @@ async def predict_match_multi(
pred_status = "degraded"
model_name = aggregator_model
pred = await _upsert_prediction(
pred = await _insert_or_find_by_fingerprint(
session,
match_id=match_id,
provider_name=settings.LLM_PROVIDER,
model=model_name,
mode="multi",
run_type="backtest" if backtest else "live",
values={
"match_id": match_id,
"provider": settings.LLM_PROVIDER,
"model": model_name,
"mode": "multi",
"run_type": "backtest" if backtest else "live",
"prompt_version": f"multi_{version}",
"prompt_hash": prompt_hash,
"system_prompt_hash": system_prompt_hash,
"temperature": 0.2,
"context_hash": context_hash,
"agent_ids": agent_ids,
"prompt_tokens": sum(r.prompt_tokens or 0 for r in reports) + agg_prompt_tokens,
"completion_tokens": sum(r.completion_tokens or 0 for r in reports) + agg_completion_tokens,
"latency_ms": latency_ms,
@@ -341,7 +349,6 @@ async def predict_match_multi(
"match_kickoff_at": match_kickoff_at,
"prediction_cutoff_at": prediction_cutoff_at,
"prediction_created_at": now,
"input_hash": input_hash,
},
)
src/llm/baseline.py
+20 -12
View File
@@ -5,14 +5,15 @@
"""
from __future__ import annotations
import hashlib
import logging
from datetime import datetime
from datetime import datetime, timedelta, timezone
from sqlalchemy import case, func, select
from src.db.base import AsyncSession, AsyncSessionLocal
from src.db.models import Match
from src.llm.predict import PredictResult, _upsert_prediction
from src.llm.predict import PredictResult, _insert_or_find_by_fingerprint
logger = logging.getLogger(__name__)
@@ -97,8 +98,23 @@ async def predict_baseline(
else:
pred_1x2 = "X"
# P0-03: 基线指纹——基于主客场场均进球数据(context_hash) + 截止时间
context_hash = hashlib.sha256(
f"{home_avg:.4f}:{away_avg:.4f}:{before.isoformat() if before else 'none'}".encode("utf-8")
).hexdigest()
values = {
"match_id": match_id,
"provider": "baseline",
"model": "baseline",
"mode": "baseline",
"run_type": "live",
"prompt_version": "baseline_v1",
"prompt_hash": hashlib.sha256(b"baseline_v1").hexdigest(),
"system_prompt_hash": hashlib.sha256(b"baseline").hexdigest(),
"temperature": 0.0,
"context_hash": context_hash,
"agent_ids": [],
"prompt_tokens": 0,
"completion_tokens": 0,
"latency_ms": 0,
@@ -114,17 +130,9 @@ async def predict_baseline(
"status": "success",
}
# P3-2:服务层落库,回填真实 prediction_id(与 single/multi 统一)。
# P0-03:服务层幂等插入,回填真实 prediction_id(与 single/multi 统一)。
async with get_uow() as session:
pred = await _upsert_prediction(
session,
match_id=match_id,
provider_name="baseline",
model="baseline",
mode="baseline",
run_type="live",
values=values,
)
pred = await _insert_or_find_by_fingerprint(session, values=values)
prediction_id = pred.id
return PredictResult(
src/llm/predict.py
+68 -45
View File
@@ -4,9 +4,9 @@
| 模式 | 落库位置(服务层) | 路由层(routes/predict.py) |
|-----------|-------------------------------------------------------------|---------------------------|
| single | `_predict_single` → `_upsert_prediction` | 不读 DB,仅映射 result → PredictOut |
| multi | `orchestrator.predict_match_multi` → `_upsert_prediction` | 不读 DB,仅映射 result → PredictOut |
| baseline | `predict_baseline` → `_upsert_prediction` | 不读 DB,仅映射 result → PredictOut |
| single | `_predict_single` → `_insert_or_find_by_fingerprint` | 不读 DB,仅映射 result → PredictOut |
| multi | `orchestrator.predict_match_multi` → `_insert_or_find_by_fingerprint` | 不读 DB,仅映射 result → PredictOut |
| baseline | `predict_baseline` → `_insert_or_find_by_fingerprint` | 不读 DB,仅映射 result → PredictOut |
三种模式统一在服务层经 UnitOfWork 落库并回填真实 prediction_id;
路由层永不写入 predictions,只读 result.prediction_id 做响应映射。
@@ -220,44 +220,60 @@ class PredictResult:
raw: dict | None = None
async def _upsert_prediction(
session,
*,
match_id: int,
provider_name: str,
model: str,
mode: str,
run_type: str,
values: dict,
) -> Prediction:
"""按 (match, provider, model, mode, run_type) 唯一约束写入预测。
def _compute_fingerprint(values: dict) -> str:
"""P0-03: 预测指纹(规范 JSON 的 SHA-256)。
已存在且未结算 → 覆盖更新(重新预测语义);已结算 → 拒绝(保护评估数据)。
run_type 区分 live/backtest,避免回测覆盖实盘预测。
捕获影响预测输出的全部因素:输入、提示、模型、采样、截止时间、专家。
同 fingerprint → 返回已有行(不 UPDATE/INSERT);不同 → INSERT 新行。
"""
import json as _json
canonical = {
"match_id": values.get("match_id"),
"prediction_cutoff_at": _iso(values.get("prediction_cutoff_at")),
"prompt_version": values.get("prompt_version"),
"prompt_hash": values.get("prompt_hash"),
"system_prompt_hash": values.get("system_prompt_hash"),
"provider": values.get("provider"),
"model": values.get("model"),
"mode": values.get("mode"),
"run_type": values.get("run_type"),
"temperature": values.get("temperature"),
"context_hash": values.get("context_hash"),
"agent_ids": sorted(values.get("agent_ids") or []),
}
blob = _json.dumps(canonical, sort_keys=True, separators=(',', ':'))
return hashlib.sha256(blob.encode("utf-8")).hexdigest()
def _iso(v) -> str | None:
if v is None:
return None
if hasattr(v, "isoformat"):
return v.isoformat()
return str(v)
async def _insert_or_find_by_fingerprint(session, *, values: dict) -> Prediction:
"""P0-03: 幂等插入——同 input_hash 返回已有行(不 UPDATE);不同则 INSERT。
不再按 (match, provider, model, mode, run_type) 做 upsert,避免覆盖已有预测。
values 必须包含 fingerprint 所需全部字段(见 _compute_fingerprint)。
"""
fingerprint = _compute_fingerprint(values)
values["input_hash"] = fingerprint
existing = (
await session.execute(
select(Prediction).where(
Prediction.match_id == match_id,
Prediction.provider == provider_name,
Prediction.model == model,
Prediction.mode == mode,
Prediction.run_type == run_type,
)
select(Prediction).where(Prediction.input_hash == fingerprint)
)
).scalar_one_or_none()
if existing is not None and existing.settled:
raise ValueError("该比赛已有已结算的预测,不能重新预测")
if existing is not None:
# 同指纹 → 直接返回,绝不覆盖 pred_* / reasoning / agent_outputs
return existing
pred = existing if existing is not None else Prediction(
match_id=match_id, provider=provider_name, model=model,
)
pred.mode = mode
pred.run_type = run_type
for k, v in values.items():
setattr(pred, k, v)
if existing is None:
session.add(pred)
pred = Prediction(**{k: v for k, v in values.items() if hasattr(Prediction, k)})
session.add(pred)
await session.flush() # 拿到自增 id;事务由 UnitOfWork 退出时提交
return pred
@@ -342,23 +358,26 @@ async def _predict_single(
# 1. 拼上下文(backtest/cutoff 防泄漏)
ctx = await build_context(match_id, backtest=backtest, cutoff_at=cutoff_at)
# 1.5 计算快照元数据(用于可复现性)
# 1.5 计算快照元数据(用于可复现性 + P0-03 指纹)
now = datetime.now(timezone.utc)
match_kickoff_at = ctx.match_dt
# 使用上下文实际计算的 cutoff(回测时可能为 match_dt-1天),而非开球时间
prediction_cutoff_at = ctx.cutoff if ctx.cutoff is not None else ctx.match_dt
input_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
# 2. 拼 prompt(指定版本)
template = _load_prompt_template(version)
prompt_hash = _prompt_template_hash(version)
user_prompt = template.replace("{{context}}", ctx.text)
system_prompt = "你是一个严谨的足球预测专家。只输出 JSON。"
context_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
# 3. 调 LLM
temperature = 0.3
resp = await provider.chat(
system="你是一个严谨的足球预测专家。只输出 JSON。",
system=system_prompt,
user=user_prompt,
json_mode=True,
temperature=0.3,
temperature=temperature,
max_tokens=4096, # 推理模型的 reasoning 也计入输出 token,需留足余量
)
@@ -385,15 +404,20 @@ async def _predict_single(
if m is None:
raise ValueError(f"match {match_id} not found")
pred = await _upsert_prediction(
pred = await _insert_or_find_by_fingerprint(
session,
match_id=match_id,
provider_name=settings.LLM_PROVIDER,
model=provider.model,
mode="single",
run_type="backtest" if backtest else "live",
values={
"match_id": match_id,
"provider": settings.LLM_PROVIDER,
"model": provider.model,
"mode": "single",
"run_type": "backtest" if backtest else "live",
"prompt_version": version,
"prompt_hash": prompt_hash,
"system_prompt_hash": hashlib.sha256(system_prompt.encode("utf-8")).hexdigest(),
"temperature": temperature,
"context_hash": context_hash,
"agent_ids": [],
"prompt_tokens": resp.prompt_tokens,
"completion_tokens": resp.completion_tokens,
"latency_ms": resp.latency_ms,
@@ -409,7 +433,6 @@ async def _predict_single(
"match_kickoff_at": match_kickoff_at,
"prediction_cutoff_at": prediction_cutoff_at,
"prediction_created_at": now,
"input_hash": input_hash,
},
)