shangfangjian
11da4d3631
R1 429 key 轮换路径调用不存在的 _km → NameError(且位于凭证脱敏日志行): 改用 src/data/key_ring._mask,全树不再有 _km 调用。 [注: 该改动已随另一工作流提交9ccda4b一并入库] R2 ingest_bzzoiro_standings 被截断(return 前无 upsert 逻辑,standings 表 永不写入、total_upserted 恒为 0):从f05dc1a移植完整实现,按 (league_id, season, team_id) upsert,保留逐联赛错误隔离,不加 db.commit()。 [注: 同上,已随9ccda4b入库] R3 src/llm/agents/orchestrator.py 完成日志把 list 喂给 %d → logging TypeError: 改为 len(ok_reports),与同文件降级日志行写法一致。 R4 mode=multi 静默丢弃调用方传入的 model:predict_match_multi 新增 model 关键字参数,经 _ACTIVE_MODEL_OVERRIDE 下传至 _agent_provider, 显式 model 优先级最高;override 生效时跳过 provider 缓存读写以免串味, 并在 predict.py 派发点透传。 R5 回测把字符串日期直接与 timestamptz 列比较:新增 _parse_date_bound 助手, 支持 YYYY-MM-DD / 完整 ISO / datetime / None,裸日期按 UTC 锚定, 结束日取当天末刻(闭区间,避免最后一天被静默排除),非法输入抛 ValueError。 新增 tests/test_review_required_fixes.py 覆盖 R1-R5(R2/R4 为行为测试), 20 项全通过。
328 lines
12 KiB
Python
328 lines
12 KiB
Python
"""预测服务:拼上下文 → 调 LLM → 存预测。"""
|
|
from __future__ import annotations
|
|
|
|
import functools
|
|
import hashlib
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from src.core.config import settings
|
|
from sqlalchemy import select
|
|
|
|
from src.db.base import AsyncSessionLocal
|
|
from src.db.models import Match, Prediction
|
|
from src.db.unit_of_work import get_uow
|
|
from src.llm.context_builder import build_context
|
|
from src.llm.provider import LLMProvider, get_default_provider
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_PROMPT_DIR = Path(__file__).resolve().parent / "prompts"
|
|
|
|
# ── LLM 响应缓存(match+provider+model+version → 结果) ──
|
|
_CACHE_TTL_SEC = 300 # 5 分钟
|
|
_CACHE_MAX_SIZE = 200 # P3-1: 有上限,避免长期运行内存无限增长
|
|
# P1-5: 缓存仅在 asyncio 协程内同步访问(dict 操作 GIL 原子),无需 threading.Lock。
|
|
# 删除 _cache_lock,避免同步锁阻塞事件循环;dict 的 get/set 在 CPython 下原子。
|
|
_cache: dict[str, tuple[float, PredictResult]] = {}
|
|
|
|
|
|
def _cache_key(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> str:
|
|
"""缓存键:含 prompt 模板内容 hash。
|
|
|
|
仅用 version 做键不够 —— 编辑器里改动 `match_prediction_v1.md` 而版本号
|
|
不变时,进程内缓存仍会返回旧模板产生的旧结果(见审查报告 P2-6)。
|
|
把模板内容 hash 纳入键,模板一改缓存自动失效。
|
|
"""
|
|
return f"{match_id}:{provider}:{model}:{version}:{tpl_hash[:12]}"
|
|
|
|
|
|
def _get_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> PredictResult | None:
|
|
# P1-5: 无锁访问。dict get/del 在 CPython GIL 下原子,且无 await 穿插。
|
|
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
|
entry = _cache.get(key)
|
|
if entry is not None:
|
|
ts, result = entry
|
|
if time.time() - ts < _CACHE_TTL_SEC:
|
|
return result
|
|
_cache.pop(key, None)
|
|
return None
|
|
|
|
|
|
def _set_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str, result: PredictResult) -> None:
|
|
# P1-5: 无锁写入。同上,dict set 原子。
|
|
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
|
_cache[key] = (time.time(), result)
|
|
# P3-1: 超过上限时淘汰最旧条目(按时间戳排序)
|
|
if len(_cache) > _CACHE_MAX_SIZE:
|
|
oldest_key = min(_cache, key=lambda k: _cache[k][0])
|
|
_cache.pop(oldest_key, None)
|
|
|
|
|
|
def clear_prompt_cache() -> None:
|
|
"""清空 prompt 模板缓存(供开发/热更新时手动调用)。
|
|
|
|
lru_cache 的模板缓存是进程级的,改完 .md 需要重启进程才能生效;
|
|
提供显式清理入口,避免"改了模板却看不到变化"的困惑(见审查报告 P2-5)。
|
|
"""
|
|
_load_prompt_template.cache_clear()
|
|
logger.info("prompt 模板缓存已清空")
|
|
|
|
|
|
@functools.lru_cache(maxsize=8)
|
|
def _load_prompt_template(version: str = "v1") -> str:
|
|
"""缓存 prompt 模板(进程生命周期内每个版本只读一次)。"""
|
|
path = _PROMPT_DIR / f"match_prediction_{version}.md"
|
|
if not path.exists():
|
|
raise FileNotFoundError(f"prompt 模板不存在: {path}")
|
|
with open(path, encoding="utf-8") as f:
|
|
return f.read()
|
|
|
|
|
|
def _prompt_template_hash(version: str) -> str:
|
|
"""prompt 模板内容 hash(用于缓存键,模板变更即失效)。"""
|
|
return hashlib.sha256(_load_prompt_template(version).encode("utf-8")).hexdigest()
|
|
|
|
|
|
@dataclass
|
|
class PredictResult:
|
|
prediction_id: int
|
|
provider: str
|
|
model: str
|
|
prompt_version: str
|
|
pred_home_goals: float | None
|
|
pred_away_goals: float | None
|
|
alt_pred_home_goals: int | None
|
|
alt_pred_away_goals: int | None
|
|
pred_1x2: str | None
|
|
subjective_confidence: float | None
|
|
reasoning: str | None
|
|
context: str
|
|
status: str = "success"
|
|
latency_ms: int | None = None
|
|
raw: dict | None = None
|
|
|
|
|
|
async def _upsert_prediction(
|
|
session,
|
|
*,
|
|
match_id: int,
|
|
provider_name: str,
|
|
model: str,
|
|
mode: str,
|
|
run_type: str,
|
|
values: dict,
|
|
) -> Prediction:
|
|
"""按 (match, provider, model, mode, run_type) 唯一约束写入预测。
|
|
|
|
已存在且未结算 → 覆盖更新(重新预测语义);已结算 → 拒绝(保护评估数据)。
|
|
run_type 区分 live/backtest,避免回测覆盖实盘预测。
|
|
"""
|
|
existing = (
|
|
await session.execute(
|
|
select(Prediction).where(
|
|
Prediction.match_id == match_id,
|
|
Prediction.provider == provider_name,
|
|
Prediction.model == model,
|
|
Prediction.mode == mode,
|
|
Prediction.run_type == run_type,
|
|
)
|
|
)
|
|
).scalar_one_or_none()
|
|
if existing is not None and existing.settled:
|
|
raise ValueError("该比赛已有已结算的预测,不能重新预测")
|
|
|
|
pred = existing if existing is not None else Prediction(
|
|
match_id=match_id, provider=provider_name, model=model,
|
|
)
|
|
pred.mode = mode
|
|
pred.run_type = run_type
|
|
for k, v in values.items():
|
|
setattr(pred, k, v)
|
|
if existing is None:
|
|
session.add(pred)
|
|
await session.flush() # 拿到自增 id;事务由 UnitOfWork 退出时提交
|
|
return pred
|
|
|
|
|
|
async def predict_match(
|
|
match_id: int,
|
|
*,
|
|
provider: LLMProvider | None = None,
|
|
model: str | None = None,
|
|
prompt_version: str | None = None,
|
|
mode: str = "multi",
|
|
use_cache: bool = True,
|
|
backtest: bool = False,
|
|
cutoff_at=None,
|
|
) -> "PredictResult | MultiPredictResult":
|
|
"""预测入口。mode=multi(默认)走多 agent;mode=single 走单次调用;mode=baseline 走无 LLM 基线。
|
|
|
|
Args:
|
|
mode: multi(默认,5 专家+终裁) / single(单次) / baseline(极简统计基线,不调用 LLM)。
|
|
use_cache:是否允许返回进程内缓存结果。回测必须传 False——
|
|
缓存命中不会新建 prediction 行,调用方会对同一个 prediction_id
|
|
反复 settle,把不同比赛的真实比分覆盖到同一条记录上。
|
|
backtest:是否回测模式。True 时 cutoff 自动设为 match_date-1天。
|
|
cutoff_at:显式截止时间,优先级高于 backtest 自动计算。
|
|
"""
|
|
if mode == "baseline":
|
|
from src.llm.baseline import predict_baseline
|
|
|
|
return await predict_baseline(
|
|
match_id, backtest=backtest, cutoff_at=cutoff_at,
|
|
)
|
|
if mode == "single":
|
|
return await _predict_single(
|
|
match_id,
|
|
provider=provider,
|
|
model=model,
|
|
prompt_version=prompt_version,
|
|
use_cache=use_cache,
|
|
backtest=backtest,
|
|
cutoff_at=cutoff_at,
|
|
)
|
|
from src.llm.agents.orchestrator import predict_match_multi
|
|
|
|
# 回测参数 + 模型覆盖完整传递到 multi-agent 路径
|
|
return await predict_match_multi(
|
|
match_id,
|
|
provider=provider,
|
|
version=(prompt_version or "v1").removeprefix("multi_"),
|
|
backtest=backtest,
|
|
cutoff_at=cutoff_at,
|
|
model=model,
|
|
)
|
|
|
|
|
|
async def _predict_single(
|
|
match_id: int,
|
|
*,
|
|
provider: LLMProvider | None = None,
|
|
model: str | None = None,
|
|
prompt_version: str | None = None,
|
|
use_cache: bool = True,
|
|
backtest: bool = False,
|
|
cutoff_at=None,
|
|
) -> PredictResult:
|
|
"""单次调用路径(原有实现)。"""
|
|
if provider is None:
|
|
provider = await get_default_provider()
|
|
if model:
|
|
provider.model = model
|
|
version = prompt_version or "v1"
|
|
tpl_hash = _prompt_template_hash(version)
|
|
|
|
# 0. 查缓存(同 match+provider+model+version+模板hash 5 分钟内直接返)
|
|
if use_cache:
|
|
cached = _get_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash)
|
|
if cached is not None:
|
|
logger.debug("predict cache hit match=%s", match_id)
|
|
return cached
|
|
|
|
# 1. 拼上下文(backtest/cutoff 防泄漏)
|
|
ctx = await build_context(match_id, backtest=backtest, cutoff_at=cutoff_at)
|
|
|
|
# 1.5 计算快照元数据(用于可复现性)
|
|
now = datetime.now(timezone.utc)
|
|
match_kickoff_at = ctx.match_dt
|
|
# 使用上下文实际计算的 cutoff(回测时可能为 match_dt-1天),而非开球时间
|
|
prediction_cutoff_at = ctx.cutoff if ctx.cutoff is not None else ctx.match_dt
|
|
input_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
|
|
|
|
# 2. 拼 prompt(指定版本)
|
|
template = _load_prompt_template(version)
|
|
user_prompt = template.replace("{{context}}", ctx.text)
|
|
|
|
# 3. 调 LLM
|
|
resp = await provider.chat(
|
|
system="你是一个严谨的足球预测专家。只输出 JSON。",
|
|
user=user_prompt,
|
|
json_mode=True,
|
|
temperature=0.3,
|
|
max_tokens=4096, # 推理模型的 reasoning 也计入输出 token,需留足余量
|
|
)
|
|
|
|
if resp.error:
|
|
raise RuntimeError(f"LLM error: {resp.error}")
|
|
|
|
# P0-3: json_mode 下 parsed 为 None 说明 JSON 解析失败,不能 fallback 到 {}
|
|
if resp.parsed is None:
|
|
raise RuntimeError("LLM 输出 JSON 解析失败,parsed=None")
|
|
|
|
parsed = resp.parsed
|
|
|
|
# 3.5 严格校验 LLM 输出
|
|
from src.llm.validation import validate_prediction_output
|
|
try:
|
|
validated = validate_prediction_output(parsed)
|
|
except Exception as e:
|
|
raise RuntimeError(f"LLM 输出校验失败: {e}")
|
|
|
|
# 4. 存预测(使用 UnitOfWork 统一事务)
|
|
async with get_uow() as session:
|
|
# 验证 match 存在
|
|
m = await session.get(Match, match_id)
|
|
if m is None:
|
|
raise ValueError(f"match {match_id} not found")
|
|
|
|
pred = await _upsert_prediction(
|
|
session,
|
|
match_id=match_id,
|
|
provider_name=settings.LLM_PROVIDER,
|
|
model=provider.model,
|
|
mode="single",
|
|
run_type="backtest" if backtest else "live",
|
|
values={
|
|
"prompt_version": version,
|
|
"prompt_tokens": resp.prompt_tokens,
|
|
"completion_tokens": resp.completion_tokens,
|
|
"latency_ms": resp.latency_ms,
|
|
"pred_home_goals": validated.pred_home_goals,
|
|
"pred_away_goals": validated.pred_away_goals,
|
|
"alt_pred_home_goals": validated.alt_pred_home_goals,
|
|
"alt_pred_away_goals": validated.alt_pred_away_goals,
|
|
"pred_1x2": validated.pred_1x2,
|
|
"subjective_confidence": validated.subjective_confidence,
|
|
"reasoning": validated.reasoning,
|
|
"raw_response": resp.raw,
|
|
"status": "success",
|
|
"match_kickoff_at": match_kickoff_at,
|
|
"prediction_cutoff_at": prediction_cutoff_at,
|
|
"prediction_created_at": now,
|
|
"input_hash": input_hash,
|
|
},
|
|
)
|
|
|
|
result = PredictResult(
|
|
prediction_id=pred.id,
|
|
provider=pred.provider,
|
|
model=pred.model,
|
|
prompt_version=version,
|
|
pred_home_goals=pred.pred_home_goals,
|
|
pred_away_goals=pred.pred_away_goals,
|
|
alt_pred_home_goals=pred.alt_pred_home_goals,
|
|
alt_pred_away_goals=pred.alt_pred_away_goals,
|
|
pred_1x2=pred.pred_1x2,
|
|
subjective_confidence=pred.subjective_confidence,
|
|
reasoning=pred.reasoning,
|
|
status=pred.status,
|
|
context=ctx.text,
|
|
latency_ms=resp.latency_ms,
|
|
raw=resp.raw,
|
|
)
|
|
|
|
# 5. 写入缓存(仅当允许缓存时)
|
|
if use_cache:
|
|
_set_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash, result)
|
|
logger.info(
|
|
"预测完成 match=%s mode=%s status=%s pred=%s:%s (%s) latency=%sms",
|
|
match_id, "single", "success",
|
|
validated.pred_home_goals, validated.pred_away_goals, validated.pred_1x2,
|
|
resp.latency_ms,
|
|
)
|
|
return result
|