安全加固 + 数据管线修复 + 质量改进(第2批)
安全加固: - /api/v1/predict 限流改用 get_client_ip 防 X-Forwarded-For 伪造 - 新增 TRUST_PROXY_HEADERS/REQUIRE_ADMIN_AUTH 配置,默认 fail-closed - 生产环境管理接口未配置鉴权时拒绝(503),不再放行 数据管线修复: - injuries IntegrityError 改用 begin_nested(SAVEPOINT)隔离批次 - injuries 空名单 vs 未配置语义区分(has_data 精确标记) - bzzoiro 统计字段映射(shots/possession/cards) + 入库条件放宽 - available_at 语义收紧(回测防泄漏) - team_names NFKD 去变音双重查找 + 补齐变体键 预测系统改进: - multi-agent 全专家失败时 status=degraded 跳过终裁 - agent_weights 独立持久化到 predictions 表 新增迁移: - 0014_predictions_agent_weights.py 新增测试(8个文件): - test_ip_spoofing.py: 限流防伪造 - test_require_admin_fail_closed.py: 生产 fail-closed - test_injuries_integrity_rollback.py: SAVEPOINT 隔离 - test_injuries_slice_semantic.py: 空名单 vs 未配置 - test_available_at.py: 回测防泄漏 - test_bzzoirot_stats.py: 统计字段映射 - test_team_names_normalize.py: NFKD 变体 - test_multi_agent_degraded.py: 全失败 degraded - test_agent_weights_persist.py: 权重持久化
This commit is contained in:
+39
-6
@@ -76,6 +76,17 @@ async def require_admin(
|
||||
用法: `@router.get("/leagues", dependencies=[Depends(require_admin)])`
|
||||
"""
|
||||
if not await auth_configured():
|
||||
# 生产环境 fail-closed:未配置鉴权则拒绝,不放行
|
||||
if settings.REQUIRE_ADMIN_AUTH or settings.APP_ENV == "production":
|
||||
logger.error(
|
||||
"生产环境管理接口未配置鉴权(REQUIRE_ADMIN_AUTH=True 或 APP_ENV=production),"
|
||||
"拒绝访问。请设置 ADMIN_PASSWORD 或 ADMIN_API_KEY。"
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="服务未配置管理鉴权,请联系管理员",
|
||||
)
|
||||
# 开发环境 fail-open + warning
|
||||
logger.warning(
|
||||
"管理员密码 / ADMIN_API_KEY 均未设置,管理接口当前【无鉴权】。"
|
||||
"生产环境请至少设置其中一项。"
|
||||
@@ -98,6 +109,31 @@ async def require_admin(
|
||||
raise HTTPException(status_code=401, detail="未登录或凭证无效")
|
||||
|
||||
|
||||
# ── 客户端 IP 提取(防 X-Forwarded-For 伪造) ──
|
||||
|
||||
def get_client_ip(request: Request) -> str:
|
||||
"""获取客户端真实 IP,防 X-Forwarded-For 伪造。
|
||||
|
||||
规则:
|
||||
- TRUST_PROXY_HEADERS=False(默认):只用 request.client.host,
|
||||
忽略 X-Forwarded-For,防止客户端伪造。
|
||||
- TRUST_PROXY_HEADERS=True:解析 X-Forwarded-For 第一个 IP,
|
||||
适用于 Nginx 等可信反代后方。
|
||||
|
||||
部署建议:
|
||||
- 公网必须设 TRUST_PROXY_HEADERS=True,并在 Nginx 配置:
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
- Nginx 层也建议做限流(limit_req),作为第二道防线。
|
||||
"""
|
||||
if settings.TRUST_PROXY_HEADERS:
|
||||
# 信任反代:X-Forwarded-For 可能包含多个 IP(代理链),取第一个
|
||||
forwarded = request.headers.get("X-Forwarded-For")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()
|
||||
# 默认或无反代头:直接用连接层 IP
|
||||
return request.client.host if request.client else "unknown"
|
||||
|
||||
|
||||
# ── 简易内存限流(按 IP,无外部依赖) ──
|
||||
|
||||
class _RateLimiter:
|
||||
@@ -139,13 +175,10 @@ _predict_limiter = _RateLimiter(max_requests=10, window_seconds=60)
|
||||
async def rate_limit_predict(request: Request) -> None:
|
||||
"""POST /api/v1/predict 限流依赖。
|
||||
|
||||
基于客户端 IP(考虑 X-Forwarded-For),超过 10 次/分钟返回 429。
|
||||
基于客户端 IP,超过 10 次/分钟返回 429。
|
||||
IP 提取逻辑:优先用 get_client_ip(防伪造)。
|
||||
"""
|
||||
# 获取客户端 IP(支持反向代理)
|
||||
client_ip = request.headers.get("X-Forwarded-For", request.client.host if request.client else "unknown")
|
||||
# X-Forwarded-For 可能包含多个 IP(代理链),取第一个
|
||||
if "," in client_ip:
|
||||
client_ip = client_ip.split(",")[0].strip()
|
||||
client_ip = get_client_ip(request)
|
||||
|
||||
if not _predict_limiter.is_allowed(client_ip):
|
||||
logger.warning("rate limit exceeded for %s", client_ip)
|
||||
|
||||
@@ -54,7 +54,9 @@ class PasswordChangeIn(BaseModel):
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
return request.client.host if request.client else "unknown"
|
||||
"""获取客户端 IP,与限流共用同一套逻辑(防伪造)。"""
|
||||
from src.api.deps import get_client_ip
|
||||
return get_client_ip(request)
|
||||
|
||||
|
||||
def _is_locked(ip: str) -> bool:
|
||||
|
||||
@@ -11,6 +11,9 @@ class Settings(BaseSettings):
|
||||
# --- app ---
|
||||
APP_ENV: str = "development"
|
||||
LOG_LEVEL: str = "INFO"
|
||||
# 生产环境强制要求管理鉴权配置,即使 APP_ENV=production 也生效。
|
||||
# True 时若 auth_configured() 为 False 则拒绝(503),development 保持 fail-open。
|
||||
REQUIRE_ADMIN_AUTH: bool = False
|
||||
|
||||
# --- database ---
|
||||
DATABASE_URL: str = "postgresql+asyncpg://football:football@localhost:5432/football"
|
||||
@@ -30,6 +33,12 @@ class Settings(BaseSettings):
|
||||
BZZOIRO_BASE: str = "https://sports.bzzoiro.com/api/v2"
|
||||
API_FOOTBALL_KEY: str = ""
|
||||
|
||||
# --- 代理头信任 ---
|
||||
# 为 True 时才解析 X-Forwarded-For,否则只用 request.client.host。
|
||||
# 公网部署时应设为 True,并确保仅 Nginx 等可信反代能访问 API,
|
||||
# 且 Nginx 层已覆盖真实 IP(X-Real-IP / proxy_protocol)。
|
||||
TRUST_PROXY_HEADERS: bool = False
|
||||
|
||||
# --- CORS ---
|
||||
CORS_ORIGINS: str = "http://localhost:5173,http://localhost:3000"
|
||||
CORS_METHODS: str = "GET,POST,PUT,DELETE,OPTIONS"
|
||||
|
||||
+17
-4
@@ -266,8 +266,12 @@ class BzzoiroSource:
|
||||
db.add(m)
|
||||
await db.flush()
|
||||
existing_matches[match_key] = m # 防止同批重复
|
||||
if nm.home_xg is not None or nm.away_xg is not None:
|
||||
# 存在任一统计字段即可创建 MatchStats(不再强制要求 xG)
|
||||
if any(getattr(nm, f) is not None for f in ['home_xg', 'away_xg', 'home_shots', 'away_shots', 'home_shots_on_target', 'away_shots_on_target', 'home_corners', 'away_corners', 'home_possession', 'home_yellow_cards', 'away_yellow_cards', 'home_red_cards', 'away_red_cards']):
|
||||
now = datetime.now(timezone.utc)
|
||||
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
|
||||
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
|
||||
available_at = nm.date if nm.date else now
|
||||
stats = MatchStats(
|
||||
match_id=m.id,
|
||||
home_xg=nm.home_xg,
|
||||
@@ -286,7 +290,7 @@ class BzzoiroSource:
|
||||
source="bzzoiro",
|
||||
source_record_id=str(raw.get("id", "")),
|
||||
retrieved_at=now,
|
||||
available_at=now,
|
||||
available_at=available_at,
|
||||
)
|
||||
db.add(stats)
|
||||
league_r["inserted"] += 1
|
||||
@@ -305,14 +309,23 @@ class BzzoiroSource:
|
||||
if existing_match.match_stage is None and nm.match_stage:
|
||||
existing_match.match_stage = nm.match_stage
|
||||
changed = True
|
||||
if existing_match.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
# 存在任一统计字段即可创建 MatchStats(不再强制要求 xG)
|
||||
if existing_match.stats is None and (
|
||||
nm.home_xg is not None or nm.away_xg is not None
|
||||
or nm.home_shots is not None or nm.away_shots is not None
|
||||
or nm.home_corners is not None or nm.away_corners is not None
|
||||
or nm.home_possession is not None
|
||||
):
|
||||
now = datetime.now(timezone.utc)
|
||||
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
|
||||
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
|
||||
available_at = nm.date if nm.date else now
|
||||
existing_match.stats = MatchStats(
|
||||
match_id=existing_match.id,
|
||||
source="bzzoiro",
|
||||
source_record_id=str(raw.get("id", "")),
|
||||
retrieved_at=now,
|
||||
available_at=now,
|
||||
available_at=available_at,
|
||||
)
|
||||
db.add(existing_match.stats)
|
||||
await db.flush()
|
||||
|
||||
+82
-36
@@ -12,11 +12,25 @@ import tempfile
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from src.core.runtime_config import get_runtime_value
|
||||
|
||||
|
||||
@dataclass
|
||||
class InjuryQueryResult:
|
||||
"""伤停查询结果(区分「查询成功但为空」与「查询失败/源未配置」)。"""
|
||||
|
||||
records: list["Injury"]
|
||||
query_status: str # "success" | "source_not_configured" | "query_error"
|
||||
|
||||
@property
|
||||
def has_data(self) -> bool:
|
||||
"""成功查询(即使结果为空)视为有明确名单,has_data=True。"""
|
||||
return self.query_status == "success"
|
||||
from src.core.http_client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -213,42 +227,59 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
rows = (await db.execute(stmt)).all()
|
||||
existing_keys = {(r[0], r[1], r[2]) for r in rows}
|
||||
|
||||
# Fix 1: 使用 SAVEPOINT(begin_nested)避免整批回滚丢数据
|
||||
# 每个 batch 使用独立的 savepoint,失败时只回滚该 batch
|
||||
# Fix 1: 使用 begin_nested(SAVEPOINT)隔离每批 flush
|
||||
# IntegrityError 时只回滚到 savepoint,不影响其它已成功批次
|
||||
BATCH_SIZE = 50
|
||||
batch: list[Injury] = []
|
||||
|
||||
async def _flush_batch():
|
||||
"""使用 savepoint flush 一批记录;失败只回滚本批。"""
|
||||
if not batch:
|
||||
return
|
||||
async with db.begin_nested():
|
||||
for obj in batch:
|
||||
db.add(obj)
|
||||
await db.flush()
|
||||
batch.clear()
|
||||
|
||||
for i, rec in enumerate(pending_records):
|
||||
key = (rec["player_id"], rec["fixture_id"], rec["injury_type"])
|
||||
if key in existing_keys:
|
||||
continue
|
||||
|
||||
injury = Injury(**rec)
|
||||
db.add(injury)
|
||||
batch.append(Injury(**rec))
|
||||
result["inserted"] += 1
|
||||
|
||||
# 每 BATCH_SIZE 条 flush 一次,使用 SAVEPOINT 隔离
|
||||
if result["inserted"] % BATCH_SIZE == 0:
|
||||
# 每 BATCH_SIZE 条 flush 一次
|
||||
if len(batch) >= BATCH_SIZE:
|
||||
try:
|
||||
await db.flush()
|
||||
await _flush_batch()
|
||||
except IntegrityError:
|
||||
# 只回滚到上一个 savepoint,不影响已提交的数据
|
||||
await db.rollback()
|
||||
logger.warning("injuries batch IntegrityError at record %d, continuing", i + 1)
|
||||
# 从当前位置继续处理剩余记录
|
||||
logger.warning(
|
||||
"injuries batch IntegrityError at record %d, "
|
||||
"rolled back to savepoint, continuing",
|
||||
i + 1,
|
||||
)
|
||||
# begin_nested 已回滚到 savepoint,清空 batch 继续
|
||||
batch.clear()
|
||||
continue
|
||||
|
||||
# 最终 flush(剩余不足一批的记录)
|
||||
try:
|
||||
await db.flush()
|
||||
await _flush_batch()
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
logger.warning("injuries final flush IntegrityError, some records may be lost")
|
||||
logger.warning(
|
||||
"injuries final flush IntegrityError, "
|
||||
"rolled back to savepoint, some records may be lost",
|
||||
)
|
||||
batch.clear()
|
||||
|
||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
|
||||
return result
|
||||
|
||||
|
||||
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> list[Injury]:
|
||||
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> InjuryQueryResult:
|
||||
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。
|
||||
|
||||
Args:
|
||||
@@ -258,31 +289,46 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> li
|
||||
as_of: 数据截止时间(用于回测防泄漏)
|
||||
|
||||
Returns:
|
||||
伤停记录列表
|
||||
InjuryQueryResult:包含查询记录与状态
|
||||
- query_status="success": 查询成功(即使结果也为空)
|
||||
- query_status="source_not_configured": API_FOOTBALL_KEY 未配置
|
||||
- query_status="query_error": 查询异常
|
||||
|
||||
语义区分:
|
||||
- 成功查询 + 空结果 → has_data=True(明确知道「无人伤停」)
|
||||
- 源未配置 / 查询失败 → has_data=False(无法判断,跳过 LLM)
|
||||
"""
|
||||
from sqlalchemy import select, func
|
||||
from src.db.models import Injury
|
||||
|
||||
# Fix 3: 统一用 timezone-aware datetime 比较,禁止 date() 截断
|
||||
# retrieved_at 是 timestamptz,as_of 也应该是 datetime
|
||||
# 比较时统一转为 date 避免时间部分导致当天数据不可见
|
||||
if hasattr(match_date, "date") and callable(match_date.date):
|
||||
match_date = match_date.date()
|
||||
# 检查 API 是否配置(只读配置,不发网络)
|
||||
api_key = await get_runtime_value("API_FOOTBALL_KEY")
|
||||
if not api_key:
|
||||
logger.debug("API_FOOTBALL_KEY 未配置,跳过伤停查询 team=%s", team_id)
|
||||
return InjuryQueryResult(records=[], query_status="source_not_configured")
|
||||
|
||||
stmt = (
|
||||
select(Injury)
|
||||
.where(Injury.team_id == team_id)
|
||||
.where(Injury.injury_date <= match_date)
|
||||
.where(
|
||||
(Injury.return_date.is_(None)) | (Injury.return_date >= match_date)
|
||||
try:
|
||||
# Fix 3: 统一用 timezone-aware datetime 比较,禁止 date() 截断
|
||||
if hasattr(match_date, "date") and callable(match_date.date):
|
||||
match_date = match_date.date()
|
||||
|
||||
stmt = (
|
||||
select(Injury)
|
||||
.where(Injury.team_id == team_id)
|
||||
.where(Injury.injury_date <= match_date)
|
||||
.where(
|
||||
(Injury.return_date.is_(None)) | (Injury.return_date >= match_date)
|
||||
)
|
||||
)
|
||||
)
|
||||
if as_of is not None:
|
||||
# Fix 3: 统一用 date 比较,避免 timestamptz vs date 的时区问题
|
||||
if hasattr(as_of, "date") and callable(as_of.date):
|
||||
as_of = as_of.date()
|
||||
# 使用 func.date() 将 timestamptz 转为 date,确保当天白天采到的数据对当晚比赛可见
|
||||
stmt = stmt.where(func.date(Injury.retrieved_at) <= as_of)
|
||||
if as_of is not None:
|
||||
# Fix 3: 统一用 date 比较,避免 timestamptz vs date 的时区问题
|
||||
if hasattr(as_of, "date") and callable(as_of.date):
|
||||
as_of = as_of.date()
|
||||
stmt = stmt.where(func.date(Injury.retrieved_at) <= as_of)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
result = await db.execute(stmt)
|
||||
records = list(result.scalars().all())
|
||||
return InjuryQueryResult(records=records, query_status="success")
|
||||
except Exception as e:
|
||||
logger.exception("伤停查询异常 team=%s: %s", team_id, e)
|
||||
return InjuryQueryResult(records=[], query_status="query_error")
|
||||
|
||||
@@ -177,6 +177,29 @@ def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
|
||||
m.match_stage = _rn_name
|
||||
elif _rn:
|
||||
m.match_stage = f"第 {_rn} 轮"
|
||||
|
||||
# 统计字段映射(API 字段名 → NormalizedMatch)
|
||||
# API 可能提供的字段:home_shots/away_shots, shots_on_target, corners, possession, xg, cards
|
||||
# API 没有的字段保持 None,不伪造
|
||||
m.home_shots = _to_int(raw.get("home_shots", raw.get("shots_home")))
|
||||
m.away_shots = _to_int(raw.get("away_shots", raw.get("shots_away")))
|
||||
m.home_shots_on_target = _to_int(raw.get("home_shots_on_target", raw.get("sot_home")))
|
||||
m.away_shots_on_target = _to_int(raw.get("away_shots_on_target", raw.get("sot_away")))
|
||||
m.home_corners = _to_int(raw.get("home_corners", raw.get("corners_home")))
|
||||
m.away_corners = _to_int(raw.get("away_corners", raw.get("corners_away")))
|
||||
# 控球率:API 通常只给 home 值,away = 100 - home
|
||||
possession_home = _to_float(raw.get("home_possession", raw.get("possession")))
|
||||
if possession_home is not None:
|
||||
m.home_possession = possession_home
|
||||
# xG
|
||||
m.home_xg = _to_float(raw.get("home_xg", raw.get("xg_home", raw.get("expected_goals_home"))))
|
||||
m.away_xg = _to_float(raw.get("away_xg", raw.get("xg_away", raw.get("expected_goals_away"))))
|
||||
# 牌
|
||||
m.home_yellow_cards = _to_int(raw.get("home_yellow_cards", raw.get("yellow_cards_home")))
|
||||
m.away_yellow_cards = _to_int(raw.get("away_yellow_cards", raw.get("yellow_cards_away")))
|
||||
m.home_red_cards = _to_int(raw.get("home_red_cards", raw.get("red_cards_home")))
|
||||
m.away_red_cards = _to_int(raw.get("away_red_cards", raw.get("red_cards_away")))
|
||||
|
||||
if m.match_status == "finished" and m.home_goals is None:
|
||||
m.match_status = "scheduled"
|
||||
return m
|
||||
|
||||
+28
-2
@@ -1,6 +1,9 @@
|
||||
"""队名归一化:各源队名 → 统一规范名。
|
||||
|
||||
迁移自旧项目 app/data/team_names.py。
|
||||
|
||||
NFKD 归一化会剥离变音符号(ü→u),导致「Bayern München」与「Bayern Munich」
|
||||
映射到不同规范名。修复:双重查找(原始名 + NFKD 名) + 补齐常见变体键。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -61,10 +64,15 @@ NORMALIZE_MAP = {
|
||||
"Barcelona": "Barcelona",
|
||||
# ---- 德甲 ----
|
||||
"Bayern Munich": "Bayern München",
|
||||
"Bayern Munchen": "Bayern München", # NFKD stripped variant
|
||||
"Bayern München": "Bayern München", # canonical with umlaut (direct hit)
|
||||
"FC Koln": "FC Köln",
|
||||
"FC Köln": "FC Köln",
|
||||
"RB Leipzig": "RB Leipzig",
|
||||
"Borussia Dortmund": "Borussia Dortmund",
|
||||
"Borussia M'gladbach": "Borussia Mönchengladbach",
|
||||
"Borussia Monchengladbach": "Borussia Mönchengladbach", # NFKD stripped
|
||||
"Borussia Mönchengladbach": "Borussia Mönchengladbach", # canonical
|
||||
"Bayer Leverkusen": "Bayer Leverkusen",
|
||||
"Eintracht Frankfurt": "Eintracht Frankfurt",
|
||||
"VfB Stuttgart": "VfB Stuttgart",
|
||||
@@ -121,6 +129,9 @@ NORMALIZE_MAP = {
|
||||
"Le Havre": "Le Havre AC",
|
||||
"Lorient": "FC Lorient",
|
||||
"Saint-Etienne": "AS Saint-Étienne",
|
||||
"Saint-Etienne": "AS Saint-Étienne", # NFKD stripped (ê → e)
|
||||
"AS Saint-Étienne": "AS Saint-Étienne", # canonical prefix
|
||||
"AS Saint-Etienne": "AS Saint-Étienne", # NFKD stripped with prefix
|
||||
"Angers": "Angers SCO",
|
||||
"Auxerre": "AJ Auxerre",
|
||||
"Leganes": "Leganés",
|
||||
@@ -128,10 +139,25 @@ NORMALIZE_MAP = {
|
||||
|
||||
|
||||
def normalize(name: str) -> str:
|
||||
"""队名归一化:变音符号变体 → 统一规范名。
|
||||
|
||||
双重查找策略:
|
||||
1. 原始名(带变音)直接查表
|
||||
2. NFKD 去变音后再查表
|
||||
|
||||
确保「Bayern München」「Bayern Munchen」「Bayern Munich」
|
||||
都映射到同一规范名「Bayern München」。
|
||||
"""
|
||||
if not name:
|
||||
return ""
|
||||
# unicode 归一(重音)
|
||||
n = unicodedata.normalize("NFKD", name)
|
||||
|
||||
# 1. 原始名直接查表(保留变音变体)
|
||||
stripped = name.strip()
|
||||
if stripped in NORMALIZE_MAP:
|
||||
return NORMALIZE_MAP[stripped]
|
||||
|
||||
# 2. NFKD 去变音后查表
|
||||
n = unicodedata.normalize("NFKD", stripped)
|
||||
n = "".join(c for c in n if not unicodedata.combining(c))
|
||||
n = n.strip()
|
||||
return NORMALIZE_MAP.get(n, n)
|
||||
|
||||
@@ -199,12 +199,16 @@ class UnderstatSource:
|
||||
# 回填 xG
|
||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
now = datetime.now(timezone.utc)
|
||||
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
|
||||
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
|
||||
match_date = existing.match_date if existing.match_date else now
|
||||
available_at = match_date
|
||||
existing.stats = MatchStats(
|
||||
match_id=existing.id,
|
||||
source="understat",
|
||||
source_record_id=str(raw.get("id", "")),
|
||||
retrieved_at=now,
|
||||
available_at=now,
|
||||
available_at=available_at,
|
||||
)
|
||||
db.add(existing.stats)
|
||||
await db.flush()
|
||||
|
||||
@@ -205,6 +205,8 @@ class Prediction(Base):
|
||||
# multi-agent 模式: 各专家报告
|
||||
mode: Mapped[str] = mapped_column(String(20), nullable=False, default="single")
|
||||
agent_outputs: Mapped[dict | None] = mapped_column(JSONB)
|
||||
# Fix: agent_weights 独立持久化到列(原本只在 raw_response 中)
|
||||
agent_weights: Mapped[dict | None] = mapped_column(JSONB)
|
||||
# 预测状态: success / failed / degraded
|
||||
status: Mapped[str] = mapped_column(String(20), nullable=False, default="success")
|
||||
# Fix: run_type 区分实盘(live)与回测(backtest),避免回测覆盖实盘预测
|
||||
|
||||
@@ -243,11 +243,35 @@ async def predict_match_multi(
|
||||
# 2. 并行专家(各自独立配置,使用统一 cutoff)
|
||||
reports = await run_specialists(header, version=version, before=cutoff)
|
||||
|
||||
# 3. 终裁
|
||||
aggregator_provider = await _agent_provider("aggregator", tier="aggregator")
|
||||
final, agg_prompt_tokens, agg_completion_tokens = await run_aggregator(
|
||||
header, reports, provider=aggregator_provider, version=version
|
||||
)
|
||||
# 2.5 统计有效专家报告数量
|
||||
ok_reports = [r for r in reports if r.status == "ok"]
|
||||
has_valid_data = len(ok_reports) > 0
|
||||
|
||||
# 3. 终裁(仅当有有效专家报告时执行)
|
||||
if has_valid_data:
|
||||
aggregator_provider = await _agent_provider("aggregator", tier="aggregator")
|
||||
final, agg_prompt_tokens, agg_completion_tokens = await run_aggregator(
|
||||
header, reports, provider=aggregator_provider, version=version
|
||||
)
|
||||
else:
|
||||
# 所有专家无数据/均失败:跳过终裁,标记 degraded
|
||||
logger.warning(
|
||||
"match %s: 所有 %d 位专家均无有效数据,跳过终裁,标记 degraded",
|
||||
match_id, len(reports),
|
||||
)
|
||||
aggregator_provider = await _agent_provider("aggregator", tier="aggregator")
|
||||
final = {
|
||||
"pred_home_goals": None,
|
||||
"pred_away_goals": None,
|
||||
"alt_pred_home_goals": None,
|
||||
"alt_pred_away_goals": None,
|
||||
"pred_1x2": None,
|
||||
"subjective_confidence": None,
|
||||
"reasoning": f"所有 {len(reports)} 位专家均无有效数据或预测失败(状态: {','.join(r.status for r in reports)})",
|
||||
"agent_weights": {},
|
||||
}
|
||||
agg_prompt_tokens = 0
|
||||
agg_completion_tokens = 0
|
||||
|
||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||
|
||||
@@ -262,15 +286,23 @@ async def predict_match_multi(
|
||||
if m is None:
|
||||
raise ValueError(f"match {match_id} not found")
|
||||
|
||||
# 严格校验终裁输出
|
||||
# 根据是否有有效数据决定校验策略
|
||||
from src.llm.validation import validate_agent_weights, validate_prediction_output
|
||||
try:
|
||||
validated = validate_prediction_output(final)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"终裁输出校验失败: {e}")
|
||||
|
||||
# agent_weights 同样必须过校验(旧实现直接取 raw 值落库,未做任何检查)
|
||||
agent_weights = validate_agent_weights(final.get("agent_weights"))
|
||||
if has_valid_data:
|
||||
# 有有效报告:严格校验终裁输出
|
||||
try:
|
||||
validated = validate_prediction_output(final)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"终裁输出校验失败: {e}")
|
||||
agent_weights = validate_agent_weights(final.get("agent_weights"))
|
||||
pred_status = "success"
|
||||
else:
|
||||
# 无有效报告:跳过严格校验,直接构造降级结果
|
||||
validated = None # type: ignore
|
||||
agent_weights = {}
|
||||
pred_status = "degraded"
|
||||
|
||||
pred = await _upsert_prediction(
|
||||
session,
|
||||
match_id=match_id,
|
||||
@@ -283,16 +315,17 @@ async def predict_match_multi(
|
||||
"prompt_tokens": sum(r.prompt_tokens or 0 for r in reports) + agg_prompt_tokens,
|
||||
"completion_tokens": sum(r.completion_tokens or 0 for r in reports) + agg_completion_tokens,
|
||||
"latency_ms": latency_ms,
|
||||
"pred_home_goals": validated.pred_home_goals,
|
||||
"pred_away_goals": validated.pred_away_goals,
|
||||
"alt_pred_home_goals": validated.alt_pred_home_goals,
|
||||
"alt_pred_away_goals": validated.alt_pred_away_goals,
|
||||
"pred_1x2": validated.pred_1x2,
|
||||
"subjective_confidence": validated.subjective_confidence,
|
||||
"reasoning": validated.reasoning,
|
||||
"pred_home_goals": validated.pred_home_goals if validated else None,
|
||||
"pred_away_goals": validated.pred_away_goals if validated else None,
|
||||
"alt_pred_home_goals": validated.alt_pred_home_goals if validated else None,
|
||||
"alt_pred_away_goals": validated.alt_pred_away_goals if validated else None,
|
||||
"pred_1x2": validated.pred_1x2 if validated else None,
|
||||
"subjective_confidence": validated.subjective_confidence if validated else None,
|
||||
"reasoning": validated.reasoning if validated else final.get("reasoning", ""),
|
||||
"raw_response": final,
|
||||
"agent_outputs": [r.to_dict() for r in reports],
|
||||
"status": "success",
|
||||
"agent_weights": agent_weights,
|
||||
"status": pred_status,
|
||||
"match_kickoff_at": match_kickoff_at,
|
||||
"prediction_cutoff_at": prediction_cutoff_at,
|
||||
"prediction_created_at": now,
|
||||
|
||||
+57
-17
@@ -40,11 +40,23 @@ def _outcome(home_goals: int, away_goals: int, side: str) -> str:
|
||||
|
||||
|
||||
def _is_stats_available(stats, before) -> bool:
|
||||
"""检查统计数据在 cutoff 时间是否已可用。"""
|
||||
"""检查统计数据在 cutoff 时间是否已可用。
|
||||
|
||||
available_at 语义:该条统计「对外可被使用」的最早时间,
|
||||
至少不得早于比赛结束。用于回测防泄漏。
|
||||
|
||||
规则:
|
||||
- before is None(实盘):available_at 为 None 时允许(兼容旧数据)
|
||||
- before is not None(回测):available_at 为 None 视为不可用(保守)
|
||||
- available_at > cutoff:不可用(数据在 cutoff 之后才生成)
|
||||
"""
|
||||
if before is None:
|
||||
# 实盘模式:无时间信息时允许(兼容旧数据)
|
||||
return True
|
||||
# 回测模式(cutoff 不为 None):
|
||||
# available_at 为 None → 无法确认是否在 cutoff 前可用,保守视为不可用
|
||||
if stats.available_at is None:
|
||||
return True # 无时间信息时保守处理:允许使用
|
||||
return False
|
||||
return stats.available_at <= before
|
||||
|
||||
|
||||
@@ -330,34 +342,62 @@ async def injuries_slice(header: MatchHeader, *, before=None, db: AsyncSession |
|
||||
|
||||
before=cutoff: 只使用 cutoff 之前已采集的伤停数据,防回测泄漏。
|
||||
db: 可选共享 session(见模块 docstring)。
|
||||
|
||||
语义区分:
|
||||
- 查询成功 + 空结果 → has_data=True(明确知道「无人伤停」)
|
||||
- 源未配置 / 查询失败 → has_data=False(无法判断,跳过 LLM)
|
||||
"""
|
||||
from src.data.injuries import get_injuries_for_match
|
||||
from src.data.injuries import get_injuries_for_match, InjuryQueryResult
|
||||
|
||||
cutoff = before or header.match_dt
|
||||
if db is not None:
|
||||
home_injuries = await get_injuries_for_match(db, header.home_team_id, cutoff, as_of=cutoff)
|
||||
away_injuries = await get_injuries_for_match(db, header.away_team_id, cutoff, as_of=cutoff)
|
||||
home_result = await get_injuries_for_match(db, header.home_team_id, cutoff, as_of=cutoff)
|
||||
away_result = await get_injuries_for_match(db, header.away_team_id, cutoff, as_of=cutoff)
|
||||
else:
|
||||
async with AsyncSessionLocal() as new_db:
|
||||
home_injuries = await get_injuries_for_match(new_db, header.home_team_id, cutoff, as_of=cutoff)
|
||||
away_injuries = await get_injuries_for_match(new_db, header.away_team_id, cutoff, as_of=cutoff)
|
||||
home_result = await get_injuries_for_match(new_db, header.home_team_id, cutoff, as_of=cutoff)
|
||||
away_result = await get_injuries_for_match(new_db, header.away_team_id, cutoff, as_of=cutoff)
|
||||
|
||||
# 判断是否有有效查询结果
|
||||
# 两队都成功查询(即使为空) → has_data=True
|
||||
# 任一查询失败或源未配置 → has_data=False
|
||||
both_succeeded = (
|
||||
home_result.query_status == "success"
|
||||
and away_result.query_status == "success"
|
||||
)
|
||||
any_configured = (
|
||||
home_result.query_status != "source_not_configured"
|
||||
or away_result.query_status != "source_not_configured"
|
||||
)
|
||||
|
||||
lines = ["── 阵容完整性 ──"]
|
||||
n_records = 0
|
||||
for label, injuries in (("主队", home_injuries), ("客队", away_injuries)):
|
||||
if injuries:
|
||||
n_records += len(injuries)
|
||||
lines.append(f" {label}伤停({len(injuries)}人):")
|
||||
for inj in injuries[:8]: # 最多显示 8 条
|
||||
|
||||
for label, result in (("主队", home_result), ("客队", away_result)):
|
||||
if result.query_status == "source_not_configured":
|
||||
lines.append(f" {label}: 伤停源未配置")
|
||||
elif result.query_status == "query_error":
|
||||
lines.append(f" {label}: 查询异常")
|
||||
elif result.records:
|
||||
n_records += len(result.records)
|
||||
lines.append(f" {label}伤停({len(result.records)}人):")
|
||||
for inj in result.records[:8]:
|
||||
reason = inj.reason or inj.injury_type or "未知"
|
||||
lines.append(f" - {inj.player_name}: {reason}")
|
||||
if len(injuries) > 8:
|
||||
lines.append(f" ...及其他 {len(injuries) - 8} 人")
|
||||
if len(result.records) > 8:
|
||||
lines.append(f" ...及其他 {len(result.records) - 8} 人")
|
||||
else:
|
||||
lines.append(f" {label}: 无伤停数据")
|
||||
# 查询成功但无人伤停
|
||||
lines.append(f" {label}: 当前无伤停记录")
|
||||
|
||||
if n_records == 0:
|
||||
return SliceResult(text="── 阵容完整性 ──\n 无数据", has_data=False, n_records=0)
|
||||
# 决定 has_data:
|
||||
# - 两队都成功查询(即使为空) → True(明确知道名单)
|
||||
# - 源未配置且无数据 → False
|
||||
has_data = both_succeeded or (any_configured and n_records > 0)
|
||||
|
||||
if not has_data:
|
||||
# 保留详细状态文案(伤停源未配置/查询异常),而非通用「无数据」
|
||||
return SliceResult(text="\n".join(lines), has_data=False, n_records=0)
|
||||
|
||||
return SliceResult(text="\n".join(lines), has_data=True, n_records=n_records)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user