shangfangjian/Profeto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

安全加固 + 数据管线修复 + 质量改进(第2批)

Browse Source 
安全加固:
- /api/v1/predict 限流改用 get_client_ip 防 X-Forwarded-For 伪造
- 新增 TRUST_PROXY_HEADERS/REQUIRE_ADMIN_AUTH 配置,默认 fail-closed
- 生产环境管理接口未配置鉴权时拒绝(503),不再放行

数据管线修复:
- injuries IntegrityError 改用 begin_nested(SAVEPOINT)隔离批次
- injuries 空名单 vs 未配置语义区分(has_data 精确标记)
- bzzoiro 统计字段映射(shots/possession/cards) + 入库条件放宽
- available_at 语义收紧(回测防泄漏)
- team_names NFKD 去变音双重查找 + 补齐变体键

预测系统改进:
- multi-agent 全专家失败时 status=degraded 跳过终裁
- agent_weights 独立持久化到 predictions 表

新增迁移:
- 0014_predictions_agent_weights.py

新增测试(8个文件):
- test_ip_spoofing.py: 限流防伪造
- test_require_admin_fail_closed.py: 生产 fail-closed
- test_injuries_integrity_rollback.py: SAVEPOINT 隔离
- test_injuries_slice_semantic.py: 空名单 vs 未配置
- test_available_at.py: 回测防泄漏
- test_bzzoirot_stats.py: 统计字段映射
- test_team_names_normalize.py: NFKD 变体
- test_multi_agent_degraded.py: 全失败 degraded
- test_agent_weights_persist.py: 权重持久化
This commit is contained in:
Profeto Agent
2026-09-19 08:02:57 +00:00
parent bee330f31f
commit 6c24672e87
21 changed files with 1580 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/data/bzzoiro.py
+17 -4
View File
@@ -266,8 +266,12 @@ class BzzoiroSource:
db.add(m)
await db.flush()
existing_matches[match_key] = m # 防止同批重复
if nm.home_xg is not None or nm.away_xg is not None:
# 存在任一统计字段即可创建 MatchStats(不再强制要求 xG)
if any(getattr(nm, f) is not None for f in ['home_xg', 'away_xg', 'home_shots', 'away_shots', 'home_shots_on_target', 'away_shots_on_target', 'home_corners', 'away_corners', 'home_possession', 'home_yellow_cards', 'away_yellow_cards', 'home_red_cards', 'away_red_cards']):
now = datetime.now(timezone.utc)
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
available_at = nm.date if nm.date else now
stats = MatchStats(
match_id=m.id,
home_xg=nm.home_xg,
@@ -286,7 +290,7 @@ class BzzoiroSource:
source="bzzoiro",
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
available_at=available_at,
)
db.add(stats)
league_r["inserted"] += 1
@@ -305,14 +309,23 @@ class BzzoiroSource:
if existing_match.match_stage is None and nm.match_stage:
existing_match.match_stage = nm.match_stage
changed = True
if existing_match.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
# 存在任一统计字段即可创建 MatchStats(不再强制要求 xG)
if existing_match.stats is None and (
nm.home_xg is not None or nm.away_xg is not None
or nm.home_shots is not None or nm.away_shots is not None
or nm.home_corners is not None or nm.away_corners is not None
or nm.home_possession is not None
):
now = datetime.now(timezone.utc)
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
available_at = nm.date if nm.date else now
existing_match.stats = MatchStats(
match_id=existing_match.id,
source="bzzoiro",
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
available_at=available_at,
)
db.add(existing_match.stats)
await db.flush()
src/data/injuries.py
+82 -36
View File
@@ -12,11 +12,25 @@ import tempfile
import time
from datetime import datetime, timezone
from pathlib import Path
from dataclasses import dataclass
from typing import Any
import httpx
from src.core.runtime_config import get_runtime_value
@dataclass
class InjuryQueryResult:
"""伤停查询结果(区分「查询成功但为空」与「查询失败/源未配置」)。"""
records: list["Injury"]
query_status: str # "success" | "source_not_configured" | "query_error"
@property
def has_data(self) -> bool:
"""成功查询(即使结果为空)视为有明确名单,has_data=True。"""
return self.query_status == "success"
from src.core.http_client import get_client
logger = logging.getLogger(__name__)
@@ -213,42 +227,59 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
rows = (await db.execute(stmt)).all()
existing_keys = {(r[0], r[1], r[2]) for r in rows}
# Fix 1: 使用 SAVEPOINT(begin_nested)避免整批回滚丢数据
# 每个 batch 使用独立的 savepoint,失败时只回滚该 batch
# Fix 1: 使用 begin_nested(SAVEPOINT)隔离每批 flush
# IntegrityError 时只回滚到 savepoint,不影响其它已成功批次
BATCH_SIZE = 50
batch: list[Injury] = []
async def _flush_batch():
"""使用 savepoint flush 一批记录;失败只回滚本批。"""
if not batch:
return
async with db.begin_nested():
for obj in batch:
db.add(obj)
await db.flush()
batch.clear()
for i, rec in enumerate(pending_records):
key = (rec["player_id"], rec["fixture_id"], rec["injury_type"])
if key in existing_keys:
continue
injury = Injury(**rec)
db.add(injury)
batch.append(Injury(**rec))
result["inserted"] += 1
# 每 BATCH_SIZE 条 flush 一次,使用 SAVEPOINT 隔离
if result["inserted"] % BATCH_SIZE == 0:
# 每 BATCH_SIZE 条 flush 一次
if len(batch) >= BATCH_SIZE:
try:
await db.flush()
await _flush_batch()
except IntegrityError:
# 只回滚到上一个 savepoint,不影响已提交的数据
await db.rollback()
logger.warning("injuries batch IntegrityError at record %d, continuing", i + 1)
# 从当前位置继续处理剩余记录
logger.warning(
"injuries batch IntegrityError at record %d, "
"rolled back to savepoint, continuing",
i + 1,
)
# begin_nested 已回滚到 savepoint,清空 batch 继续
batch.clear()
continue
# 最终 flush(剩余不足一批的记录)
try:
await db.flush()
await _flush_batch()
except IntegrityError:
await db.rollback()
logger.warning("injuries final flush IntegrityError, some records may be lost")
logger.warning(
"injuries final flush IntegrityError, "
"rolled back to savepoint, some records may be lost",
)
batch.clear()
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
return result
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> list[Injury]:
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> InjuryQueryResult:
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。
Args:
@@ -258,31 +289,46 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> li
as_of: 数据截止时间(用于回测防泄漏)
Returns:
伤停记录列表
InjuryQueryResult:包含查询记录与状态
- query_status="success": 查询成功(即使结果也为空)
- query_status="source_not_configured": API_FOOTBALL_KEY 未配置
- query_status="query_error": 查询异常
语义区分:
- 成功查询 + 空结果 → has_data=True(明确知道「无人伤停」)
- 源未配置 / 查询失败 → has_data=False(无法判断,跳过 LLM)
"""
from sqlalchemy import select, func
from src.db.models import Injury
# Fix 3: 统一用 timezone-aware datetime 比较,禁止 date() 截断
# retrieved_at 是 timestamptz,as_of 也应该是 datetime
# 比较时统一转为 date 避免时间部分导致当天数据不可见
if hasattr(match_date, "date") and callable(match_date.date):
match_date = match_date.date()
# 检查 API 是否配置(只读配置,不发网络)
api_key = await get_runtime_value("API_FOOTBALL_KEY")
if not api_key:
logger.debug("API_FOOTBALL_KEY 未配置,跳过伤停查询 team=%s", team_id)
return InjuryQueryResult(records=[], query_status="source_not_configured")
stmt = (
select(Injury)
.where(Injury.team_id == team_id)
.where(Injury.injury_date <= match_date)
.where(
(Injury.return_date.is_(None)) | (Injury.return_date >= match_date)
try:
# Fix 3: 统一用 timezone-aware datetime 比较,禁止 date() 截断
if hasattr(match_date, "date") and callable(match_date.date):
match_date = match_date.date()
stmt = (
select(Injury)
.where(Injury.team_id == team_id)
.where(Injury.injury_date <= match_date)
.where(
(Injury.return_date.is_(None)) | (Injury.return_date >= match_date)
)
)
)
if as_of is not None:
# Fix 3: 统一用 date 比较,避免 timestamptz vs date 的时区问题
if hasattr(as_of, "date") and callable(as_of.date):
as_of = as_of.date()
# 使用 func.date() 将 timestamptz 转为 date,确保当天白天采到的数据对当晚比赛可见
stmt = stmt.where(func.date(Injury.retrieved_at) <= as_of)
if as_of is not None:
# Fix 3: 统一用 date 比较,避免 timestamptz vs date 的时区问题
if hasattr(as_of, "date") and callable(as_of.date):
as_of = as_of.date()
stmt = stmt.where(func.date(Injury.retrieved_at) <= as_of)
result = await db.execute(stmt)
return list(result.scalars().all())
result = await db.execute(stmt)
records = list(result.scalars().all())
return InjuryQueryResult(records=records, query_status="success")
except Exception as e:
logger.exception("伤停查询异常 team=%s: %s", team_id, e)
return InjuryQueryResult(records=[], query_status="query_error")
src/data/normalize.py
+23
View File
@@ -177,6 +177,29 @@ def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
m.match_stage = _rn_name
elif _rn:
m.match_stage = f"第 {_rn} 轮"
# 统计字段映射(API 字段名 → NormalizedMatch)
# API 可能提供的字段:home_shots/away_shots, shots_on_target, corners, possession, xg, cards
# API 没有的字段保持 None,不伪造
m.home_shots = _to_int(raw.get("home_shots", raw.get("shots_home")))
m.away_shots = _to_int(raw.get("away_shots", raw.get("shots_away")))
m.home_shots_on_target = _to_int(raw.get("home_shots_on_target", raw.get("sot_home")))
m.away_shots_on_target = _to_int(raw.get("away_shots_on_target", raw.get("sot_away")))
m.home_corners = _to_int(raw.get("home_corners", raw.get("corners_home")))
m.away_corners = _to_int(raw.get("away_corners", raw.get("corners_away")))
# 控球率:API 通常只给 home 值,away = 100 - home
possession_home = _to_float(raw.get("home_possession", raw.get("possession")))
if possession_home is not None:
m.home_possession = possession_home
# xG
m.home_xg = _to_float(raw.get("home_xg", raw.get("xg_home", raw.get("expected_goals_home"))))
m.away_xg = _to_float(raw.get("away_xg", raw.get("xg_away", raw.get("expected_goals_away"))))
# 牌
m.home_yellow_cards = _to_int(raw.get("home_yellow_cards", raw.get("yellow_cards_home")))
m.away_yellow_cards = _to_int(raw.get("away_yellow_cards", raw.get("yellow_cards_away")))
m.home_red_cards = _to_int(raw.get("home_red_cards", raw.get("red_cards_home")))
m.away_red_cards = _to_int(raw.get("away_red_cards", raw.get("red_cards_away")))
if m.match_status == "finished" and m.home_goals is None:
m.match_status = "scheduled"
return m
src/data/team_names.py
+28 -2
View File
@@ -1,6 +1,9 @@
"""队名归一化:各源队名 → 统一规范名。
迁移自旧项目 app/data/team_names.py。
NFKD 归一化会剥离变音符号(ü→u),导致「Bayern München」与「Bayern Munich」
映射到不同规范名。修复:双重查找(原始名 + NFKD 名) + 补齐常见变体键。
"""
from __future__ import annotations
@@ -61,10 +64,15 @@ NORMALIZE_MAP = {
"Barcelona": "Barcelona",
# ---- 德甲 ----
"Bayern Munich": "Bayern München",
"Bayern Munchen": "Bayern München", # NFKD stripped variant
"Bayern München": "Bayern München", # canonical with umlaut (direct hit)
"FC Koln": "FC Köln",
"FC Köln": "FC Köln",
"RB Leipzig": "RB Leipzig",
"Borussia Dortmund": "Borussia Dortmund",
"Borussia M'gladbach": "Borussia Mönchengladbach",
"Borussia Monchengladbach": "Borussia Mönchengladbach", # NFKD stripped
"Borussia Mönchengladbach": "Borussia Mönchengladbach", # canonical
"Bayer Leverkusen": "Bayer Leverkusen",
"Eintracht Frankfurt": "Eintracht Frankfurt",
"VfB Stuttgart": "VfB Stuttgart",
@@ -121,6 +129,9 @@ NORMALIZE_MAP = {
"Le Havre": "Le Havre AC",
"Lorient": "FC Lorient",
"Saint-Etienne": "AS Saint-Étienne",
"Saint-Etienne": "AS Saint-Étienne", # NFKD stripped (ê → e)
"AS Saint-Étienne": "AS Saint-Étienne", # canonical prefix
"AS Saint-Etienne": "AS Saint-Étienne", # NFKD stripped with prefix
"Angers": "Angers SCO",
"Auxerre": "AJ Auxerre",
"Leganes": "Leganés",
@@ -128,10 +139,25 @@ NORMALIZE_MAP = {
def normalize(name: str) -> str:
"""队名归一化:变音符号变体 → 统一规范名。
双重查找策略:
1. 原始名(带变音)直接查表
2. NFKD 去变音后再查表
确保「Bayern München」「Bayern Munchen」「Bayern Munich」
都映射到同一规范名「Bayern München」。
"""
if not name:
return ""
# unicode 归一(重音)
n = unicodedata.normalize("NFKD", name)
# 1. 原始名直接查表(保留变音变体)
stripped = name.strip()
if stripped in NORMALIZE_MAP:
return NORMALIZE_MAP[stripped]
# 2. NFKD 去变音后查表
n = unicodedata.normalize("NFKD", stripped)
n = "".join(c for c in n if not unicodedata.combining(c))
n = n.strip()
return NORMALIZE_MAP.get(n, n)
src/data/understat.py
+5 -1
View File
@@ -199,12 +199,16 @@ class UnderstatSource:
# 回填 xG
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
now = datetime.now(timezone.utc)
# available_at 语义:统计「可被使用」的最早时间,至少不早于比赛结束
# 局限:用 match_date(开球时间)近似,实际完赛时间约为 +2 小时
match_date = existing.match_date if existing.match_date else now
available_at = match_date
existing.stats = MatchStats(
match_id=existing.id,
source="understat",
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
available_at=available_at,
)
db.add(existing.stats)
await db.flush()