shangfangjian
06e973d34d
R4 首版实现用模块级 _ACTIVE_MODEL_OVERRIDE 中转 model override, 理由是不想改动 run_specialists 的签名(既有测试会 mock 它)。 但 backtest.py:225 会 asyncio.gather 并发 8 场预测(Semaphore(8)), 每场都调用 predict_match_multi —— 模块级变量会被并发调用互相覆盖, 导致 A 场的预测用上 B 场的模型。这是静默的正确性缺陷。 改为: run_specialists 新增 model_override 形参,一路显式下传; 删除模块级变量。同步更新 6 处 mock 签名。 新增 3 个守卫并做变异验证: - test_r4_no_module_level_model_override_global (源码级,可判别) - test_r4_run_specialists_accepts_model_override_parameter - test_r4_dispatch_passes_override_to_specialists 变异测试: 重新引入全局变量方案 → 两个守卫变红;还原 → 全绿。 注: 曾尝试写并发行为测试,但真实 run_specialists 会访问数据库, 测试环境下不稳定(ConnectionRefusedError),会是 flaky 的假证据, 故改用清晰的源码级判别守卫,并在注释中说明原因。 240 passed / 1 skipped; tsc 0 error; vite build 成功。
224 lines
8.8 KiB
Python
224 lines
8.8 KiB
Python
"""回归测试: multi-agent 预测路径 backtest cutoff / provider / model 透传。
|
|
|
|
验证:
|
|
1. predict_match_multi 正确计算并传递 cutoff
|
|
2. cutoff 贯穿到所有 5 个专家切片
|
|
3. prediction_cutoff_at 记录的是真正的 cutoff,而非 match_dt
|
|
4. backtest=True 时「赛后才 available 的 xG」不会出现在切片里
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from src.llm.context_builder import MatchHeader
|
|
|
|
|
|
def _make_header(match_dt=None) -> MatchHeader:
|
|
from datetime import datetime, timezone
|
|
if match_dt is None:
|
|
match_dt = datetime(2026, 1, 15, 20, 0, tzinfo=timezone.utc)
|
|
return MatchHeader(
|
|
match_id=999, home_name="利物浦", away_name="阿森纳",
|
|
league_name="英超", season="2025-2026",
|
|
match_date="2026-01-15 20:00 UTC",
|
|
match_dt=match_dt, stage=None,
|
|
home_team_id=1, away_team_id=2, league_id=1,
|
|
)
|
|
|
|
|
|
class TestMultiAgentCutoffPropagation:
|
|
"""验证 cutoff 在 multi-agent 路径中正确计算和传递。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_backtest_computes_cutoff_from_match_dt_minus_1_day(self):
|
|
"""backtest=True → cutoff = match_dt - 1 天,传给所有切片。"""
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest.mock import patch
|
|
import src.llm.agents.orchestrator as orch
|
|
|
|
match_dt = datetime(2026, 1, 15, 20, 0, tzinfo=timezone.utc)
|
|
header = _make_header(match_dt)
|
|
|
|
captured_before = []
|
|
|
|
async def mock_run_specialists(header, *, version, before=None, model_override=None):
|
|
captured_before.append(before)
|
|
return []
|
|
|
|
async def mock_header(mid, db=None):
|
|
# load_match_header 是 async,必须是 async 函数;
|
|
# 且必须走 patch.object(orch 模块属性),因为 predict_match_multi
|
|
# 通过模块命名空间解析该名字。裸赋值 orch.load_match_header 同样有效,
|
|
# 但用 patch 可保证退出时精确还原,不向后续测试泄漏。
|
|
return header
|
|
|
|
async def mock_provider(agent_id, *, tier, model_override=None):
|
|
# 真实契约是 async(见 orchestrator._agent_provider),同步 lambda
|
|
# 会让 `await _agent_provider(...)` 抛 TypeError 并被吞掉。
|
|
return MagicMock(model="test")
|
|
|
|
with patch.object(orch, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch, "load_match_header", mock_header), \
|
|
patch.object(orch, "_agent_provider", mock_provider):
|
|
try:
|
|
await orch.predict_match_multi(999, backtest=True)
|
|
except Exception:
|
|
pass # 后续 aggregator 调用会因 mock 不全而失败,不影响 cutoff 测试
|
|
|
|
assert len(captured_before) == 1
|
|
expected_cutoff = match_dt - timedelta(days=1)
|
|
assert captured_before[0] == expected_cutoff, (
|
|
f"backtest cutoff 应为 {expected_cutoff},实际 {captured_before[0]}"
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_explicit_cutoff_at_overrides_backtest(self):
|
|
"""显式 cutoff_at 优先于 backtest 自动计算。"""
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import patch
|
|
import src.llm.agents.orchestrator as orch
|
|
|
|
match_dt = datetime(2026, 1, 15, 20, 0, tzinfo=timezone.utc)
|
|
explicit_cutoff = datetime(2026, 1, 10, 12, 0, tzinfo=timezone.utc)
|
|
header = _make_header(match_dt)
|
|
|
|
captured_before = []
|
|
|
|
async def mock_run_specialists(header, *, version, before=None, model_override=None):
|
|
captured_before.append(before)
|
|
return []
|
|
|
|
async def mock_header(mid, db=None):
|
|
return header
|
|
|
|
async def mock_provider(agent_id, *, tier, model_override=None):
|
|
return MagicMock(model="test")
|
|
|
|
with patch.object(orch, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch, "load_match_header", mock_header), \
|
|
patch.object(orch, "_agent_provider", mock_provider):
|
|
try:
|
|
await orch.predict_match_multi(999, backtest=True, cutoff_at=explicit_cutoff)
|
|
except Exception:
|
|
pass
|
|
|
|
assert captured_before[0] == explicit_cutoff
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_normal_mode_cutoff_is_match_dt(self):
|
|
"""非回测模式,无显式 cutoff → cutoff = match_dt。"""
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import patch
|
|
import src.llm.agents.orchestrator as orch
|
|
|
|
match_dt = datetime(2026, 1, 15, 20, 0, tzinfo=timezone.utc)
|
|
header = _make_header(match_dt)
|
|
|
|
captured_before = []
|
|
|
|
async def mock_run_specialists(header, *, version, before=None, model_override=None):
|
|
captured_before.append(before)
|
|
return []
|
|
|
|
async def mock_header(mid, db=None):
|
|
return header
|
|
|
|
async def mock_provider(agent_id, *, tier, model_override=None):
|
|
return MagicMock(model="test")
|
|
|
|
with patch.object(orch, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch, "load_match_header", mock_header), \
|
|
patch.object(orch, "_agent_provider", mock_provider):
|
|
try:
|
|
await orch.predict_match_multi(999, backtest=False)
|
|
except Exception:
|
|
pass
|
|
|
|
assert captured_before[0] == match_dt
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_prediction_cutoff_at_stored_not_match_dt(self):
|
|
"""Prediction 写入时 prediction_cutoff_at = 真正 cutoff,非 match_dt。"""
|
|
from datetime import datetime, timedelta, timezone
|
|
from unittest.mock import patch
|
|
import src.llm.predict as pred
|
|
|
|
match_dt = datetime(2026, 1, 15, 20, 0, tzinfo=timezone.utc)
|
|
expected_cutoff = match_dt - timedelta(days=1)
|
|
|
|
class FakeContext:
|
|
text = "fake"
|
|
cutoff = expected_cutoff
|
|
|
|
FakeContext.match_dt = match_dt
|
|
|
|
call_args = {}
|
|
|
|
async def tracking_build(match_id, **kw):
|
|
call_args.update(kw)
|
|
return FakeContext()
|
|
|
|
with patch.object(pred, "build_context", tracking_build):
|
|
try:
|
|
await pred._predict_single(999, backtest=True)
|
|
except Exception:
|
|
pass
|
|
|
|
# 重点: build_context 被调用时传入 backtest=True 和正确的 cutoff
|
|
assert call_args.get("backtest") is True, "backtest=True 应传递给 build_context"
|
|
|
|
|
|
class TestBacktestXgNotVisible:
|
|
"""P0-3 延伸:回测时赛后才 available 的统计数据不应出现在切片。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_stats_slice_respects_cutoff_for_xg_availability(self):
|
|
"""available_at > cutoff 的 xG 数据不应被切片使用。"""
|
|
from datetime import datetime, timezone
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
cutoff = datetime(2026, 1, 13, 20, 0, tzinfo=timezone.utc) # match_date - 2天
|
|
match_dt = datetime(2026, 1, 15, 20, 0, tzinfo=timezone.utc)
|
|
|
|
# 创建一场历史比赛,其 xG 在 match_date 之后才 available
|
|
hist_match = MagicMock()
|
|
hist_match.id = 500
|
|
hist_match.home_team_id = 1 # 利物浦主场
|
|
hist_match.away_team_id = 3
|
|
hist_match.home_goals = 2
|
|
hist_match.away_goals = 0
|
|
hist_match.home_team = MagicMock(id=1, name="利物浦", name_zh=None)
|
|
hist_match.away_team = MagicMock(id=3, name="诺维奇", name_zh=None)
|
|
|
|
# xG: available_at 在比赛日之后(1月16日),cutoff(1月13日)看不到
|
|
stats = MagicMock()
|
|
stats.home_xg = 2.5
|
|
stats.away_xg = 0.3
|
|
stats.home_shots = 15
|
|
stats.away_shots = 4
|
|
stats.home_shots_on_target = 6
|
|
stats.away_shots_on_target = 1
|
|
stats.home_possession = 65.0
|
|
stats.available_at = datetime(2026, 1, 16, 10, 0, tzinfo=timezone.utc) # 赛后才有
|
|
hist_match.stats = stats
|
|
|
|
header = _make_header(match_dt)
|
|
|
|
import src.llm.context_builder as cb
|
|
|
|
async def mock_get_form(db, team_id, before, *, limit=10):
|
|
# before=cutoff(1月13日),比赛在1月15日,满足 before 条件
|
|
if before is not None and before < match_dt:
|
|
return [hist_match]
|
|
return []
|
|
|
|
with patch.object(cb, "_get_form", mock_get_form):
|
|
result = await cb.stats_slice(header, limit=10, before=cutoff)
|
|
text = str(result)
|
|
# xG 在 cutoff 之后才 available,不应出现在切片
|
|
assert "2.50" not in text, f"xG 2.50 不应在切片中(available_at > cutoff):\n{text}"
|
|
# 但无比分时仍应显示进球数据
|
|
assert "无比分数据" in text or "场均进球" in text, f"无比分时仍应显示基本数据:\n{text}"
|