shangfangjian
06e973d34d
R4 首版实现用模块级 _ACTIVE_MODEL_OVERRIDE 中转 model override, 理由是不想改动 run_specialists 的签名(既有测试会 mock 它)。 但 backtest.py:225 会 asyncio.gather 并发 8 场预测(Semaphore(8)), 每场都调用 predict_match_multi —— 模块级变量会被并发调用互相覆盖, 导致 A 场的预测用上 B 场的模型。这是静默的正确性缺陷。 改为: run_specialists 新增 model_override 形参,一路显式下传; 删除模块级变量。同步更新 6 处 mock 签名。 新增 3 个守卫并做变异验证: - test_r4_no_module_level_model_override_global (源码级,可判别) - test_r4_run_specialists_accepts_model_override_parameter - test_r4_dispatch_passes_override_to_specialists 变异测试: 重新引入全局变量方案 → 两个守卫变红;还原 → 全绿。 注: 曾尝试写并发行为测试,但真实 run_specialists 会访问数据库, 测试环境下不稳定(ConnectionRefusedError),会是 flaky 的假证据, 故改用清晰的源码级判别守卫,并在注释中说明原因。 240 passed / 1 skipped; tsc 0 error; vite build 成功。
307 lines
12 KiB
Python
307 lines
12 KiB
Python
"""回归测试: multi-agent 全专家失败/无数据时 status=degraded。
|
|
|
|
验证:
|
|
1. 5 个专家全 no_data/error → status="degraded",不调终裁
|
|
2. 至少 1 个专家 ok → status="success",正常走终裁
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from src.llm.agents.base import AgentReport
|
|
from src.llm.agents import orchestrator as orch_mod
|
|
|
|
|
|
def _make_header():
|
|
from src.llm.context_builder import MatchHeader
|
|
return MatchHeader(
|
|
match_id=999, home_name="A", away_name="B",
|
|
league_name="X", season=None, match_date="?",
|
|
match_dt=None, stage=None,
|
|
home_team_id=1, away_team_id=2, league_id=1,
|
|
)
|
|
|
|
|
|
def _all_error_reports():
|
|
"""5 个专家全 error."""
|
|
return [
|
|
AgentReport(agent="form", status="error", analysis="slice failed"),
|
|
AgentReport(agent="stats", status="error", analysis="slice failed"),
|
|
AgentReport(agent="home_away", status="error", analysis="slice failed"),
|
|
AgentReport(agent="standings", status="error", analysis="slice failed"),
|
|
AgentReport(agent="h2h", status="error", analysis="slice failed"),
|
|
]
|
|
|
|
|
|
def _all_no_data_reports():
|
|
"""5 个专家全 no_data."""
|
|
return [
|
|
AgentReport(agent="form", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="stats", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="home_away", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="standings", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="h2h", status="no_data", analysis="无数据"),
|
|
]
|
|
|
|
|
|
def _mixed_reports():
|
|
"""1 个 ok,4 个 error."""
|
|
return [
|
|
AgentReport(agent="form", status="ok", analysis="good"),
|
|
AgentReport(agent="stats", status="error", analysis="failed"),
|
|
AgentReport(agent="home_away", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="standings", status="error", analysis="failed"),
|
|
AgentReport(agent="h2h", status="no_data", analysis="无数据"),
|
|
]
|
|
|
|
|
|
class TestAllExpertsFailed:
|
|
"""全部专家失败/无数据时,status 应为 degraded。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_error_reports_yields_degraded(self):
|
|
"""5 个专家全 error → status=degraded,不调终裁。"""
|
|
header = _make_header()
|
|
|
|
# Mock run_specialists 返回全 error
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _all_error_reports()
|
|
|
|
# Mock _agent_provider
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
return MagicMock(model="test-model")
|
|
|
|
# Mock load_match_header
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
# Mock _upsert_prediction — 捕获写入的 status
|
|
captured_status = {}
|
|
|
|
async def mock_upsert(session, **kw):
|
|
captured_status.update(kw.get("values", {}))
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = "test"
|
|
mock_pred.prompt_version = "v1"
|
|
mock_pred.pred_home_goals = None
|
|
mock_pred.pred_away_goals = None
|
|
mock_pred.alt_pred_home_goals = None
|
|
mock_pred.alt_pred_away_goals = None
|
|
mock_pred.pred_1x2 = None
|
|
mock_pred.subjective_confidence = None
|
|
mock_pred.reasoning = kw["values"].get("reasoning")
|
|
mock_pred.agent_outputs = []
|
|
return mock_pred
|
|
|
|
# Mock get_uow
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock() # match exists
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
result = await orch_mod.predict_match_multi(999)
|
|
|
|
# 断言:status 是 degraded,不是 success
|
|
assert captured_status.get("status") == "degraded", \
|
|
f"期望 status=degraded,实际 {captured_status.get('status')}"
|
|
# 断言:reasoning 包含说明
|
|
assert "专家" in captured_status.get("reasoning", ""), \
|
|
f"reasoning 应说明原因,实际 {captured_status.get('reasoning')}"
|
|
# 断言:pred_* 全为 None
|
|
assert captured_status.get("pred_home_goals") is None
|
|
assert captured_status.get("pred_1x2") is None
|
|
print(f"PASS: 全 error → status={captured_status.get('status')}")
|
|
print(f" reasoning={captured_status.get('reasoning')[:50]}...")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_no_data_reports_yields_degraded(self):
|
|
"""5 个专家全 no_data → status=degraded,不调终裁。"""
|
|
header = _make_header()
|
|
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _all_no_data_reports()
|
|
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
return MagicMock(model="test-model")
|
|
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
captured_status = {}
|
|
|
|
async def mock_upsert(session, **kw):
|
|
captured_status.update(kw.get("values", {}))
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = "test"
|
|
mock_pred.prompt_version = "v1"
|
|
mock_pred.pred_home_goals = None
|
|
mock_pred.pred_away_goals = None
|
|
mock_pred.alt_pred_home_goals = None
|
|
mock_pred.alt_pred_away_goals = None
|
|
mock_pred.pred_1x2 = None
|
|
mock_pred.subjective_confidence = None
|
|
mock_pred.reasoning = kw["values"].get("reasoning")
|
|
mock_pred.agent_outputs = []
|
|
return mock_pred
|
|
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock()
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
result = await orch_mod.predict_match_multi(999)
|
|
|
|
assert captured_status.get("status") == "degraded", \
|
|
f"期望 status=degraded,实际 {captured_status.get('status')}"
|
|
print(f"PASS: 全 no_data → status={captured_status.get('status')}")
|
|
|
|
|
|
class TestPartialExpertsOk:
|
|
"""部分专家 ok 时,status 仍可为 success。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_one_ok_report_allows_success(self):
|
|
"""1 个 ok + 4 个 error → status=success(走终裁)。"""
|
|
header = _make_header()
|
|
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _mixed_reports()
|
|
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
return MagicMock(model="test-model")
|
|
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
captured_status = {}
|
|
|
|
async def mock_aggregator(header, reports, *, provider, version):
|
|
# 终裁返回合法 JSON
|
|
return {
|
|
"pred_home_goals": 2,
|
|
"pred_away_goals": 1,
|
|
"pred_1x2": "1",
|
|
"subjective_confidence": 0.7,
|
|
"reasoning": "test prediction",
|
|
"agent_weights": {"form": 0.5},
|
|
}, 100, 50
|
|
|
|
async def mock_upsert(session, **kw):
|
|
captured_status.update(kw.get("values", {}))
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = "test"
|
|
mock_pred.prompt_version = "v1"
|
|
mock_pred.pred_home_goals = 2
|
|
mock_pred.pred_away_goals = 1
|
|
mock_pred.pred_1x2 = "1"
|
|
mock_pred.subjective_confidence = 0.7
|
|
mock_pred.reasoning = "test"
|
|
return mock_pred
|
|
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock()
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
|
patch.object(orch_mod, "run_aggregator", mock_aggregator), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
result = await orch_mod.predict_match_multi(999)
|
|
|
|
assert captured_status.get("status") == "success", \
|
|
f"期望 status=success,实际 {captured_status.get('status')}"
|
|
print(f"PASS: 1 ok + 4 error → status={captured_status.get('status')}")
|
|
|
|
|
|
class TestNoAggregatorCallOnDegraded:
|
|
"""全专家失败时,aggregator provider 不应被调用。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_error_no_aggregator_call(self):
|
|
"""5 个专家全 error → 不应调用 _agent_provider('aggregator')。"""
|
|
header = _make_header()
|
|
aggregator_called = []
|
|
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _all_error_reports()
|
|
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
aggregator_called.append((agent_id, tier))
|
|
return MagicMock(model="test-model")
|
|
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
captured_values = {}
|
|
|
|
async def mock_upsert(session, **kw):
|
|
# model / provider_name / mode 是 _upsert_prediction 的顶层关键字参数,
|
|
# 不在 values 字典里(见 orchestrator.py 的调用点)。原测试只取
|
|
# kw["values"],导致 model 断言永远为 None。
|
|
captured_values.update(kw.get("values", {}))
|
|
captured_values.update(
|
|
{k: kw.get(k) for k in ("model", "provider_name", "mode", "run_type")}
|
|
)
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = kw.get("model")
|
|
return mock_pred
|
|
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock()
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
await orch_mod.predict_match_multi(999)
|
|
|
|
# 断言:aggregator provider 未被调用
|
|
assert len(aggregator_called) == 0, \
|
|
f"全失败时不应调用 aggregator provider,实际调用: {aggregator_called}"
|
|
# 断言:model 使用 settings 默认值
|
|
assert captured_values.get("model") is not None
|
|
assert captured_values.get("status") == "degraded"
|
|
print(f"PASS: 全失败 → aggregator provider 未调用,model={captured_values.get('model')}")
|