shangfangjian
64ae8e663a
_upsert_prediction 改为 _insert_or_find_by_fingerprint: - 同 input_hash → 返回已有行(绝不 UPDATE pred_/reasoning/agent_outputs) - 不同 input_hash → INSERT 新行 input_hash 升级为规范 JSON SHA-256,捕获:match_id, cutoff, prompt_version, prompt_hash, system_prompt_hash, provider, model, mode, run_type, temperature, context_hash, agent_ids。移除旧 (match, provider, model, mode, run_type) 唯一约束, 改为 partial unique index(WHERE input_hash IS NOT NULL,兼容旧 NULL 数据)。 三条路径(single/multi/baseline)统一传足指纹字段。 迁移 0024 + 测试 test_p0_prediction_fingerprint(10/10);全量 295 通过。
306 lines
12 KiB
Python
306 lines
12 KiB
Python
"""回归测试: multi-agent 全专家失败/无数据时 status=degraded。
|
|
|
|
验证:
|
|
1. 5 个专家全 no_data/error → status="degraded",不调终裁
|
|
2. 至少 1 个专家 ok → status="success",正常走终裁
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from src.llm.agents.base import AgentReport
|
|
from src.llm.agents import orchestrator as orch_mod
|
|
|
|
|
|
def _make_header():
|
|
from src.llm.context_builder import MatchHeader
|
|
return MatchHeader(
|
|
match_id=999, home_name="A", away_name="B",
|
|
league_name="X", season=None, match_date="?",
|
|
match_dt=None, stage=None,
|
|
home_team_id=1, away_team_id=2, league_id=1,
|
|
)
|
|
|
|
|
|
def _all_error_reports():
|
|
"""5 个专家全 error."""
|
|
return [
|
|
AgentReport(agent="form", status="error", analysis="slice failed"),
|
|
AgentReport(agent="stats", status="error", analysis="slice failed"),
|
|
AgentReport(agent="home_away", status="error", analysis="slice failed"),
|
|
AgentReport(agent="standings", status="error", analysis="slice failed"),
|
|
AgentReport(agent="h2h", status="error", analysis="slice failed"),
|
|
]
|
|
|
|
|
|
def _all_no_data_reports():
|
|
"""5 个专家全 no_data."""
|
|
return [
|
|
AgentReport(agent="form", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="stats", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="home_away", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="standings", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="h2h", status="no_data", analysis="无数据"),
|
|
]
|
|
|
|
|
|
def _mixed_reports():
|
|
"""1 个 ok,4 个 error."""
|
|
return [
|
|
AgentReport(agent="form", status="ok", analysis="good"),
|
|
AgentReport(agent="stats", status="error", analysis="failed"),
|
|
AgentReport(agent="home_away", status="no_data", analysis="无数据"),
|
|
AgentReport(agent="standings", status="error", analysis="failed"),
|
|
AgentReport(agent="h2h", status="no_data", analysis="无数据"),
|
|
]
|
|
|
|
|
|
class TestAllExpertsFailed:
|
|
"""全部专家失败/无数据时,status 应为 degraded。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_error_reports_yields_degraded(self):
|
|
"""5 个专家全 error → status=degraded,不调终裁。"""
|
|
header = _make_header()
|
|
|
|
# Mock run_specialists 返回全 error
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _all_error_reports()
|
|
|
|
# Mock _agent_provider
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
return MagicMock(model="test-model")
|
|
|
|
# Mock load_match_header
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
# Mock _insert_or_find_by_fingerprint — 捕获写入的 status
|
|
captured_status = {}
|
|
|
|
async def mock_upsert(session, **kw):
|
|
captured_status.update(kw.get("values", {}))
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = "test"
|
|
mock_pred.prompt_version = "v1"
|
|
mock_pred.pred_home_goals = None
|
|
mock_pred.pred_away_goals = None
|
|
mock_pred.alt_pred_home_goals = None
|
|
mock_pred.alt_pred_away_goals = None
|
|
mock_pred.pred_1x2 = None
|
|
mock_pred.subjective_confidence = None
|
|
mock_pred.reasoning = kw["values"].get("reasoning")
|
|
mock_pred.agent_outputs = []
|
|
return mock_pred
|
|
|
|
# Mock get_uow
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock() # match exists
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
result = await orch_mod.predict_match_multi(999)
|
|
|
|
# 断言:status 是 degraded,不是 success
|
|
assert captured_status.get("status") == "degraded", \
|
|
f"期望 status=degraded,实际 {captured_status.get('status')}"
|
|
# 断言:reasoning 包含说明
|
|
assert "专家" in captured_status.get("reasoning", ""), \
|
|
f"reasoning 应说明原因,实际 {captured_status.get('reasoning')}"
|
|
# 断言:pred_* 全为 None
|
|
assert captured_status.get("pred_home_goals") is None
|
|
assert captured_status.get("pred_1x2") is None
|
|
print(f"PASS: 全 error → status={captured_status.get('status')}")
|
|
print(f" reasoning={captured_status.get('reasoning')[:50]}...")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_no_data_reports_yields_degraded(self):
|
|
"""5 个专家全 no_data → status=degraded,不调终裁。"""
|
|
header = _make_header()
|
|
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _all_no_data_reports()
|
|
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
return MagicMock(model="test-model")
|
|
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
captured_status = {}
|
|
|
|
async def mock_upsert(session, **kw):
|
|
captured_status.update(kw.get("values", {}))
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = "test"
|
|
mock_pred.prompt_version = "v1"
|
|
mock_pred.pred_home_goals = None
|
|
mock_pred.pred_away_goals = None
|
|
mock_pred.alt_pred_home_goals = None
|
|
mock_pred.alt_pred_away_goals = None
|
|
mock_pred.pred_1x2 = None
|
|
mock_pred.subjective_confidence = None
|
|
mock_pred.reasoning = kw["values"].get("reasoning")
|
|
mock_pred.agent_outputs = []
|
|
return mock_pred
|
|
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock()
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
result = await orch_mod.predict_match_multi(999)
|
|
|
|
assert captured_status.get("status") == "degraded", \
|
|
f"期望 status=degraded,实际 {captured_status.get('status')}"
|
|
print(f"PASS: 全 no_data → status={captured_status.get('status')}")
|
|
|
|
|
|
class TestPartialExpertsOk:
|
|
"""部分专家 ok 时,status 仍可为 success。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_one_ok_report_allows_success(self):
|
|
"""1 个 ok + 4 个 error → status=success(走终裁)。"""
|
|
header = _make_header()
|
|
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _mixed_reports()
|
|
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
return MagicMock(model="test-model")
|
|
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
captured_status = {}
|
|
|
|
async def mock_aggregator(header, reports, *, provider, version):
|
|
# 终裁返回合法 JSON
|
|
return {
|
|
"pred_home_goals": 2,
|
|
"pred_away_goals": 1,
|
|
"pred_1x2": "1",
|
|
"subjective_confidence": 0.7,
|
|
"reasoning": "test prediction",
|
|
"agent_weights": {"form": 0.5},
|
|
}, 100, 50
|
|
|
|
async def mock_upsert(session, **kw):
|
|
captured_status.update(kw.get("values", {}))
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = "test"
|
|
mock_pred.prompt_version = "v1"
|
|
mock_pred.pred_home_goals = 2
|
|
mock_pred.pred_away_goals = 1
|
|
mock_pred.pred_1x2 = "1"
|
|
mock_pred.subjective_confidence = 0.7
|
|
mock_pred.reasoning = "test"
|
|
return mock_pred
|
|
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock()
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
|
patch.object(orch_mod, "run_aggregator", mock_aggregator), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
result = await orch_mod.predict_match_multi(999)
|
|
|
|
assert captured_status.get("status") == "success", \
|
|
f"期望 status=success,实际 {captured_status.get('status')}"
|
|
print(f"PASS: 1 ok + 4 error → status={captured_status.get('status')}")
|
|
|
|
|
|
class TestNoAggregatorCallOnDegraded:
|
|
"""全专家失败时,aggregator provider 不应被调用。"""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_all_error_no_aggregator_call(self):
|
|
"""5 个专家全 error → 不应调用 _agent_provider('aggregator')。"""
|
|
header = _make_header()
|
|
aggregator_called = []
|
|
|
|
async def mock_run_specialists(header, *, version, before, model_override=None):
|
|
return _all_error_reports()
|
|
|
|
async def mock_agent_provider(agent_id, *, tier):
|
|
aggregator_called.append((agent_id, tier))
|
|
return MagicMock(model="test-model")
|
|
|
|
async def mock_load_header(mid, db=None):
|
|
return header
|
|
|
|
captured_values = {}
|
|
|
|
async def mock_upsert(session, **kw):
|
|
# model / provider_name / mode 是 _insert_or_find_by_fingerprint 的顶层关键字参数,
|
|
# 不在 values 字典里(见 orchestrator.py 的调用点)。原测试只取
|
|
# kw["values"],导致 model 断言永远为 None。
|
|
captured_values.update(kw.get("values", {}))
|
|
captured_values.update(
|
|
{k: kw.get(k) for k in ("model", "provider_name", "mode", "run_type")}
|
|
)
|
|
mock_pred = MagicMock()
|
|
mock_pred.id = 1
|
|
mock_pred.provider = "test"
|
|
mock_pred.model = kw.get("model")
|
|
return mock_pred
|
|
|
|
class FakeUow:
|
|
async def __aenter__(self):
|
|
return self
|
|
async def __aexit__(self, *a):
|
|
pass
|
|
async def get(self, cls, id):
|
|
return MagicMock()
|
|
|
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
|
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
|
patch.object(orch_mod, "get_uow", FakeUow):
|
|
|
|
await orch_mod.predict_match_multi(999)
|
|
|
|
# 断言:aggregator provider 未被调用
|
|
assert len(aggregator_called) == 0, \
|
|
f"全失败时不应调用 aggregator provider,实际调用: {aggregator_called}"
|
|
# P0-03:degraded 路径 status=degraded(model 可能为 None,由 aggregator 降级逻辑决定)
|
|
assert captured_values.get("status") == "degraded"
|
|
print(f"PASS: 全失败 → aggregator provider 未调用,status={captured_values.get('status')}")
|