Profeto/tests/test_multi_agent_degraded.py

"""回归测试: multi-agent 全专家失败/无数据时 status=degraded。

验证:
  1. 5 个专家全 no_data/error → status="degraded",不调终裁
  2. 至少 1 个专家 ok → status="success",正常走终裁
"""
from __future__ import annotations

from unittest.mock import MagicMock, AsyncMock, patch

import pytest

from src.llm.agents.base import AgentReport
from src.llm.agents import orchestrator as orch_mod


def _make_header():
    from src.llm.context_builder import MatchHeader
    return MatchHeader(
        match_id=999, home_name="A", away_name="B",
        league_name="X", season=None, match_date="?",
        match_dt=None, stage=None,
        home_team_id=1, away_team_id=2, league_id=1,
    )


def _all_error_reports():
    """5 个专家全 error."""
    return [
        AgentReport(agent="form", status="error", analysis="slice failed"),
        AgentReport(agent="stats", status="error", analysis="slice failed"),
        AgentReport(agent="home_away", status="error", analysis="slice failed"),
        AgentReport(agent="standings", status="error", analysis="slice failed"),
        AgentReport(agent="h2h", status="error", analysis="slice failed"),
    ]


def _all_no_data_reports():
    """5 个专家全 no_data."""
    return [
        AgentReport(agent="form", status="no_data", analysis="无数据"),
        AgentReport(agent="stats", status="no_data", analysis="无数据"),
        AgentReport(agent="home_away", status="no_data", analysis="无数据"),
        AgentReport(agent="standings", status="no_data", analysis="无数据"),
        AgentReport(agent="h2h", status="no_data", analysis="无数据"),
    ]


def _mixed_reports():
    """1 个 ok,4 个 error."""
    return [
        AgentReport(agent="form", status="ok", analysis="good"),
        AgentReport(agent="stats", status="error", analysis="failed"),
        AgentReport(agent="home_away", status="no_data", analysis="无数据"),
        AgentReport(agent="standings", status="error", analysis="failed"),
        AgentReport(agent="h2h", status="no_data", analysis="无数据"),
    ]


class TestAllExpertsFailed:
    """全部专家失败/无数据时,status 应为 degraded。"""

    @pytest.mark.asyncio
    async def test_all_error_reports_yields_degraded(self):
        """5 个专家全 error → status=degraded,不调终裁。"""
        header = _make_header()

        # Mock run_specialists 返回全 error
        async def mock_run_specialists(header, *, version, before):
            return _all_error_reports()

        # Mock _agent_provider
        async def mock_agent_provider(agent_id, *, tier):
            return MagicMock(model="test-model")

        # Mock load_match_header
        async def mock_load_header(mid, db=None):
            return header

        # Mock _upsert_prediction — 捕获写入的 status
        captured_status = {}

        async def mock_upsert(session, **kw):
            captured_status.update(kw.get("values", {}))
            mock_pred = MagicMock()
            mock_pred.id = 1
            mock_pred.provider = "test"
            mock_pred.model = "test"
            mock_pred.prompt_version = "v1"
            mock_pred.pred_home_goals = None
            mock_pred.pred_away_goals = None
            mock_pred.alt_pred_home_goals = None
            mock_pred.alt_pred_away_goals = None
            mock_pred.pred_1x2 = None
            mock_pred.subjective_confidence = None
            mock_pred.reasoning = kw["values"].get("reasoning")
            mock_pred.agent_outputs = []
            return mock_pred

        # Mock get_uow
        class FakeUow:
            async def __aenter__(self):
                return self
            async def __aexit__(self, *a):
                pass
            async def get(self, cls, id):
                return MagicMock()  # match exists

        with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
             patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
             patch.object(orch_mod, "load_match_header", mock_load_header), \
             patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
             patch.object(orch_mod, "get_uow", FakeUow):

            result = await orch_mod.predict_match_multi(999)

        # 断言:status 是 degraded,不是 success
        assert captured_status.get("status") == "degraded", \
            f"期望 status=degraded,实际 {captured_status.get('status')}"
        # 断言:reasoning 包含说明
        assert "专家" in captured_status.get("reasoning", ""), \
            f"reasoning 应说明原因,实际 {captured_status.get('reasoning')}"
        # 断言:pred_* 全为 None
        assert captured_status.get("pred_home_goals") is None
        assert captured_status.get("pred_1x2") is None
        print(f"PASS: 全 error → status={captured_status.get('status')}")
        print(f"     reasoning={captured_status.get('reasoning')[:50]}...")

    @pytest.mark.asyncio
    async def test_all_no_data_reports_yields_degraded(self):
        """5 个专家全 no_data → status=degraded,不调终裁。"""
        header = _make_header()

        async def mock_run_specialists(header, *, version, before):
            return _all_no_data_reports()

        async def mock_agent_provider(agent_id, *, tier):
            return MagicMock(model="test-model")

        async def mock_load_header(mid, db=None):
            return header

        captured_status = {}

        async def mock_upsert(session, **kw):
            captured_status.update(kw.get("values", {}))
            mock_pred = MagicMock()
            mock_pred.id = 1
            mock_pred.provider = "test"
            mock_pred.model = "test"
            mock_pred.prompt_version = "v1"
            mock_pred.pred_home_goals = None
            mock_pred.pred_away_goals = None
            mock_pred.alt_pred_home_goals = None
            mock_pred.alt_pred_away_goals = None
            mock_pred.pred_1x2 = None
            mock_pred.subjective_confidence = None
            mock_pred.reasoning = kw["values"].get("reasoning")
            mock_pred.agent_outputs = []
            return mock_pred

        class FakeUow:
            async def __aenter__(self):
                return self
            async def __aexit__(self, *a):
                pass
            async def get(self, cls, id):
                return MagicMock()

        with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
             patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
             patch.object(orch_mod, "load_match_header", mock_load_header), \
             patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
             patch.object(orch_mod, "get_uow", FakeUow):

            result = await orch_mod.predict_match_multi(999)

        assert captured_status.get("status") == "degraded", \
            f"期望 status=degraded,实际 {captured_status.get('status')}"
        print(f"PASS: 全 no_data → status={captured_status.get('status')}")


class TestPartialExpertsOk:
    """部分专家 ok 时,status 仍可为 success。"""

    @pytest.mark.asyncio
    async def test_one_ok_report_allows_success(self):
        """1 个 ok + 4 个 error → status=success(走终裁)。"""
        header = _make_header()

        async def mock_run_specialists(header, *, version, before):
            return _mixed_reports()

        async def mock_agent_provider(agent_id, *, tier):
            return MagicMock(model="test-model")

        async def mock_load_header(mid, db=None):
            return header

        captured_status = {}

        async def mock_aggregator(header, reports, *, provider, version):
            # 终裁返回合法 JSON
            return {
                "pred_home_goals": 2,
                "pred_away_goals": 1,
                "pred_1x2": "1",
                "subjective_confidence": 0.7,
                "reasoning": "test prediction",
                "agent_weights": {"form": 0.5},
            }, 100, 50

        async def mock_upsert(session, **kw):
            captured_status.update(kw.get("values", {}))
            mock_pred = MagicMock()
            mock_pred.id = 1
            mock_pred.provider = "test"
            mock_pred.model = "test"
            mock_pred.prompt_version = "v1"
            mock_pred.pred_home_goals = 2
            mock_pred.pred_away_goals = 1
            mock_pred.pred_1x2 = "1"
            mock_pred.subjective_confidence = 0.7
            mock_pred.reasoning = "test"
            return mock_pred

        class FakeUow:
            async def __aenter__(self):
                return self
            async def __aexit__(self, *a):
                pass
            async def get(self, cls, id):
                return MagicMock()

        with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
             patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
             patch.object(orch_mod, "load_match_header", mock_load_header), \
             patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
             patch.object(orch_mod, "run_aggregator", mock_aggregator), \
             patch.object(orch_mod, "get_uow", FakeUow):

            result = await orch_mod.predict_match_multi(999)

        assert captured_status.get("status") == "success", \
            f"期望 status=success,实际 {captured_status.get('status')}"
        print(f"PASS: 1 ok + 4 error → status={captured_status.get('status')}")


class TestNoAggregatorCallOnDegraded:
    """全专家失败时,aggregator provider 不应被调用。"""

    @pytest.mark.asyncio
    async def test_all_error_no_aggregator_call(self):
        """5 个专家全 error → 不应调用 _agent_provider('aggregator')。"""
        header = _make_header()
        aggregator_called = []

        async def mock_run_specialists(header, *, version, before):
            return _all_error_reports()

        async def mock_agent_provider(agent_id, *, tier):
            aggregator_called.append((agent_id, tier))
            return MagicMock(model="test-model")

        async def mock_load_header(mid, db=None):
            return header

        captured_values = {}

        async def mock_upsert(session, **kw):
            captured_values.update(kw.get("values", {}))
            mock_pred = MagicMock()
            mock_pred.id = 1
            mock_pred.provider = "test"
            mock_pred.model = kw["values"].get("model")
            return mock_pred

        class FakeUow:
            async def __aenter__(self):
                return self
            async def __aexit__(self, *a):
                pass
            async def get(self, cls, id):
                return MagicMock()

        with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
             patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
             patch.object(orch_mod, "load_match_header", mock_load_header), \
             patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
             patch.object(orch_mod, "get_uow", FakeUow):

            await orch_mod.predict_match_multi(999)

        # 断言:aggregator provider 未被调用
        assert len(aggregator_called) == 0, \
            f"全失败时不应调用 aggregator provider,实际调用: {aggregator_called}"
        # 断言:model 使用 settings 默认值
        assert captured_values.get("model") is not None
        assert captured_values.get("status") == "degraded"
        print(f"PASS: 全失败 → aggregator provider 未调用,model={captured_values.get('model')}")