"""D1 工程债回归测试: events 成功路径必须写 Bronze 层(RawEvent + DataLineage)。 背景: stats 回填管线早已有 RawEvent/DataLineage 写入,但 events 管线(比赛 主数据的唯一入口)成功插入/更新后既不留原始载荷,也不留血缘 —— 数据溯源 链条在最关键的一环断掉。本测试守护: 1. 插入新比赛 → RawEvent(幂等键=source_event_id 或合成键) + Lineage (target_table="matches", transform_name="events_ingest") 2. 变更更新(如补比分/状态) → 同样写血缘 3. 无变化跳过 → 不写(避免 lineage 刷屏) 4. RawEvent 幂等: 同 source_record_id 已存在则跳过 5. 基础设施写入失败 → 只 warning,不拖垮采集主流程 范式: 假 db(按查询实体分发预置数据 + 记录 add,flush 分配自增 id) + monkeypatch 抓取函数,不依赖真实数据库。 """ from __future__ import annotations from datetime import date, datetime, timezone import pytest import src.data.bzzoiro as bz from src.db.models import DataLineage, League, Match, RawEvent, Team, TeamAlias def _event(eid=1001, status="finished", home="Arsenal", away="Chelsea", hs=2, as_=1): """构造一条最小合法的 bzzoiro /events/ 原始载荷。""" raw = { "event_date": "2026-09-20 15:00:00", "status": status, "home_team": home, "away_team": away, "home_score": hs, "away_score": as_, } if eid is not None: raw["id"] = eid return raw class _FakeResult: """支持 .scalars().all() / .scalar_one_or_none() 的最小假结果集。""" def __init__(self, items): self._items = list(items) def scalars(self): return self def __iter__(self): return iter(self._items) def all(self): return self._items def scalar_one_or_none(self): return self._items[0] if self._items else None def scalar(self): return None class _FakeDB: """按查询实体分发预置数据;记录 add();flush 为无 id 对象分配自增主键。""" def __init__(self, matches=(), teams=(), leagues=(), raw_events=()): self.added = [] self._by_entity = { Match: list(matches), Team: list(teams), League: list(leagues), RawEvent: list(raw_events), } self._teams_by_id: dict[int, Team] = {t.id: t for t in teams if getattr(t, "id", None)} self._aliases: dict[str, TeamAlias] = {} self._next_id = max((t.id for t in teams if getattr(t, "id", None)), default=0) def add(self, obj): self.added.append(obj) async def get(self, cls, key): if cls is Team: return self._teams_by_id.get(key) if cls is TeamAlias: return self._aliases.get(key) return None async def execute(self, stmt): entities = set() for d in (stmt.column_descriptions or []): entities.add(d.get("entity") or d.get("type")) for entity, items in self._by_entity.items(): if entity in entities: return _FakeResult(items) return _FakeResult([]) async def flush(self): for obj in self.added: if getattr(obj, "id", None) is None: self._next_id += 1 obj.id = self._next_id if isinstance(obj, Team) and obj.id is not None: self._teams_by_id[obj.id] = obj @pytest.fixture(autouse=True) def _no_request_interval(monkeypatch): monkeypatch.setattr(bz, "REQUEST_INTERVAL", 0) def _patch_fetch(monkeypatch, events): async def _fetch(league_code, **kwargs): return list(events) monkeypatch.setattr(bz, "fetch_bzzoiro_events", _fetch) def _matches(db): return [o for o in db.added if isinstance(o, Match)] def _raw_events(db): return [o for o in db.added if isinstance(o, RawEvent)] def _lineages(db): return [o for o in db.added if isinstance(o, DataLineage)] # ============================================================ # 1. 插入新比赛 → RawEvent + DataLineage # ============================================================ class TestEventsBronzeOnInsert: async def test_insert_writes_raw_event_and_lineage(self, monkeypatch): _patch_fetch(monkeypatch, [_event()]) db = _FakeDB() result = await bz.BzzoiroSource().ingest(db, leagues=["E0"]) assert result["total_inserted"] == 1 raws = _raw_events(db) assert len(raws) == 1 raw = raws[0] assert raw.source_system == "bzzoiro" assert raw.source_record_id == "1001" # 有上游 id 时直接用 assert raw.ingest_batch_id.startswith("bzzoiro-events-E0-") assert raw.raw_payload["id"] == 1001 # 原始载荷完整保留 lineages = _lineages(db) assert len(lineages) == 1 lin = lineages[0] assert lin.source_system == "bzzoiro" assert lin.source_record_id == "1001" assert lin.target_table == "matches" assert lin.target_id == _matches(db)[0].id assert lin.transform_name == "events_ingest" # RawEvent 与 Lineage 同批次,便于按批追溯 assert lin.batch_id == raw.ingest_batch_id async def test_missing_source_id_uses_synthetic_stable_key(self, monkeypatch): """上游 id 缺失时,用 (league:home:away:date) 合成稳定幂等键。""" _patch_fetch(monkeypatch, [_event(eid=None)]) db = _FakeDB() result = await bz.BzzoiroSource().ingest(db, leagues=["E0"]) assert result["total_inserted"] == 1 raws = _raw_events(db) assert len(raws) == 1 # 期望键基于 normalize 后的队名与天级日期 —— 与 _match_key 同口径, # 不依赖 DB 自增 id,跨批次可复现 nm = bz.normalize_bzzoiro(_event(eid=None), "E0") expected = f"E0:{nm.home_team}:{nm.away_team}:{nm.date.date().isoformat()}" assert raws[0].source_record_id == expected async def test_existing_raw_event_is_skipped(self, monkeypatch): """RawEvent 幂等: 同 source_record_id 已存在则不再新增,但血缘照写。""" existing = RawEvent( source_system="bzzoiro", source_record_id="1001", raw_payload={"old": True}, ) _patch_fetch(monkeypatch, [_event()]) db = _FakeDB(raw_events=[existing]) await bz.BzzoiroSource().ingest(db, leagues=["E0"]) new_raws = [r for r in _raw_events(db) if r is not existing] assert new_raws == [] assert len(_lineages(db)) == 1 # 血缘仍然记录本次采集 # ============================================================ # 2. 变更更新 → 写血缘;无变化 → 不写 # ============================================================ class TestEventsBronzeOnUpdate: def _existing_match(self, **overrides): m = Match( league_id=1, home_team_id=2, # 与本轮 Team 创建后 fake 自增 id 对齐(league=1, home=2, away=3) away_team_id=3, match_date=datetime(2026, 9, 20, 15, 0, tzinfo=timezone.utc), match_date_date=date(2026, 9, 20), match_status="scheduled", source_event_id=1001, ) m.id = 42 for k, v in overrides.items(): setattr(m, k, v) return m async def test_changed_update_writes_lineage(self, monkeypatch): # 已有比赛处于 scheduled 且无比分;新载荷为 finished 2:1 → 触发变更更新 db = _FakeDB(matches=[self._existing_match()]) _patch_fetch(monkeypatch, [_event()]) result = await bz.BzzoiroSource().ingest(db, leagues=["E0"]) assert result["total_inserted"] == 0 assert result["leagues"]["E0"]["updated"] == 1 lineages = _lineages(db) assert len(lineages) == 1 assert lineages[0].target_id == 42 assert lineages[0].target_table == "matches" assert lineages[0].transform_name == "events_ingest" async def test_unchanged_match_writes_nothing(self, monkeypatch): # 已有比赛与新载荷完全一致 → 无变化,不应产生 RawEvent/Lineage existing = self._existing_match( match_status="finished", home_goals=2, away_goals=1, ) db = _FakeDB(matches=[existing]) _patch_fetch(monkeypatch, [_event()]) result = await bz.BzzoiroSource().ingest(db, leagues=["E0"]) assert result["total_inserted"] == 0 assert result["leagues"]["E0"]["updated"] == 0 assert _raw_events(db) == [] assert _lineages(db) == [] # ============================================================ # 3. 基础设施写入失败: 尽力而为,不拖垮主流程 # ============================================================ class TestEventsBronzeIsBestEffort: async def test_bronze_write_failure_does_not_break_ingest(self, monkeypatch): import src.data.bzzoiro_events as bz_events async def _boom(*args, **kwargs): raise RuntimeError("infra down") # Bronze 写入助手直接 import 到 bzzoiro_events 命名空间,需 patch 该处 monkeypatch.setattr(bz_events, "_write_raw_event", _boom) monkeypatch.setattr(bz_events, "_write_lineage", _boom) _patch_fetch(monkeypatch, [_event()]) db = _FakeDB() # 不应抛异常:Bronze 写不进去只记 warning result = await bz.BzzoiroSource().ingest(db, leagues=["E0"]) assert result["total_inserted"] == 1 assert len(_matches(db)) == 1