debt(D1): events 成功路径补写 Bronze 层(RawEvent + DataLineage)

- 新增 _events_record_id: 上游 id 缺失时用 (league:home:away:date) 合成稳定幂等键 - 新增 _write_events_bronze: best-effort 写 RawEvent(幂等) + Lineage(matches/events_ingest) - ingest 插入与变更更新后触发;同批 seen 集合防重复;基础设施失败只 warning - TDD: 6 测试(插入/合成键/幂等跳过/变更更新/无变化不写/失败不拖垮),双变异验证通过
2026-09-21 19:23:37 +08:00
parent f6c0145c32
commit 3a9f3f5a0e
2 changed files with 338 additions and 0 deletions
@@ -0,0 +1,259 @@
+"""D1 工程债回归测试: events 成功路径必须写 Bronze 层(RawEvent + DataLineage)。
+
+背景: stats 回填管线早已有 RawEvent/DataLineage 写入,但 events 管线(比赛
+主数据的唯一入口)成功插入/更新后既不留原始载荷,也不留血缘 —— 数据溯源
+链条在最关键的一环断掉。本测试守护:
+  1. 插入新比赛 → RawEvent(幂等键=source_event_id 或合成键) + Lineage
+     (target_table="matches", transform_name="events_ingest")
+  2. 变更更新(如补比分/状态) → 同样写血缘
+  3. 无变化跳过 → 不写(避免 lineage 刷屏)
+  4. RawEvent 幂等: 同 source_record_id 已存在则跳过
+  5. 基础设施写入失败 → 只 warning,不拖垮采集主流程
+
+范式: 假 db(按查询实体分发预置数据 + 记录 add,flush 分配自增 id)
+ monkeypatch 抓取函数,不依赖真实数据库。
+"""
+from __future__ import annotations
+
+from datetime import date, datetime, timezone
+
+import pytest
+
+import src.data.bzzoiro as bz
+from src.db.models import DataLineage, League, Match, RawEvent, Team
+
+
+def _event(eid=1001, status="finished", home="Arsenal", away="Chelsea", hs=2, as_=1):
+    """构造一条最小合法的 bzzoiro /events/ 原始载荷。"""
+    raw = {
+        "event_date": "2026-09-20 15:00:00",
+        "status": status,
+        "home_team": home,
+        "away_team": away,
+        "home_score": hs,
+        "away_score": as_,
+    }
+    if eid is not None:
+        raw["id"] = eid
+    return raw
+
+
+class _FakeResult:
+    """支持 .scalars().all() / .scalar_one_or_none() 的最小假结果集。"""
+
+    def __init__(self, items):
+        self._items = list(items)
+
+    def scalars(self):
+        return self
+
+    def __iter__(self):
+        return iter(self._items)
+
+    def all(self):
+        return self._items
+
+    def scalar_one_or_none(self):
+        return self._items[0] if self._items else None
+
+    def scalar(self):
+        return None
+
+
+class _FakeDB:
+    """按查询实体分发预置数据;记录 add();flush 为无 id 对象分配自增主键。"""
+
+    def __init__(self, matches=(), teams=(), leagues=(), raw_events=()):
+        self.added = []
+        self._by_entity = {
+            Match: list(matches),
+            Team: list(teams),
+            League: list(leagues),
+            RawEvent: list(raw_events),
+        }
+        self._next_id = 0
+
+    def add(self, obj):
+        self.added.append(obj)
+
+    async def execute(self, stmt):
+        entities = set()
+        for d in (stmt.column_descriptions or []):
+            entities.add(d.get("entity") or d.get("type"))
+        for entity, items in self._by_entity.items():
+            if entity in entities:
+                return _FakeResult(items)
+        return _FakeResult([])
+
+    async def flush(self):
+        for obj in self.added:
+            if getattr(obj, "id", None) is None:
+                self._next_id += 1
+                obj.id = self._next_id
+
+
+@pytest.fixture(autouse=True)
+def _no_request_interval(monkeypatch):
+    monkeypatch.setattr(bz, "REQUEST_INTERVAL", 0)
+
+
+def _patch_fetch(monkeypatch, events):
+    async def _fetch(league_code, **kwargs):
+        return list(events)
+
+    monkeypatch.setattr(bz, "fetch_bzzoiro_events", _fetch)
+
+
+def _matches(db):
+    return [o for o in db.added if isinstance(o, Match)]
+
+
+def _raw_events(db):
+    return [o for o in db.added if isinstance(o, RawEvent)]
+
+
+def _lineages(db):
+    return [o for o in db.added if isinstance(o, DataLineage)]
+
+
+# ============================================================
+# 1. 插入新比赛 → RawEvent + DataLineage
+# ============================================================
+
+
+class TestEventsBronzeOnInsert:
+    async def test_insert_writes_raw_event_and_lineage(self, monkeypatch):
+        _patch_fetch(monkeypatch, [_event()])
+        db = _FakeDB()
+
+        result = await bz.BzzoiroSource().ingest(db, leagues=["E0"])
+
+        assert result["total_inserted"] == 1
+
+        raws = _raw_events(db)
+        assert len(raws) == 1
+        raw = raws[0]
+        assert raw.source_system == "bzzoiro"
+        assert raw.source_record_id == "1001"  # 有上游 id 时直接用
+        assert raw.ingest_batch_id.startswith("bzzoiro-events-E0-")
+        assert raw.raw_payload["id"] == 1001  # 原始载荷完整保留
+
+        lineages = _lineages(db)
+        assert len(lineages) == 1
+        lin = lineages[0]
+        assert lin.source_system == "bzzoiro"
+        assert lin.source_record_id == "1001"
+        assert lin.target_table == "matches"
+        assert lin.target_id == _matches(db)[0].id
+        assert lin.transform_name == "events_ingest"
+        # RawEvent 与 Lineage 同批次,便于按批追溯
+        assert lin.batch_id == raw.ingest_batch_id
+
+    async def test_missing_source_id_uses_synthetic_stable_key(self, monkeypatch):
+        """上游 id 缺失时,用 (league:home:away:date) 合成稳定幂等键。"""
+        _patch_fetch(monkeypatch, [_event(eid=None)])
+        db = _FakeDB()
+
+        result = await bz.BzzoiroSource().ingest(db, leagues=["E0"])
+
+        assert result["total_inserted"] == 1
+        raws = _raw_events(db)
+        assert len(raws) == 1
+        # 期望键基于 normalize 后的队名与天级日期 —— 与 _match_key 同口径,
+        # 不依赖 DB 自增 id,跨批次可复现
+        nm = bz.normalize_bzzoiro(_event(eid=None), "E0")
+        expected = f"E0:{nm.home_team}:{nm.away_team}:{nm.date.date().isoformat()}"
+        assert raws[0].source_record_id == expected
+
+    async def test_existing_raw_event_is_skipped(self, monkeypatch):
+        """RawEvent 幂等: 同 source_record_id 已存在则不再新增,但血缘照写。"""
+        existing = RawEvent(
+            source_system="bzzoiro",
+            source_record_id="1001",
+            raw_payload={"old": True},
+        )
+        _patch_fetch(monkeypatch, [_event()])
+        db = _FakeDB(raw_events=[existing])
+
+        await bz.BzzoiroSource().ingest(db, leagues=["E0"])
+
+        new_raws = [r for r in _raw_events(db) if r is not existing]
+        assert new_raws == []
+        assert len(_lineages(db)) == 1  # 血缘仍然记录本次采集
+
+
+# ============================================================
+# 2. 变更更新 → 写血缘;无变化 → 不写
+# ============================================================
+
+
+class TestEventsBronzeOnUpdate:
+    def _existing_match(self, **overrides):
+        m = Match(
+            league_id=1,
+            home_team_id=2,  # 与本轮 Team 创建后 fake 自增 id 对齐(league=1, home=2, away=3)
+            away_team_id=3,
+            match_date=datetime(2026, 9, 20, 15, 0, tzinfo=timezone.utc),
+            match_date_date=date(2026, 9, 20),
+            match_status="scheduled",
+            source_event_id=1001,
+        )
+        m.id = 42
+        for k, v in overrides.items():
+            setattr(m, k, v)
+        return m
+
+    async def test_changed_update_writes_lineage(self, monkeypatch):
+        # 已有比赛处于 scheduled 且无比分;新载荷为 finished 2:1 → 触发变更更新
+        db = _FakeDB(matches=[self._existing_match()])
+        _patch_fetch(monkeypatch, [_event()])
+
+        result = await bz.BzzoiroSource().ingest(db, leagues=["E0"])
+
+        assert result["total_inserted"] == 0
+        assert result["leagues"]["E0"]["updated"] == 1
+
+        lineages = _lineages(db)
+        assert len(lineages) == 1
+        assert lineages[0].target_id == 42
+        assert lineages[0].target_table == "matches"
+        assert lineages[0].transform_name == "events_ingest"
+
+    async def test_unchanged_match_writes_nothing(self, monkeypatch):
+        # 已有比赛与新载荷完全一致 → 无变化,不应产生 RawEvent/Lineage
+        existing = self._existing_match(
+            match_status="finished",
+            home_goals=2,
+            away_goals=1,
+        )
+        db = _FakeDB(matches=[existing])
+        _patch_fetch(monkeypatch, [_event()])
+
+        result = await bz.BzzoiroSource().ingest(db, leagues=["E0"])
+
+        assert result["total_inserted"] == 0
+        assert result["leagues"]["E0"]["updated"] == 0
+        assert _raw_events(db) == []
+        assert _lineages(db) == []
+
+
+# ============================================================
+# 3. 基础设施写入失败: 尽力而为,不拖垮主流程
+# ============================================================
+
+
+class TestEventsBronzeIsBestEffort:
+    async def test_bronze_write_failure_does_not_break_ingest(self, monkeypatch):
+        async def _boom(*args, **kwargs):
+            raise RuntimeError("infra down")
+
+        monkeypatch.setattr(bz, "_write_raw_event", _boom)
+        monkeypatch.setattr(bz, "_write_lineage", _boom)
+        _patch_fetch(monkeypatch, [_event()])
+        db = _FakeDB()
+
+        # 不应抛异常:Bronze 写不进去只记 warning
+        result = await bz.BzzoiroSource().ingest(db, leagues=["E0"])
+
+        assert result["total_inserted"] == 1
+        assert len(_matches(db)) == 1