fix(critical): 恢复 bzzoiro events 采集管线,消除数据源注册表静默失效
C2 数据丢失修复。重构提交6a49940删除了 src/data/bzzoiro.py 中的 fetch_bzzoiro_events 与 BzzoiroSource 后未做替换,同时 sources.py 用 `try/except Exception: pass` 吞掉了 ImportError,导致: - _SOURCES 注册表恒为空 - get_source("bzzoiro") 恒抛 ValueError("未知数据源: bzzoiro") - src/api/routes/ingest.py 与 schedules.py 运行时全线失效, 且日志中无任何导入错误痕迹 修复内容: 1. 从f05dc1a恢复并移植 events 管线: - fetch_bzzoiro_events():分页抓取 /events/(纯异步,复用现有 _fetch_json_async) - @register class BzzoiroSource + ingest():保留原签名 (db, *, leagues, date_from, date_to, status),保持「不 commit, 事务由调用方 UnitOfWork 控制」的契约,并保留单联赛抓取失败的 错误隔离(记录 error 后 continue) - 复用现有 _to_date / _to_int_or_none / _match_key,未重复定义 - 血缘字段继续使用与 nm 配对的 raw(避免 P0-3 回归) 2. sources.py:导入失败改为 logger.exception 记录,新增 _loaded 标记 保证 _load_sources() 至多执行一次;失败时不置位以便后续重试。 get_source() 对真正未知的名字仍抛 ValueError。 验证: - 新增 tests/test_bzzoiro_source_registry.py(8 项):注册表非空、 get_source/list_sources 可用、ingest 签名契约、不再静默吞异常 - test_regressions.py:3 failed -> 2 failed - 全量:14 failed -> 13 failed,通过数 190 -> 199 注意事项:未改动任何 caller(routes/ingest.py、routes/schedules.py 原样 可用,证明签名保持正确)。
This commit is contained in:
@@ -0,0 +1,92 @@
|
||||
"""回归测试:锁定 bzzoiro events 管线被删除 + 注册表静默失效的缺陷不再复发。
|
||||
|
||||
背景(真实数据丢失事故):
|
||||
重构提交 6a49940 删除了 src/data/bzzoiro.py 中的 fetch_bzzoiro_events 与
|
||||
BzzoiroSource,但 src/data/sources.py 的模块级预热把 ImportError 用
|
||||
`try/except Exception: pass` 吞掉了。后果:
|
||||
- _SOURCES 注册表恒为空
|
||||
- get_source("bzzoiro") 恒抛 ValueError("未知数据源: bzzoiro")
|
||||
- src/api/routes/ingest.py 与 schedules.py 在运行时全线失效,
|
||||
且日志中看不到任何导入错误的痕迹 —— 这才是它长期漏网的原因。
|
||||
|
||||
本测试用静态结构断言 + 真实导入来锁死这两点,不 mock 网络:
|
||||
- 注册表必须非空(直接守卫「静默吞异常」回归)
|
||||
- get_source("bzzoiro") 必须返回真实实例
|
||||
- ingest() 的签名必须保持 caller 依赖的 keyword-only 参数
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from src.data.bzzoiro import BzzoiroSource, fetch_bzzoiro_events
|
||||
from src.data.sources import get_source, list_sources
|
||||
|
||||
SRC = Path(__file__).resolve().parent.parent / "src"
|
||||
|
||||
|
||||
class TestSourceRegistry:
|
||||
"""P0: 注册表必须真的装载到 bzzoiro,而不是静默为空。"""
|
||||
|
||||
def test_registry_is_not_empty(self):
|
||||
"""注册表恒非空 —— 直接守卫 `try/except: pass` 静默吞 ImportError。"""
|
||||
assert list_sources(), (
|
||||
"数据源注册表为空 —— _load_sources() 的导入失败了,"
|
||||
"且(修复前)异常被静默吞掉。任何导入错误都必须被 logger.exception 记录。"
|
||||
)
|
||||
|
||||
def test_get_source_returns_bzzoiro(self):
|
||||
"""get_source('bzzoiro') 必须返回实例,而不是抛 ValueError。"""
|
||||
source = get_source("bzzoiro")
|
||||
assert source.name == "bzzoiro"
|
||||
|
||||
def test_list_sources_contains_bzzoiro(self):
|
||||
assert "bzzoiro" in list_sources()
|
||||
|
||||
def test_register_decorator_still_exports_the_class(self):
|
||||
"""@register 必须仍然返回原类(不能再被改成返回实例)。"""
|
||||
assert isinstance(BzzoiroSource, type), "@register 不应把类替换成实例"
|
||||
|
||||
|
||||
class TestIngestContract:
|
||||
"""caller 依赖 ingest() 的签名,routes 传的就是这些参数。"""
|
||||
|
||||
def test_ingest_exists_and_is_async(self):
|
||||
assert hasattr(BzzoiroSource, "ingest"), "BzzoiroSource.ingest 丢失(events 管线被删)"
|
||||
assert inspect.iscoroutinefunction(BzzoiroSource.ingest), "ingest 必须是 async"
|
||||
|
||||
def test_ingest_accepts_caller_keyword_args(self):
|
||||
"""ingest.py:65 与 schedules.py:41 依赖的 keyword-only 参数必须齐全。"""
|
||||
params = inspect.signature(BzzoiroSource.ingest).parameters
|
||||
for name in ("leagues", "date_from", "date_to", "status"):
|
||||
assert name in params, f"ingest() 缺少 keyword 参数 {name!r} —— caller 会 TypeError"
|
||||
# leagues 必须 keyword-only(src/api/routes/ingest.py 用 leagues=[code] 传)
|
||||
assert params["leagues"].kind is inspect.Parameter.KEYWORD_ONLY
|
||||
# 默认值契约:schedules.py 只传 leagues + status
|
||||
assert params["date_from"].default is None
|
||||
assert params["date_to"].default is None
|
||||
assert params["status"].default == "finished"
|
||||
|
||||
def test_fetch_bzzoiro_events_signature(self):
|
||||
"""抓取函数也必须存在,且 leagues 抓取走 keyword-only 的 status/日期。"""
|
||||
assert callable(fetch_bzzoiro_events)
|
||||
params = inspect.signature(fetch_bzzoiro_events).parameters
|
||||
assert "league_code" in params
|
||||
for name in ("status", "date_from", "date_to"):
|
||||
assert name in params, f"fetch_bzzoiro_events() 缺少 {name!r}"
|
||||
|
||||
|
||||
class TestNoSilentImportSwallow:
|
||||
"""sources.py 不许再用 `except Exception: pass` 吞掉导入失败。"""
|
||||
|
||||
def test_load_sources_logs_failure(self):
|
||||
src = (SRC / "data" / "sources.py").read_text(encoding="utf-8")
|
||||
assert "logger" in src, "sources.py 缺少模块 logger,无法记录导入失败"
|
||||
body = src[src.index("def _load_sources"):]
|
||||
assert "logger.exception" in body, (
|
||||
"_load_sources() 失败时未记日志 —— 静默吞异常会让注册表恒为空,"
|
||||
"把 ImportError 伪装成「未知数据源」(P0 事故根因)"
|
||||
)
|
||||
assert "pass" not in body.split("def ")[0], "不得再用裸 pass 吞掉导入异常"
|
||||
Reference in New Issue
Block a user