"""采集任务状态(ingest_jobs)测试。 背景: POST /ingest/bzzoiro 此前 fire-and-forget —— 触发后只能翻系统日志, 无法程序化查询「这次采集跑到哪了/成没成」。本次改造: 1. 路由创建 pending job → 响应返回 job_id 2. 后台 _run_bzzoiro 维护 running → success / failed + result/error 3. admin 端点 /admin/ingest/jobs/{job_id} 与列表可查询 守护点: - job 更新是「尽力而为」: _update_job 自身失败被吞掉,不影响采集主流程 - job(任务级)与 IngestFailure 死信(行级)相互独立,可同时存在 (死信路径由 test_ingest_deadletter.py 守护,本文件不动 bzzoiro 内部) 范式: 假 UoW(记录 add / 返回预设 job)+ monkeypatch source,不依赖真实数据库 (与 test_ingest_deadletter.py / test_public_readonly_api.py 相同)。 """ from __future__ import annotations from contextlib import asynccontextmanager from datetime import datetime, timezone from fastapi import FastAPI from fastapi.testclient import TestClient import src.api.routes.ingest as ingest import src.api.routes.schedules as schedules from src.api.deps import require_admin from src.api.schemas import IngestBzzoiroRequest from src.db.base import get_db_read from src.db.models import IngestJob # ── 假基础设施 ────────────────────────────────────────────────── class _FakeResult: """支持 .scalars().all() / .scalar_one_or_none() / .scalar() 的最小假结果集。""" def __init__(self, items): self._items = items def scalars(self): return self def all(self): return self._items def scalar_one_or_none(self): return self._items[0] if self._items else None def scalar(self): return None class _FakeSession: """记录 add();execute 按预设队列依次返回(与真实 UoW 的单会话用法对齐)。""" def __init__(self, results=None): self.added = [] self._results = list(results or []) def add(self, obj): self.added.append(obj) async def execute(self, stmt): if self._results: return self._results.pop(0) return _FakeResult([]) def _patch_uow(monkeypatch, session) -> None: """把 ingest 模块的 get_uow 指向假会话。""" @asynccontextmanager async def _fake(): yield session monkeypatch.setattr(ingest, "get_uow", _fake) def _ingest_app() -> TestClient: app = FastAPI() app.include_router(ingest.router) app.dependency_overrides[require_admin] = lambda: None return TestClient(app) def _admin_app(fake_db) -> TestClient: app = FastAPI() app.include_router(schedules.router) app.dependency_overrides[require_admin] = lambda: None app.dependency_overrides[get_db_read] = lambda: fake_db return TestClient(app) def _job(jid: str = "job-1", status: str = "success", **kw) -> IngestJob: return IngestJob( id=jid, task=kw.pop("task", "standings"), params=kw.pop("params", {"leagues": ["E0"]}), status=status, result=kw.pop("result", {"total_upserted": 20}), error=kw.pop("error", None), created_at=kw.pop("created_at", datetime.now(timezone.utc)), **kw, ) # ── 1. 路由:创建 job + 返回 job_id ───────────────────────────── class TestRouteCreatesJob: def test_post_returns_job_id_and_persists_pending_job(self, monkeypatch): session = _FakeSession() _patch_uow(monkeypatch, session) spawned: list = [] monkeypatch.setattr(ingest, "_spawn", lambda coro: spawned.append(coro)) client = _ingest_app() resp = client.post( "/api/v1/ingest/bzzoiro", json={"task": "standings", "leagues": ["E0", "SP1"], "season": "2026-2027"}, ) assert resp.status_code == 200 body = resp.json() assert body["ok"] is True jobs = [o for o in session.added if isinstance(o, IngestJob)] assert len(jobs) == 1 job = jobs[0] assert body["job_id"] == job.id assert job.status == "pending" assert job.task == "standings" # params 记录的是「实际将执行」的参数(含默认联赛展开) assert job.params["leagues"] == ["E0", "SP1"] assert job.params["season"] == "2026-2027" # 后台协程被捕获但未执行;显式关闭避免 un-awaited 告警 assert len(spawned) == 1 spawned[0].close() def test_post_invalid_task_422_and_no_job(self, monkeypatch): session = _FakeSession() _patch_uow(monkeypatch, session) monkeypatch.setattr(ingest, "_spawn", lambda coro: coro.close()) client = _ingest_app() resp = client.post("/api/v1/ingest/bzzoiro", json={"task": "bogus"}) assert resp.status_code == 422 assert not [o for o in session.added if isinstance(o, IngestJob)] # ── 2. 后台执行:状态流转 ──────────────────────────────────────── class TestRunBzzoiroJobLifecycle: async def test_success_flow_updates_job_running_then_success(self, monkeypatch): job = _job(jid="job-ok", status="pending", task="events") session = _FakeSession(results=[_FakeResult([job])] * 10) _patch_uow(monkeypatch, session) class _FakeSource: async def ingest(self, session, **kw): return {"inserted": 3, "updated": 1, "total_inserted": 3, "total_updated": 1, "errors": ["e1", "e2"]} monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource()) req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished") await ingest._run_bzzoiro("job-ok", "events", ["E0"], req) assert job.status == "success" assert job.started_at is not None assert job.finished_at is not None assert job.error is None # result 含 events 子任务摘要,errors 截断到前 10 条 assert job.result["events"]["total_inserted"] == 3 assert job.result["events"]["errors"] == ["e1", "e2"] async def test_failure_flow_marks_failed_with_error(self, monkeypatch): job = _job(jid="job-bad", status="running", task="standings", result=None) session = _FakeSession(results=[_FakeResult([job])] * 10) _patch_uow(monkeypatch, session) async def _boom(*args, **kwargs): raise RuntimeError("network down") monkeypatch.setattr(ingest, "ingest_bzzoiro_standings", _boom) req = IngestBzzoiroRequest(task="standings", leagues=["E0"]) await ingest._run_bzzoiro("job-bad", "standings", ["E0"], req) assert job.status == "failed" assert "network down" in job.error assert job.finished_at is not None assert job.result is None async def test_all_task_collects_per_task_summaries(self, monkeypatch): job = _job(jid="job-all", status="pending", task="all") session = _FakeSession(results=[_FakeResult([job])] * 20) _patch_uow(monkeypatch, session) class _FakeSource: async def ingest(self, session, **kw): return {"inserted": 1, "updated": 0, "total_inserted": 1, "total_updated": 0, "errors": []} async def _standings(*args, **kwargs): return {"total_upserted": 20, "errors": []} async def _stats(*args, **kwargs): return {"total_inserted": 5, "total_updated": 2, "errors": []} monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource()) monkeypatch.setattr(ingest, "ingest_bzzoiro_standings", _standings) monkeypatch.setattr(ingest, "ingest_bzzoiro_event_stats", _stats) req = IngestBzzoiroRequest(task="all", leagues=["E0"]) await ingest._run_bzzoiro("job-all", "all", ["E0"], req) assert job.status == "success" assert set(job.result.keys()) == {"events", "standings", "stats"} assert job.result["standings"]["total_upserted"] == 20 assert job.result["stats"]["total_inserted"] == 5 async def test_update_job_failure_does_not_break_ingest(self, monkeypatch): """_update_job 抛错必须被吞掉:job 可观测性失败 ≠ 采集失败。""" session = _FakeSession() _patch_uow(monkeypatch, session) class _FakeSource: async def ingest(self, session, **kw): return {"inserted": 1, "updated": 0, "total_inserted": 1, "total_updated": 0, "errors": []} monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource()) # 让 execute 抛错(_update_job 内部会捕获) async def _broken_execute(stmt): raise RuntimeError("db gone") session.execute = _broken_execute req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished") # 不抛异常即通过;采集逻辑本身照常跑完 await ingest._run_bzzoiro("job-x", "events", ["E0"], req) async def test_job_and_deadletter_are_independent_layers(self, monkeypatch): """任务级(job)与行级(死信)互不干扰:source 内部返回 errors 时, job 仍为 success(部分失败不算任务失败),死信由 bzzoiro 层另行记录。""" job = _job(jid="job-part", status="pending", task="events") session = _FakeSession(results=[_FakeResult([job])] * 10) _patch_uow(monkeypatch, session) class _FakeSource: async def ingest(self, session, **kw): # 模拟 bzzoiro 管线:单条失败已写死信,汇总 errors 非空但返回正常 return {"inserted": 9, "updated": 0, "total_inserted": 9, "total_updated": 0, "errors": ["league F1 fetch failed"]} monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource()) req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished") await ingest._run_bzzoiro("job-part", "events", ["E0"], req) assert job.status == "success" assert job.result["events"]["errors"] == ["league F1 fetch failed"] # ── 3. admin 查询端点 ────────────────────────────────────────── class TestAdminJobEndpoints: def test_get_job_detail_200(self, monkeypatch): job = _job(jid="abc-123", status="running") client = _admin_app(_FakeSession(results=[_FakeResult([job])])) resp = client.get("/api/v1/admin/ingest/jobs/abc-123") assert resp.status_code == 200 body = resp.json() assert body["id"] == "abc-123" assert body["status"] == "running" assert body["task"] == "standings" assert body["params"] == {"leagues": ["E0"]} assert body["result"] == {"total_upserted": 20} assert body["error"] is None assert body["created_at"] is not None def test_get_job_detail_404(self): client = _admin_app(_FakeSession(results=[_FakeResult([])])) resp = client.get("/api/v1/admin/ingest/jobs/missing") assert resp.status_code == 404 def test_list_jobs_returns_serialized_rows(self): j1 = _job(jid="j1", status="success") j2 = _job(jid="j2", status="failed", task="events", result=None, error="RuntimeError('x')") client = _admin_app(_FakeSession(results=[_FakeResult([j1, j2])])) resp = client.get("/api/v1/admin/ingest/jobs?limit=10") assert resp.status_code == 200 rows = resp.json() assert [r["id"] for r in rows] == ["j1", "j2"] assert rows[1]["status"] == "failed" assert rows[1]["error"] == "RuntimeError('x')" def test_job_routes_live_under_admin_router_with_require_admin(self): """结构守护:job 端点必须挂在 /api/v1/admin 路由(路由级 require_admin)。""" assert any(dep.dependency is require_admin for dep in schedules.router.dependencies) paths = {getattr(r, "path", "") for r in schedules.router.routes} assert "/api/v1/admin/ingest/jobs" in paths assert "/api/v1/admin/ingest/jobs/{job_id}" in paths