WorkBuddy
983363dab7
- 新表 ingest_jobs(迁移 0019): id UUID/task/params JSONB/
status(pending|running|success|failed,CheckConstraint)/
result JSONB(统计摘要)/error/created_at/started_at/finished_at
- POST /ingest/bzzoiro: 启动后台前创建 pending job,响应返回 job_id;
仍 require_admin。后台 _run_bzzoiro 流转 running→success/failed,
result 按子任务(events/standings/stats)记录摘要(errors 截断 10 条)
- _update_job 尽力而为: 状态更新失败只记日志,绝不拖垮采集主流程;
与 IngestFailure 死信独立(行级 vs 任务级,可同时存在)
- 新增 admin 端点(挂 /api/v1/admin 路由,路由级 require_admin):
GET /admin/ingest/jobs/{job_id} 与 GET /admin/ingest/jobs?limit&status
- 前端采集页: 提交后凭 job_id 3 秒轮询,终态展示结果摘要/失败原因;
无 job_id 时回退旧的 30 秒盲等 + 系统日志提示
- 测试 11 项: 建 job+job_id 契约、非法 task 422、成功/失败/all 流转、
update 失败不拖垮采集、部分失败仍 success、admin 端点 200/404/列表、
结构守护(job 路由在 admin 路由且带 require_admin)
- 禁止项确认: 未动分批 UoW、BzzoiroSource、死信与 Bronze 写入;
docs(01/03/05/07/README)同步 13 张表与端点说明
314 lines
12 KiB
Python
314 lines
12 KiB
Python
"""采集任务状态(ingest_jobs)测试。
|
|
|
|
背景: POST /ingest/bzzoiro 此前 fire-and-forget —— 触发后只能翻系统日志,
|
|
无法程序化查询「这次采集跑到哪了/成没成」。本次改造:
|
|
1. 路由创建 pending job → 响应返回 job_id
|
|
2. 后台 _run_bzzoiro 维护 running → success / failed + result/error
|
|
3. admin 端点 /admin/ingest/jobs/{job_id} 与列表可查询
|
|
|
|
守护点:
|
|
- job 更新是「尽力而为」: _update_job 自身失败被吞掉,不影响采集主流程
|
|
- job(任务级)与 IngestFailure 死信(行级)相互独立,可同时存在
|
|
(死信路径由 test_ingest_deadletter.py 守护,本文件不动 bzzoiro 内部)
|
|
|
|
范式: 假 UoW(记录 add / 返回预设 job)+ monkeypatch source,不依赖真实数据库
|
|
(与 test_ingest_deadletter.py / test_public_readonly_api.py 相同)。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from contextlib import asynccontextmanager
|
|
from datetime import datetime, timezone
|
|
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
|
|
import src.api.routes.ingest as ingest
|
|
import src.api.routes.schedules as schedules
|
|
from src.api.deps import require_admin
|
|
from src.api.schemas import IngestBzzoiroRequest
|
|
from src.db.base import get_db_read
|
|
from src.db.models import IngestJob
|
|
|
|
|
|
# ── 假基础设施 ──────────────────────────────────────────────────
|
|
|
|
|
|
class _FakeResult:
|
|
"""支持 .scalars().all() / .scalar_one_or_none() / .scalar() 的最小假结果集。"""
|
|
|
|
def __init__(self, items):
|
|
self._items = items
|
|
|
|
def scalars(self):
|
|
return self
|
|
|
|
def all(self):
|
|
return self._items
|
|
|
|
def scalar_one_or_none(self):
|
|
return self._items[0] if self._items else None
|
|
|
|
def scalar(self):
|
|
return None
|
|
|
|
|
|
class _FakeSession:
|
|
"""记录 add();execute 按预设队列依次返回(与真实 UoW 的单会话用法对齐)。"""
|
|
|
|
def __init__(self, results=None):
|
|
self.added = []
|
|
self._results = list(results or [])
|
|
|
|
def add(self, obj):
|
|
self.added.append(obj)
|
|
|
|
async def execute(self, stmt):
|
|
if self._results:
|
|
return self._results.pop(0)
|
|
return _FakeResult([])
|
|
|
|
|
|
def _patch_uow(monkeypatch, session) -> None:
|
|
"""把 ingest 模块的 get_uow 指向假会话。"""
|
|
|
|
@asynccontextmanager
|
|
async def _fake():
|
|
yield session
|
|
|
|
monkeypatch.setattr(ingest, "get_uow", _fake)
|
|
|
|
|
|
def _ingest_app() -> TestClient:
|
|
app = FastAPI()
|
|
app.include_router(ingest.router)
|
|
app.dependency_overrides[require_admin] = lambda: None
|
|
return TestClient(app)
|
|
|
|
|
|
def _admin_app(fake_db) -> TestClient:
|
|
app = FastAPI()
|
|
app.include_router(schedules.router)
|
|
app.dependency_overrides[require_admin] = lambda: None
|
|
app.dependency_overrides[get_db_read] = lambda: fake_db
|
|
return TestClient(app)
|
|
|
|
|
|
def _job(jid: str = "job-1", status: str = "success", **kw) -> IngestJob:
|
|
return IngestJob(
|
|
id=jid,
|
|
task=kw.pop("task", "standings"),
|
|
params=kw.pop("params", {"leagues": ["E0"]}),
|
|
status=status,
|
|
result=kw.pop("result", {"total_upserted": 20}),
|
|
error=kw.pop("error", None),
|
|
created_at=kw.pop("created_at", datetime.now(timezone.utc)),
|
|
**kw,
|
|
)
|
|
|
|
|
|
# ── 1. 路由:创建 job + 返回 job_id ─────────────────────────────
|
|
|
|
|
|
class TestRouteCreatesJob:
|
|
def test_post_returns_job_id_and_persists_pending_job(self, monkeypatch):
|
|
session = _FakeSession()
|
|
_patch_uow(monkeypatch, session)
|
|
|
|
spawned: list = []
|
|
monkeypatch.setattr(ingest, "_spawn", lambda coro: spawned.append(coro))
|
|
|
|
client = _ingest_app()
|
|
resp = client.post(
|
|
"/api/v1/ingest/bzzoiro",
|
|
json={"task": "standings", "leagues": ["E0", "SP1"], "season": "2026-2027"},
|
|
)
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["ok"] is True
|
|
|
|
jobs = [o for o in session.added if isinstance(o, IngestJob)]
|
|
assert len(jobs) == 1
|
|
job = jobs[0]
|
|
assert body["job_id"] == job.id
|
|
assert job.status == "pending"
|
|
assert job.task == "standings"
|
|
# params 记录的是「实际将执行」的参数(含默认联赛展开)
|
|
assert job.params["leagues"] == ["E0", "SP1"]
|
|
assert job.params["season"] == "2026-2027"
|
|
|
|
# 后台协程被捕获但未执行;显式关闭避免 un-awaited 告警
|
|
assert len(spawned) == 1
|
|
spawned[0].close()
|
|
|
|
def test_post_invalid_task_422_and_no_job(self, monkeypatch):
|
|
session = _FakeSession()
|
|
_patch_uow(monkeypatch, session)
|
|
monkeypatch.setattr(ingest, "_spawn", lambda coro: coro.close())
|
|
|
|
client = _ingest_app()
|
|
resp = client.post("/api/v1/ingest/bzzoiro", json={"task": "bogus"})
|
|
assert resp.status_code == 422
|
|
assert not [o for o in session.added if isinstance(o, IngestJob)]
|
|
|
|
|
|
# ── 2. 后台执行:状态流转 ────────────────────────────────────────
|
|
|
|
|
|
class TestRunBzzoiroJobLifecycle:
|
|
async def test_success_flow_updates_job_running_then_success(self, monkeypatch):
|
|
job = _job(jid="job-ok", status="pending", task="events")
|
|
session = _FakeSession(results=[_FakeResult([job])] * 10)
|
|
_patch_uow(monkeypatch, session)
|
|
|
|
class _FakeSource:
|
|
async def ingest(self, session, **kw):
|
|
return {"inserted": 3, "updated": 1, "total_inserted": 3,
|
|
"total_updated": 1, "errors": ["e1", "e2"]}
|
|
|
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
|
|
|
req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished")
|
|
await ingest._run_bzzoiro("job-ok", "events", ["E0"], req)
|
|
|
|
assert job.status == "success"
|
|
assert job.started_at is not None
|
|
assert job.finished_at is not None
|
|
assert job.error is None
|
|
# result 含 events 子任务摘要,errors 截断到前 10 条
|
|
assert job.result["events"]["total_inserted"] == 3
|
|
assert job.result["events"]["errors"] == ["e1", "e2"]
|
|
|
|
async def test_failure_flow_marks_failed_with_error(self, monkeypatch):
|
|
job = _job(jid="job-bad", status="running", task="standings", result=None)
|
|
session = _FakeSession(results=[_FakeResult([job])] * 10)
|
|
_patch_uow(monkeypatch, session)
|
|
|
|
async def _boom(*args, **kwargs):
|
|
raise RuntimeError("network down")
|
|
|
|
monkeypatch.setattr(ingest, "ingest_bzzoiro_standings", _boom)
|
|
|
|
req = IngestBzzoiroRequest(task="standings", leagues=["E0"])
|
|
await ingest._run_bzzoiro("job-bad", "standings", ["E0"], req)
|
|
|
|
assert job.status == "failed"
|
|
assert "network down" in job.error
|
|
assert job.finished_at is not None
|
|
assert job.result is None
|
|
|
|
async def test_all_task_collects_per_task_summaries(self, monkeypatch):
|
|
job = _job(jid="job-all", status="pending", task="all")
|
|
session = _FakeSession(results=[_FakeResult([job])] * 20)
|
|
_patch_uow(monkeypatch, session)
|
|
|
|
class _FakeSource:
|
|
async def ingest(self, session, **kw):
|
|
return {"inserted": 1, "updated": 0, "total_inserted": 1,
|
|
"total_updated": 0, "errors": []}
|
|
|
|
async def _standings(*args, **kwargs):
|
|
return {"total_upserted": 20, "errors": []}
|
|
|
|
async def _stats(*args, **kwargs):
|
|
return {"total_inserted": 5, "total_updated": 2, "errors": []}
|
|
|
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
|
monkeypatch.setattr(ingest, "ingest_bzzoiro_standings", _standings)
|
|
monkeypatch.setattr(ingest, "ingest_bzzoiro_event_stats", _stats)
|
|
|
|
req = IngestBzzoiroRequest(task="all", leagues=["E0"])
|
|
await ingest._run_bzzoiro("job-all", "all", ["E0"], req)
|
|
|
|
assert job.status == "success"
|
|
assert set(job.result.keys()) == {"events", "standings", "stats"}
|
|
assert job.result["standings"]["total_upserted"] == 20
|
|
assert job.result["stats"]["total_inserted"] == 5
|
|
|
|
async def test_update_job_failure_does_not_break_ingest(self, monkeypatch):
|
|
"""_update_job 抛错必须被吞掉:job 可观测性失败 ≠ 采集失败。"""
|
|
session = _FakeSession()
|
|
_patch_uow(monkeypatch, session)
|
|
|
|
class _FakeSource:
|
|
async def ingest(self, session, **kw):
|
|
return {"inserted": 1, "updated": 0, "total_inserted": 1,
|
|
"total_updated": 0, "errors": []}
|
|
|
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
|
|
|
# 让 execute 抛错(_update_job 内部会捕获)
|
|
async def _broken_execute(stmt):
|
|
raise RuntimeError("db gone")
|
|
|
|
session.execute = _broken_execute
|
|
|
|
req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished")
|
|
# 不抛异常即通过;采集逻辑本身照常跑完
|
|
await ingest._run_bzzoiro("job-x", "events", ["E0"], req)
|
|
|
|
async def test_job_and_deadletter_are_independent_layers(self, monkeypatch):
|
|
"""任务级(job)与行级(死信)互不干扰:source 内部返回 errors 时,
|
|
job 仍为 success(部分失败不算任务失败),死信由 bzzoiro 层另行记录。"""
|
|
job = _job(jid="job-part", status="pending", task="events")
|
|
session = _FakeSession(results=[_FakeResult([job])] * 10)
|
|
_patch_uow(monkeypatch, session)
|
|
|
|
class _FakeSource:
|
|
async def ingest(self, session, **kw):
|
|
# 模拟 bzzoiro 管线:单条失败已写死信,汇总 errors 非空但返回正常
|
|
return {"inserted": 9, "updated": 0, "total_inserted": 9,
|
|
"total_updated": 0, "errors": ["league F1 fetch failed"]}
|
|
|
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
|
|
|
req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished")
|
|
await ingest._run_bzzoiro("job-part", "events", ["E0"], req)
|
|
|
|
assert job.status == "success"
|
|
assert job.result["events"]["errors"] == ["league F1 fetch failed"]
|
|
|
|
|
|
# ── 3. admin 查询端点 ──────────────────────────────────────────
|
|
|
|
|
|
class TestAdminJobEndpoints:
|
|
def test_get_job_detail_200(self, monkeypatch):
|
|
job = _job(jid="abc-123", status="running")
|
|
client = _admin_app(_FakeSession(results=[_FakeResult([job])]))
|
|
|
|
resp = client.get("/api/v1/admin/ingest/jobs/abc-123")
|
|
assert resp.status_code == 200
|
|
body = resp.json()
|
|
assert body["id"] == "abc-123"
|
|
assert body["status"] == "running"
|
|
assert body["task"] == "standings"
|
|
assert body["params"] == {"leagues": ["E0"]}
|
|
assert body["result"] == {"total_upserted": 20}
|
|
assert body["error"] is None
|
|
assert body["created_at"] is not None
|
|
|
|
def test_get_job_detail_404(self):
|
|
client = _admin_app(_FakeSession(results=[_FakeResult([])]))
|
|
resp = client.get("/api/v1/admin/ingest/jobs/missing")
|
|
assert resp.status_code == 404
|
|
|
|
def test_list_jobs_returns_serialized_rows(self):
|
|
j1 = _job(jid="j1", status="success")
|
|
j2 = _job(jid="j2", status="failed", task="events",
|
|
result=None, error="RuntimeError('x')")
|
|
client = _admin_app(_FakeSession(results=[_FakeResult([j1, j2])]))
|
|
|
|
resp = client.get("/api/v1/admin/ingest/jobs?limit=10")
|
|
assert resp.status_code == 200
|
|
rows = resp.json()
|
|
assert [r["id"] for r in rows] == ["j1", "j2"]
|
|
assert rows[1]["status"] == "failed"
|
|
assert rows[1]["error"] == "RuntimeError('x')"
|
|
|
|
def test_job_routes_live_under_admin_router_with_require_admin(self):
|
|
"""结构守护:job 端点必须挂在 /api/v1/admin 路由(路由级 require_admin)。"""
|
|
assert any(dep.dependency is require_admin for dep in schedules.router.dependencies)
|
|
paths = {getattr(r, "path", "") for r in schedules.router.routes}
|
|
assert "/api/v1/admin/ingest/jobs" in paths
|
|
assert "/api/v1/admin/ingest/jobs/{job_id}" in paths
|