shangfangjian/Profeto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ingest): 采集任务状态跟踪(ingest_jobs),解决 fire-and-forget 不可观测

Browse Source 
- 新表 ingest_jobs(迁移 0019): id UUID/task/params JSONB/
  status(pending|running|success|failed,CheckConstraint)/
  result JSONB(统计摘要)/error/created_at/started_at/finished_at
- POST /ingest/bzzoiro: 启动后台前创建 pending job,响应返回 job_id;
  仍 require_admin。后台 _run_bzzoiro 流转 running→success/failed,
  result 按子任务(events/standings/stats)记录摘要(errors 截断 10 条)
- _update_job 尽力而为: 状态更新失败只记日志,绝不拖垮采集主流程;
  与 IngestFailure 死信独立(行级 vs 任务级,可同时存在)
- 新增 admin 端点(挂 /api/v1/admin 路由,路由级 require_admin):
  GET /admin/ingest/jobs/{job_id} 与 GET /admin/ingest/jobs?limit&status
- 前端采集页: 提交后凭 job_id 3 秒轮询,终态展示结果摘要/失败原因;
  无 job_id 时回退旧的 30 秒盲等 + 系统日志提示
- 测试 11 项: 建 job+job_id 契约、非法 task 422、成功/失败/all 流转、
  update 失败不拖垮采集、部分失败仍 success、admin 端点 200/404/列表、
  结构守护(job 路由在 admin 路由且带 require_admin)
- 禁止项确认: 未动分批 UoW、BzzoiroSource、死信与 Bronze 写入;
  docs(01/03/05/07/README)同步 13 张表与端点说明
This commit is contained in:
WorkBuddy
2026-09-21 21:19:21 +08:00
parent 6bdb1f8ae6
commit 983363dab7
13 changed files with 666 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/routes/ingest.py
+73 -6
View File
@@ -10,14 +10,17 @@ from __future__ import annotations
import asyncio
import logging
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy import select
from src.api.deps import require_admin
from src.api.schemas import IngestBzzoiroRequest
from src.data.config import BZZOIRO_LEAGUE_IDS
from src.data.bzzoiro import ingest_bzzoiro_event_stats, ingest_bzzoiro_standings
from src.data.sources import get_source
from src.db.models import IngestJob
from src.db.unit_of_work import get_uow
logger = logging.getLogger(__name__)
@@ -30,6 +33,29 @@ _background_tasks: set[asyncio.Task] = set()
VALID_TASKS = {"events", "standings", "stats", "all"}
def _utcnow() -> datetime:
return datetime.now(timezone.utc)
async def _update_job(job_id: str, **fields) -> None:
"""更新采集任务状态(尽力而为:失败只记日志,绝不拖垮采集主流程)。
job 记录是任务级可观测性,IngestFailure 死信是行级失败记录,
两者独立工作 —— 本函数抛错不应影响采集结果,故整体吞异常。
"""
try:
async with get_uow() as session:
stmt = select(IngestJob).where(IngestJob.id == job_id)
job = (await session.execute(stmt)).scalar_one_or_none()
if job is None:
logger.warning("ingest job %s 不存在,跳过状态更新(%s)", job_id, fields)
return
for k, v in fields.items():
setattr(job, k, v)
except Exception:
logger.warning("ingest job %s 状态更新失败(不影响采集): %s", job_id, fields, exc_info=True)
def _spawn(coro) -> None:
"""启动后台采集任务;异常已在任务内记录到系统日志。"""
task = asyncio.create_task(coro)
@@ -39,20 +65,50 @@ def _spawn(coro) -> None:
@router.post("/ingest/bzzoiro", dependencies=[Depends(require_admin)])
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
"""触发 bzzoiro 采集(events / standings / stats / all)。"""
"""触发 bzzoiro 采集(events / standings / stats / all)。
启动后台任务前先创建 ingest_jobs 记录(pending),响应返回 job_id,
供 admin 通过 GET /api/v1/admin/ingest/jobs/{job_id} 轮询状态。
"""
if req.task not in VALID_TASKS:
raise HTTPException(status_code=422, detail=f"未知任务类型: {req.task}(可选: {', '.join(sorted(VALID_TASKS))})")
leagues = req.leagues or list(BZZOIRO_LEAGUE_IDS.keys())
task_label = {"events": "比赛数据", "standings": "积分榜", "stats": "统计回填", "all": "全量(比赛+积分榜+统计)"}[req.task]
_spawn(_run_bzzoiro(req.task, leagues, req))
# 任务级可观测性:先落 pending 记录,后台 _run_bzzoiro 接管状态流转
job = IngestJob(
task=req.task,
params={
"leagues": leagues,
"date_from": req.date_from,
"date_to": req.date_to,
"status": req.status,
"limit": req.limit,
"season": req.season,
},
status="pending",
)
async with get_uow() as session:
session.add(job)
_spawn(_run_bzzoiro(job.id, req.task, leagues, req))
return {
"ok": True,
"message": f"采集任务已启动(后台执行,任务: {task_label}),请在「系统日志」查看进度与结果",
"job_id": job.id,
"message": f"采集任务已启动(后台执行,任务: {task_label}),可用 job_id 查询状态或在「系统日志」查看进度",
}
async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None:
"""后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。"""
async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None:
"""后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。
同时维护 ingest_jobs 状态(pending → running → success/failed):
- result 存各子任务统计摘要(errors 截断到前 10 条)
- 状态更新经 _update_job 尽力而为,失败不影响采集本身
- 单条管线抓取失败的行级记录仍由 bzzoiro 死信逻辑(IngestFailure)负责
"""
await _update_job(job_id, status="running", started_at=_utcnow())
summary: dict = {}
try:
if task in ("events", "all"):
statuses = [req.status] if req.status else ["finished", "scheduled"]
@@ -79,6 +135,7 @@ async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest)
)
if merged["errors"]:
logger.warning("bzzoiro 比赛采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3])
summary["events"] = {**merged, "errors": merged["errors"][:10]}
if task in ("standings", "all"):
async with get_uow() as session:
@@ -87,6 +144,9 @@ async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest)
logger.warning("bzzoiro 积分榜采集部分失败: %s", r["errors"][:3])
else:
logger.info("bzzoiro 积分榜采集完成: upsert %d 条", r["total_upserted"])
summary["standings"] = {k: r.get(k) for k in ("total_upserted", "errors") if k in r}
if isinstance(summary["standings"].get("errors"), list):
summary["standings"]["errors"] = summary["standings"]["errors"][:10]
if task in ("stats", "all"):
async with get_uow() as session:
@@ -95,5 +155,12 @@ async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest)
)
if r["errors"]:
logger.warning("bzzoiro 统计回填错误 %d 条: %s", len(r["errors"]), r["errors"][:3])
except Exception:
summary["stats"] = {k: r.get(k) for k in ("total_inserted", "total_updated", "errors") if k in r}
if isinstance(summary["stats"].get("errors"), list):
summary["stats"]["errors"] = summary["stats"]["errors"][:10]
except Exception as exc:
logger.exception("bzzoiro 采集任务失败(task=%s)", task)
await _update_job(job_id, status="failed", finished_at=_utcnow(), error=repr(exc))
return
await _update_job(job_id, status="success", finished_at=_utcnow(), result=summary)
src/api/routes/schedules.py
+46
View File
@@ -182,3 +182,49 @@ async def retry_ingest_failure(failure_id: int, db: AsyncSession = Depends(get_d
# 触发重试(简化版:仅标记状态,实际重试逻辑由调度器处理)
return {"ok": True, "message": f"已标记重试 (第 {failure.retry_count} 次)"}
# ── 采集任务状态(ingest_jobs) ──────────────────────────────────
def _serialize_job(j) -> dict:
"""ingest_jobs 行 → 响应 dict(时间统一 isoformat)。"""
return {
"id": j.id,
"task": j.task,
"params": j.params,
"status": j.status,
"result": j.result,
"error": j.error,
"created_at": j.created_at.isoformat() if j.created_at else None,
"started_at": j.started_at.isoformat() if j.started_at else None,
"finished_at": j.finished_at.isoformat() if j.finished_at else None,
}
@router.get("/ingest/jobs")
async def list_ingest_jobs(
limit: int = 20,
status: str | None = None,
db: AsyncSession = Depends(get_db_read),
):
"""列出采集任务状态(最新在前),可按 status 过滤。"""
from src.db.models import IngestJob
stmt = select(IngestJob).order_by(IngestJob.created_at.desc()).limit(max(1, min(limit, 100)))
if status:
stmt = stmt.where(IngestJob.status == status)
rows = (await db.execute(stmt)).scalars().all()
return [_serialize_job(j) for j in rows]
@router.get("/ingest/jobs/{job_id}")
async def get_ingest_job(job_id: str, db: AsyncSession = Depends(get_db_read)):
"""查询单次采集任务的状态与结果(供前端提交后轮询)。"""
from src.db.models import IngestJob
stmt = select(IngestJob).where(IngestJob.id == job_id)
job = (await db.execute(stmt)).scalar_one_or_none()
if job is None:
raise HTTPException(404, "采集任务不存在")
return _serialize_job(job)