feat: 数据采集管线基础设施接线 + 数据管线管理页

后端:
- bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档)
- 采集失败写入 IngestFailure(死信队列,支持重试)
- 写入 DataLineage(ETL 血缘追踪)
- 新增 DataQualityScheduler(每小时自动质量检查)
- 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发)
- 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试)
- 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛)
- lifespan 启动 quality_scheduler

前端:
- 新增「数据管线」管理页(/admin/data-pipeline)
- 采集失败记录列表(状态/重试次数/错误类型)
- 数据质量检查结果(通过/未通过/严重度)
- 手动触发质量检查按钮
- 失败记录重试按钮
- 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充)
- 导航统一为 React Router Link
- AdminStats 类型扩展(matches/stats/standings 真实计数)

Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
2026-09-21 10:04:26 +08:00
co-authored by new-provider/LongCat-2.0 <
parent b6e367a640
commit ae89d0f04f
9 changed files with 567 additions and 4 deletions
+46
View File
@@ -136,3 +136,49 @@ async def run_schedule_now(schedule_id: str):
import asyncio
asyncio.create_task(_run_scheduled_task(schedule_id))
return {"ok": True, "message": "任务已启动"}
# ── 采集失败重试 ────────────────────────────────────────────────
@router.get("/ingest-failures")
async def list_ingest_failures(db: AsyncSession = Depends(get_db_read)):
"""列出采集失败记录。"""
from src.db.models import IngestFailure
rows = (
await db.execute(
select(IngestFailure).order_by(IngestFailure.created_at.desc()).limit(50)
)
).scalars().all()
return [
{
"id": f.id,
"source": f.source_system,
"entity_type": f.entity_type,
"source_record_id": f.source_record_id,
"error_type": f.error_type,
"error_detail": f.error_detail,
"retry_count": f.retry_count,
"status": f.status,
"next_retry_at": f.next_retry_at.isoformat() if f.next_retry_at else None,
"created_at": f.created_at.isoformat() if f.created_at else None,
}
for f in rows
]
@router.post("/ingest-failures/{failure_id}/retry")
async def retry_ingest_failure(failure_id: int, db: AsyncSession = Depends(get_db_read)):
"""重试一次采集失败。"""
from src.db.models import IngestFailure
stmt = select(IngestFailure).where(IngestFailure.id == failure_id)
failure = (await db.execute(stmt)).scalar_one_or_none()
if failure is None:
raise HTTPException(404, "失败记录不存在")
failure.status = "retrying"
failure.retry_count += 1
await db.commit()
# 触发重试(简化版:仅标记状态,实际重试逻辑由调度器处理)
return {"ok": True, "message": f"已标记重试 (第 {failure.retry_count} 次)"}