feat: 数据采集管线基础设施接线 + 数据管线管理页
后端: - bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档) - 采集失败写入 IngestFailure(死信队列,支持重试) - 写入 DataLineage(ETL 血缘追踪) - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发) - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛) - lifespan 启动 quality_scheduler 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 失败记录重试按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
co-authored by
new-provider/LongCat-2.0 <
parent
b6e367a640
commit
ae89d0f04f
@@ -136,3 +136,49 @@ async def run_schedule_now(schedule_id: str):
|
||||
import asyncio
|
||||
asyncio.create_task(_run_scheduled_task(schedule_id))
|
||||
return {"ok": True, "message": "任务已启动"}
|
||||
|
||||
|
||||
# ── 采集失败重试 ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/ingest-failures")
|
||||
async def list_ingest_failures(db: AsyncSession = Depends(get_db_read)):
|
||||
"""列出采集失败记录。"""
|
||||
from src.db.models import IngestFailure
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(IngestFailure).order_by(IngestFailure.created_at.desc()).limit(50)
|
||||
)
|
||||
).scalars().all()
|
||||
return [
|
||||
{
|
||||
"id": f.id,
|
||||
"source": f.source_system,
|
||||
"entity_type": f.entity_type,
|
||||
"source_record_id": f.source_record_id,
|
||||
"error_type": f.error_type,
|
||||
"error_detail": f.error_detail,
|
||||
"retry_count": f.retry_count,
|
||||
"status": f.status,
|
||||
"next_retry_at": f.next_retry_at.isoformat() if f.next_retry_at else None,
|
||||
"created_at": f.created_at.isoformat() if f.created_at else None,
|
||||
}
|
||||
for f in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("/ingest-failures/{failure_id}/retry")
|
||||
async def retry_ingest_failure(failure_id: int, db: AsyncSession = Depends(get_db_read)):
|
||||
"""重试一次采集失败。"""
|
||||
from src.db.models import IngestFailure
|
||||
stmt = select(IngestFailure).where(IngestFailure.id == failure_id)
|
||||
failure = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if failure is None:
|
||||
raise HTTPException(404, "失败记录不存在")
|
||||
|
||||
failure.status = "retrying"
|
||||
failure.retry_count += 1
|
||||
await db.commit()
|
||||
|
||||
# 触发重试(简化版:仅标记状态,实际重试逻辑由调度器处理)
|
||||
return {"ok": True, "message": f"已标记重试 (第 {failure.retry_count} 次)"}
|
||||
|
||||
Reference in New Issue
Block a user