feat: 数据采集管线基础设施接线 + 数据管线管理页

后端:
- bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档)
- 采集失败写入 IngestFailure(死信队列,支持重试)
- 写入 DataLineage(ETL 血缘追踪)
- 新增 DataQualityScheduler(每小时自动质量检查)
- 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发)
- 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试)
- 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛)
- lifespan 启动 quality_scheduler

前端:
- 新增「数据管线」管理页(/admin/data-pipeline)
- 采集失败记录列表(状态/重试次数/错误类型)
- 数据质量检查结果(通过/未通过/严重度)
- 手动触发质量检查按钮
- 失败记录重试按钮
- 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充)
- 导航统一为 React Router Link
- AdminStats 类型扩展(matches/stats/standings 真实计数)

Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
2026-09-21 10:04:26 +08:00
co-authored by new-provider/LongCat-2.0 <
parent b6e367a640
commit ae89d0f04f
9 changed files with 567 additions and 4 deletions
+80
View File
@@ -67,6 +67,86 @@ class ScheduledTask:
logger.exception("定时任务失败: %s", self.task_id)
class DataQualityScheduler:
"""数据质量检查调度器(独立于采集任务)。"""
def __init__(self) -> None:
self._task: asyncio.Task | None = None
self._running = False
async def start(self) -> None:
self._running = True
self._task = asyncio.create_task(self._run_loop())
logger.info("数据质量检查调度器已启动")
async def stop(self) -> None:
self._running = False
if self._task:
self._task.cancel()
async def _run_loop(self) -> None:
"""每小时执行一次数据质量检查。"""
while self._running:
try:
await self._run_checks()
except Exception:
logger.exception("数据质量检查失败")
await asyncio.sleep(3600) # 每小时
async def _run_checks(self) -> None:
"""执行数据质量检查并写入 DataQualityCheck 表。"""
from src.db.base import AsyncSessionLocal
from src.db.models import DataQualityCheck, Match, MatchStats, Standing, League
from sqlalchemy import func, select
async with AsyncSessionLocal() as db:
# 检查1: 已完赛但无统计的比赛数
finished_no_stats = (
await db.execute(
select(func.count())
.select_from(Match)
.outerjoin(MatchStats, Match.id == MatchStats.match_id)
.where(Match.match_status == "finished")
.where(MatchStats.id.is_(None))
)
).scalar() or 0
db.add(DataQualityCheck(
check_name="finished_without_stats",
entity_type="match",
actual_value=float(finished_no_stats),
passed=finished_no_stats == 0,
severity="warning" if finished_no_stats > 0 else "info",
detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
))
# 检查2: 积分榜缺失的联赛数
leagues_without_standings = (
await db.execute(
select(func.count())
.select_from(League)
.outerjoin(Standing, League.id == Standing.league_id)
.where(Standing.id.is_(None))
)
).scalar() or 0
db.add(DataQualityCheck(
check_name="league_without_standings",
entity_type="league",
actual_value=float(leagues_without_standings),
passed=leagues_without_standings == 0,
severity="warning" if leagues_without_standings > 0 else "info",
detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
))
await db.commit()
logger.info("数据质量检查完成: stats=%d, standings=%d", finished_no_stats, leagues_without_standings)
# 全局单例
quality_scheduler = DataQualityScheduler()
class Scheduler:
"""全局定时任务调度器。"""