feat: 数据采集管线基础设施接线 + 数据管线管理页
后端: - bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档) - 采集失败写入 IngestFailure(死信队列,支持重试) - 写入 DataLineage(ETL 血缘追踪) - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发) - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛) - lifespan 启动 quality_scheduler 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 失败记录重试按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
co-authored by
new-provider/LongCat-2.0 <
parent
b6e367a640
commit
ae89d0f04f
@@ -67,6 +67,86 @@ class ScheduledTask:
|
||||
logger.exception("定时任务失败: %s", self.task_id)
|
||||
|
||||
|
||||
class DataQualityScheduler:
|
||||
"""数据质量检查调度器(独立于采集任务)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running = False
|
||||
|
||||
async def start(self) -> None:
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
logger.info("数据质量检查调度器已启动")
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
"""每小时执行一次数据质量检查。"""
|
||||
while self._running:
|
||||
try:
|
||||
await self._run_checks()
|
||||
except Exception:
|
||||
logger.exception("数据质量检查失败")
|
||||
await asyncio.sleep(3600) # 每小时
|
||||
|
||||
async def _run_checks(self) -> None:
|
||||
"""执行数据质量检查并写入 DataQualityCheck 表。"""
|
||||
from src.db.base import AsyncSessionLocal
|
||||
from src.db.models import DataQualityCheck, Match, MatchStats, Standing, League
|
||||
from sqlalchemy import func, select
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 检查1: 已完赛但无统计的比赛数
|
||||
finished_no_stats = (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(Match)
|
||||
.outerjoin(MatchStats, Match.id == MatchStats.match_id)
|
||||
.where(Match.match_status == "finished")
|
||||
.where(MatchStats.id.is_(None))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
db.add(DataQualityCheck(
|
||||
check_name="finished_without_stats",
|
||||
entity_type="match",
|
||||
actual_value=float(finished_no_stats),
|
||||
passed=finished_no_stats == 0,
|
||||
severity="warning" if finished_no_stats > 0 else "info",
|
||||
detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
|
||||
))
|
||||
|
||||
# 检查2: 积分榜缺失的联赛数
|
||||
leagues_without_standings = (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(League)
|
||||
.outerjoin(Standing, League.id == Standing.league_id)
|
||||
.where(Standing.id.is_(None))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
db.add(DataQualityCheck(
|
||||
check_name="league_without_standings",
|
||||
entity_type="league",
|
||||
actual_value=float(leagues_without_standings),
|
||||
passed=leagues_without_standings == 0,
|
||||
severity="warning" if leagues_without_standings > 0 else "info",
|
||||
detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
logger.info("数据质量检查完成: stats=%d, standings=%d", finished_no_stats, leagues_without_standings)
|
||||
|
||||
|
||||
# 全局单例
|
||||
quality_scheduler = DataQualityScheduler()
|
||||
|
||||
|
||||
class Scheduler:
|
||||
"""全局定时任务调度器。"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user