feat: 数据采集管线基础设施接线 + 数据管线管理页
后端: - bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档) - 采集失败写入 IngestFailure(死信队列,支持重试) - 写入 DataLineage(ETL 血缘追踪) - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发) - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛) - lifespan 启动 quality_scheduler 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 失败记录重试按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
co-authored by
new-provider/LongCat-2.0 <
parent
b6e367a640
commit
ae89d0f04f
@@ -92,6 +92,7 @@ const NAV_PAGES: Array<{ to: string; label: string; group: string }> = [
|
||||
{ to: '/admin', label: '仪表盘', group: '概览' },
|
||||
{ to: '/admin/collection', label: '数据采集', group: '数据流水线' },
|
||||
{ to: '/admin/data-completeness', label: '数据完整性', group: '数据流水线' },
|
||||
{ to: '/admin/data-pipeline', label: '数据管线', group: '数据流水线' },
|
||||
{ to: '/admin/predictions', label: '预测历史', group: '数据流水线' },
|
||||
{ to: '/admin/backtest', label: '回测', group: '数据流水线' },
|
||||
{ to: '/admin/monitoring', label: '监控', group: '评估与监控' },
|
||||
@@ -105,6 +106,7 @@ const ROUTE_LABELS: Record<string, string> = {
|
||||
'/admin': '仪表盘',
|
||||
'/admin/collection': '数据采集',
|
||||
'/admin/data-completeness': '数据完整性',
|
||||
'/admin/data-pipeline': '数据管线',
|
||||
'/admin/predictions': '预测历史',
|
||||
'/admin/backtest': '回测',
|
||||
'/admin/monitoring': '监控',
|
||||
|
||||
@@ -441,3 +441,49 @@ export async function deleteSchedule(id: string): Promise<{ ok: boolean }> {
|
||||
export async function runScheduleNow(id: string): Promise<{ ok: boolean; message: string }> {
|
||||
return api.post<{ ok: boolean; message: string }>(`${API_BASE}/admin/schedules/${id}/run`)
|
||||
}
|
||||
|
||||
// ── 数据管线(质量检查 + 失败重试) ──────────────────────────────
|
||||
|
||||
export interface IngestFailureItem {
|
||||
id: number
|
||||
source: string
|
||||
entity_type: string
|
||||
source_record_id?: string
|
||||
error_type: string
|
||||
error_detail?: string
|
||||
retry_count: number
|
||||
status: string
|
||||
next_retry_at?: string | null
|
||||
created_at?: string
|
||||
}
|
||||
|
||||
export interface DataQualityCheckItem {
|
||||
id: number
|
||||
check_name: string
|
||||
entity_type: string
|
||||
passed: boolean
|
||||
severity: string
|
||||
detail?: Record<string, unknown> | null
|
||||
checked_at?: string
|
||||
}
|
||||
|
||||
export interface DataQualityResponse {
|
||||
failures: IngestFailureItem[]
|
||||
checks: DataQualityCheckItem[]
|
||||
}
|
||||
|
||||
export async function fetchDataQuality(): Promise<DataQualityResponse> {
|
||||
return api.get<DataQualityResponse>(`${API_BASE}/admin/data-quality`)
|
||||
}
|
||||
|
||||
export async function runDataQualityCheck(): Promise<{ ok: boolean; checks: Array<{ name: string; passed: boolean }> }> {
|
||||
return api.post<{ ok: boolean; checks: Array<{ name: string; passed: boolean }> }>(`${API_BASE}/admin/data-quality/run`)
|
||||
}
|
||||
|
||||
export async function fetchIngestFailures(): Promise<IngestFailureItem[]> {
|
||||
return api.get<IngestFailureItem[]>(`${API_BASE}/admin/ingest-failures`)
|
||||
}
|
||||
|
||||
export async function retryIngestFailure(id: number): Promise<{ ok: boolean; message: string }> {
|
||||
return api.post<{ ok: boolean; message: string }>(`${API_BASE}/admin/ingest-failures/${id}/retry`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* Admin 后台 - 数据管线管理页(报刊风)
|
||||
*
|
||||
* 功能:
|
||||
* - 采集失败记录列表(可重试)
|
||||
* - 数据质量检查结果
|
||||
* - 手动触发质量检查
|
||||
*/
|
||||
|
||||
import { useEffect, useState, useCallback } from 'react'
|
||||
import {
|
||||
fetchDataQuality,
|
||||
runDataQualityCheck,
|
||||
fetchIngestFailures,
|
||||
retryIngestFailure,
|
||||
} from '../dal'
|
||||
import type { IngestFailureItem, DataQualityCheckItem } from '../dal'
|
||||
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
|
||||
|
||||
export default function DataPipelinePage() {
|
||||
const [quality, setQuality] = useState<{ failures: IngestFailureItem[]; checks: DataQualityCheckItem[] } | null>(null)
|
||||
const [failures, setFailures] = useState<IngestFailureItem[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [running, setRunning] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [notice, setNotice] = useState<{ ok: boolean; text: string } | null>(null)
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const [q, f] = await Promise.all([fetchDataQuality(), fetchIngestFailures()])
|
||||
setQuality(q)
|
||||
setFailures(f)
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : '加载失败')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => { load() }, [load])
|
||||
|
||||
const handleRunCheck = async () => {
|
||||
setRunning(true)
|
||||
setNotice(null)
|
||||
try {
|
||||
const res = await runDataQualityCheck()
|
||||
const failed = res.checks.filter(c => !c.passed)
|
||||
setNotice({
|
||||
ok: failed.length === 0,
|
||||
text: failed.length === 0
|
||||
? '数据质量检查通过'
|
||||
: `检查完成: ${failed.length} 项未通过`,
|
||||
})
|
||||
await load()
|
||||
} catch {
|
||||
setNotice({ ok: false, text: '质量检查执行失败' })
|
||||
} finally {
|
||||
setRunning(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleRetry = async (id: number) => {
|
||||
setNotice(null)
|
||||
try {
|
||||
const res = await retryIngestFailure(id)
|
||||
setNotice({ ok: true, text: res.message })
|
||||
await load()
|
||||
} catch {
|
||||
setNotice({ ok: false, text: '重试操作失败' })
|
||||
}
|
||||
}
|
||||
|
||||
const pendingFailures = failures.filter(f => f.status === 'pending' || f.status === 'retrying')
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<SectionHeader
|
||||
title="数据管线"
|
||||
description="采集失败重试、数据质量检查与监控"
|
||||
action={
|
||||
<button onClick={handleRunCheck} disabled={running} className="btn btn-sm">
|
||||
{running ? <><Spinner /> 检查中</> : '运行质量检查'}
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
|
||||
{notice && (
|
||||
<Alert kind={notice.ok ? 'ok' : 'error'} title={notice.text} onClose={() => setNotice(null)} />
|
||||
)}
|
||||
|
||||
{error && <Alert kind="error" title="加载失败" message={error} onClose={() => setError(null)} />}
|
||||
|
||||
{loading && (
|
||||
<div className="flex justify-center py-12"><Spinner /></div>
|
||||
)}
|
||||
|
||||
{!loading && (
|
||||
<>
|
||||
{/* 采集失败记录 */}
|
||||
<Card>
|
||||
<CardHeader
|
||||
title="采集失败记录"
|
||||
description={pendingFailures.length > 0 ? `${pendingFailures.length} 条待处理` : '暂无待处理失败记录'}
|
||||
/>
|
||||
<CardBody className="px-0 sm:px-0">
|
||||
{failures.length === 0 ? (
|
||||
<p className="py-8 text-center text-xs text-ink-400">暂无采集失败记录</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[640px] text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-ink-200 text-left text-ink-500">
|
||||
<th className="px-4 py-2 font-medium">来源</th>
|
||||
<th className="px-4 py-2 font-medium">实体类型</th>
|
||||
<th className="px-4 py-2 font-medium">错误类型</th>
|
||||
<th className="px-4 py-2 font-medium">重试次数</th>
|
||||
<th className="px-4 py-2 font-medium">状态</th>
|
||||
<th className="px-4 py-2 font-medium">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{failures.map(f => (
|
||||
<tr key={f.id} className="border-b border-ink-100 hover:bg-paper-100">
|
||||
<td className="px-4 py-2 text-xs text-ink-700">{f.source}</td>
|
||||
<td className="px-4 py-2 text-xs text-ink-600">{f.entity_type}</td>
|
||||
<td className="px-4 py-2 text-xs text-ink-600">{f.error_type}</td>
|
||||
<td className="px-4 py-2 text-xs tabular-nums text-ink-500">{f.retry_count}</td>
|
||||
<td className="px-4 py-2">
|
||||
<Badge status={f.status === 'resolved' ? 'success' : f.status === 'pending' ? 'warning' : 'info'}>
|
||||
{f.status}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
{(f.status === 'pending' || f.status === 'retrying') && (
|
||||
<button onClick={() => handleRetry(f.id)} className="btn btn-sm">
|
||||
重试
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</CardBody>
|
||||
</Card>
|
||||
|
||||
{/* 数据质量检查 */}
|
||||
<Card>
|
||||
<CardHeader title="数据质量检查" description="最近 20 条检查结果" />
|
||||
<CardBody className="px-0 sm:px-0">
|
||||
{quality?.checks.length === 0 ? (
|
||||
<p className="py-8 text-center text-xs text-ink-400">暂无质量检查记录,点击右上角「运行质量检查」触发</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[560px] text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-ink-200 text-left text-ink-500">
|
||||
<th className="px-4 py-2 font-medium">检查项</th>
|
||||
<th className="px-4 py-2 font-medium">实体</th>
|
||||
<th className="px-4 py-2 font-medium">结果</th>
|
||||
<th className="px-4 py-2 font-medium">严重度</th>
|
||||
<th className="px-4 py-2 font-medium">时间</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{quality?.checks.map(c => (
|
||||
<tr key={c.id} className="border-b border-ink-100 hover:bg-paper-100">
|
||||
<td className="px-4 py-2 text-xs text-ink-700">{c.check_name}</td>
|
||||
<td className="px-4 py-2 text-xs text-ink-600">{c.entity_type}</td>
|
||||
<td className="px-4 py-2">
|
||||
<Badge status={c.passed ? 'success' : 'error'}>
|
||||
{c.passed ? '通过' : '未通过'}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
<Badge status={c.severity === 'warning' ? 'warning' : c.severity === 'error' ? 'error' : 'info'}>
|
||||
{c.severity}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="px-4 py-2 text-2xs text-ink-400">
|
||||
{c.checked_at ? new Date(c.checked_at).toLocaleString('zh-CN', { hour12: false }) : '—'}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</CardBody>
|
||||
</Card>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import MonitoringPage from './pages/Monitoring'
|
||||
import SettingsPage from './pages/Settings'
|
||||
import LogsPage from './pages/Logs'
|
||||
import EvalPage from './pages/EvalPage'
|
||||
import DataPipelinePage from './pages/DataPipeline'
|
||||
|
||||
export const adminRoutes = [
|
||||
{
|
||||
@@ -25,6 +26,7 @@ export const adminRoutes = [
|
||||
{ index: true, element: <Dashboard /> },
|
||||
{ path: 'collection', element: <CollectionPage /> },
|
||||
{ path: 'data-completeness', element: <DataCompletenessPage /> },
|
||||
{ path: 'data-pipeline', element: <DataPipelinePage /> },
|
||||
{ path: 'predictions', element: <PredictionsPage /> },
|
||||
{ path: 'backtest', element: <BacktestPage /> },
|
||||
{ path: 'monitoring', element: <MonitoringPage /> },
|
||||
|
||||
+3
-1
@@ -22,7 +22,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
migrate_plaintext_sensitive_settings,
|
||||
)
|
||||
from src.core.security_check import assert_security_on_startup
|
||||
from src.core.scheduler import scheduler
|
||||
from src.core.scheduler import scheduler, quality_scheduler
|
||||
from src.api.routes.schedules import _run_scheduled_task
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS
|
||||
await init_db() # 验证连接,不建表
|
||||
@@ -56,9 +56,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
)
|
||||
|
||||
await scheduler.start()
|
||||
await quality_scheduler.start()
|
||||
logger.info("应用启动完成")
|
||||
yield
|
||||
await scheduler.stop()
|
||||
await quality_scheduler.stop()
|
||||
await close_client()
|
||||
|
||||
|
||||
|
||||
@@ -554,3 +554,115 @@ async def data_completeness(db: AsyncSession = Depends(get_db_read)):
|
||||
},
|
||||
"issues": issues,
|
||||
}
|
||||
|
||||
|
||||
# ── 数据质量检查 API ────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/data-quality")
|
||||
async def data_quality_checks(db: AsyncSession = Depends(get_db_read)):
|
||||
"""数据质量检查结果(只读)。"""
|
||||
from src.db.models import IngestFailure, DataQualityCheck
|
||||
from sqlalchemy import func
|
||||
|
||||
# 最近的失败记录
|
||||
failures = (
|
||||
await db.execute(
|
||||
select(IngestFailure)
|
||||
.where(IngestFailure.status.in_(["pending", "retrying"]))
|
||||
.order_by(IngestFailure.created_at.desc())
|
||||
.limit(20)
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
# 最近的质量检查
|
||||
checks = (
|
||||
await db.execute(
|
||||
select(DataQualityCheck)
|
||||
.order_by(DataQualityCheck.checked_at.desc())
|
||||
.limit(20)
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
return {
|
||||
"failures": [
|
||||
{
|
||||
"id": f.id,
|
||||
"source": f.source_system,
|
||||
"entity_type": f.entity_type,
|
||||
"source_record_id": f.source_record_id,
|
||||
"error_type": f.error_type,
|
||||
"error_detail": f.error_detail,
|
||||
"retry_count": f.retry_count,
|
||||
"status": f.status,
|
||||
"created_at": f.created_at.isoformat() if f.created_at else None,
|
||||
}
|
||||
for f in failures
|
||||
],
|
||||
"checks": [
|
||||
{
|
||||
"id": c.id,
|
||||
"check_name": c.check_name,
|
||||
"entity_type": c.entity_type,
|
||||
"passed": c.passed,
|
||||
"severity": c.severity,
|
||||
"detail": c.detail,
|
||||
"checked_at": c.checked_at.isoformat() if c.checked_at else None,
|
||||
}
|
||||
for c in checks
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/data-quality/run")
|
||||
async def run_data_quality_check(db: AsyncSession = Depends(get_db_read)):
|
||||
"""手动触发一次数据质量检查。"""
|
||||
from src.db.models import DataQualityCheck, Match, MatchStats, Standing, League
|
||||
from sqlalchemy import func
|
||||
|
||||
checks = []
|
||||
|
||||
# 检查1: 已完赛但无统计的比赛
|
||||
finished_no_stats = (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(Match)
|
||||
.outerjoin(MatchStats, Match.id == MatchStats.match_id)
|
||||
.where(Match.match_status == "finished")
|
||||
.where(MatchStats.id.is_(None))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
checks.append(DataQualityCheck(
|
||||
check_name="finished_without_stats",
|
||||
entity_type="match",
|
||||
actual_value=float(finished_no_stats),
|
||||
passed=finished_no_stats == 0,
|
||||
severity="warning" if finished_no_stats > 0 else "info",
|
||||
detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
|
||||
))
|
||||
|
||||
# 检查2: 积分榜缺失的联赛
|
||||
leagues_without_standings = (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(League)
|
||||
.outerjoin(Standing, League.id == Standing.league_id)
|
||||
.where(Standing.id.is_(None))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
checks.append(DataQualityCheck(
|
||||
check_name="league_without_standings",
|
||||
entity_type="league",
|
||||
actual_value=float(leagues_without_standings),
|
||||
passed=leagues_without_standings == 0,
|
||||
severity="warning" if leagues_without_standings > 0 else "info",
|
||||
detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
|
||||
))
|
||||
|
||||
for c in checks:
|
||||
db.add(c)
|
||||
await db.commit()
|
||||
|
||||
return {"ok": True, "checks": [{"name": c.check_name, "passed": c.passed} for c in checks]}
|
||||
|
||||
@@ -136,3 +136,49 @@ async def run_schedule_now(schedule_id: str):
|
||||
import asyncio
|
||||
asyncio.create_task(_run_scheduled_task(schedule_id))
|
||||
return {"ok": True, "message": "任务已启动"}
|
||||
|
||||
|
||||
# ── 采集失败重试 ────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/ingest-failures")
|
||||
async def list_ingest_failures(db: AsyncSession = Depends(get_db_read)):
|
||||
"""列出采集失败记录。"""
|
||||
from src.db.models import IngestFailure
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(IngestFailure).order_by(IngestFailure.created_at.desc()).limit(50)
|
||||
)
|
||||
).scalars().all()
|
||||
return [
|
||||
{
|
||||
"id": f.id,
|
||||
"source": f.source_system,
|
||||
"entity_type": f.entity_type,
|
||||
"source_record_id": f.source_record_id,
|
||||
"error_type": f.error_type,
|
||||
"error_detail": f.error_detail,
|
||||
"retry_count": f.retry_count,
|
||||
"status": f.status,
|
||||
"next_retry_at": f.next_retry_at.isoformat() if f.next_retry_at else None,
|
||||
"created_at": f.created_at.isoformat() if f.created_at else None,
|
||||
}
|
||||
for f in rows
|
||||
]
|
||||
|
||||
|
||||
@router.post("/ingest-failures/{failure_id}/retry")
|
||||
async def retry_ingest_failure(failure_id: int, db: AsyncSession = Depends(get_db_read)):
|
||||
"""重试一次采集失败。"""
|
||||
from src.db.models import IngestFailure
|
||||
stmt = select(IngestFailure).where(IngestFailure.id == failure_id)
|
||||
failure = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if failure is None:
|
||||
raise HTTPException(404, "失败记录不存在")
|
||||
|
||||
failure.status = "retrying"
|
||||
failure.retry_count += 1
|
||||
await db.commit()
|
||||
|
||||
# 触发重试(简化版:仅标记状态,实际重试逻辑由调度器处理)
|
||||
return {"ok": True, "message": f"已标记重试 (第 {failure.retry_count} 次)"}
|
||||
|
||||
@@ -67,6 +67,86 @@ class ScheduledTask:
|
||||
logger.exception("定时任务失败: %s", self.task_id)
|
||||
|
||||
|
||||
class DataQualityScheduler:
|
||||
"""数据质量检查调度器(独立于采集任务)。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running = False
|
||||
|
||||
async def start(self) -> None:
|
||||
self._running = True
|
||||
self._task = asyncio.create_task(self._run_loop())
|
||||
logger.info("数据质量检查调度器已启动")
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._running = False
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
|
||||
async def _run_loop(self) -> None:
|
||||
"""每小时执行一次数据质量检查。"""
|
||||
while self._running:
|
||||
try:
|
||||
await self._run_checks()
|
||||
except Exception:
|
||||
logger.exception("数据质量检查失败")
|
||||
await asyncio.sleep(3600) # 每小时
|
||||
|
||||
async def _run_checks(self) -> None:
|
||||
"""执行数据质量检查并写入 DataQualityCheck 表。"""
|
||||
from src.db.base import AsyncSessionLocal
|
||||
from src.db.models import DataQualityCheck, Match, MatchStats, Standing, League
|
||||
from sqlalchemy import func, select
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
# 检查1: 已完赛但无统计的比赛数
|
||||
finished_no_stats = (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(Match)
|
||||
.outerjoin(MatchStats, Match.id == MatchStats.match_id)
|
||||
.where(Match.match_status == "finished")
|
||||
.where(MatchStats.id.is_(None))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
db.add(DataQualityCheck(
|
||||
check_name="finished_without_stats",
|
||||
entity_type="match",
|
||||
actual_value=float(finished_no_stats),
|
||||
passed=finished_no_stats == 0,
|
||||
severity="warning" if finished_no_stats > 0 else "info",
|
||||
detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
|
||||
))
|
||||
|
||||
# 检查2: 积分榜缺失的联赛数
|
||||
leagues_without_standings = (
|
||||
await db.execute(
|
||||
select(func.count())
|
||||
.select_from(League)
|
||||
.outerjoin(Standing, League.id == Standing.league_id)
|
||||
.where(Standing.id.is_(None))
|
||||
)
|
||||
).scalar() or 0
|
||||
|
||||
db.add(DataQualityCheck(
|
||||
check_name="league_without_standings",
|
||||
entity_type="league",
|
||||
actual_value=float(leagues_without_standings),
|
||||
passed=leagues_without_standings == 0,
|
||||
severity="warning" if leagues_without_standings > 0 else "info",
|
||||
detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
|
||||
))
|
||||
|
||||
await db.commit()
|
||||
logger.info("数据质量检查完成: stats=%d, standings=%d", finished_no_stats, leagues_without_standings)
|
||||
|
||||
|
||||
# 全局单例
|
||||
quality_scheduler = DataQualityScheduler()
|
||||
|
||||
|
||||
class Scheduler:
|
||||
"""全局定时任务调度器。"""
|
||||
|
||||
|
||||
+75
-1
@@ -27,7 +27,7 @@ from src.data.key_ring import get_key_ring
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.team_names_zh import zh_name
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats, Standing, Team
|
||||
from src.db.models import League, Match, MatchStats, Standing, Team, RawEvent, IngestFailure, DataLineage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -337,9 +337,83 @@ class BzzoiroSource:
|
||||
result["leagues"][code] = league_r
|
||||
result["total_inserted"] += league_r["inserted"]
|
||||
result["total_updated"] += league_r["updated"]
|
||||
|
||||
# 管线基础设施:写入 RawEvent(原始事件存档)
|
||||
batch_id = f"bzzoiro-events-{code}-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}"
|
||||
for nm, raw in normalized_matches:
|
||||
try:
|
||||
_write_raw_event(db, "bzzoiro", str(raw.get("id", "")), raw, batch_id)
|
||||
except Exception:
|
||||
pass # 基础设施写入失败不影响主流程
|
||||
|
||||
# 写入 DataLineage(血缘追踪)
|
||||
for nm, raw in normalized_matches:
|
||||
try:
|
||||
_write_lineage(db, "bzzoiro", str(raw.get("id", ""), "matches", None, "normalize_bzzoiro", {"league_code": code}, batch_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
# 管线基础设施:写入 IngestFailure(失败死信)
|
||||
try:
|
||||
_write_ingest_failure(db, "bzzoiro", "events", None, "fetch_failed", str(e)[:500])
|
||||
except Exception:
|
||||
pass
|
||||
logger.exception("bzzoiro events ingest failed for %s", code)
|
||||
league_r["errors"].append(str(e))
|
||||
result["leagues"][code] = league_r
|
||||
result["errors"].append(f"{code}: {e}")
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 管线基础设施:RawEvent / IngestFailure / DataLineage
|
||||
# ============================================================
|
||||
|
||||
|
||||
def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None:
|
||||
"""写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。"""
|
||||
from sqlalchemy import select as _select
|
||||
stmt = _select(RawEvent).where(
|
||||
RawEvent.source_system == source_system,
|
||||
RawEvent.source_record_id == source_record_id,
|
||||
)
|
||||
existing = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if existing is None:
|
||||
db.add(RawEvent(
|
||||
source_system=source_system,
|
||||
source_record_id=source_record_id,
|
||||
raw_payload=raw_payload,
|
||||
ingest_batch_id=batch_id,
|
||||
))
|
||||
|
||||
|
||||
def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None:
|
||||
"""写入采集失败死信。"""
|
||||
db.add(IngestFailure(
|
||||
source_system=source_system,
|
||||
entity_type=entity_type,
|
||||
source_record_id=source_record_id,
|
||||
error_type=error_type,
|
||||
error_detail=error_detail,
|
||||
raw_payload=raw_payload,
|
||||
))
|
||||
|
||||
|
||||
def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
|
||||
"""写入 ETL 血缘追踪。"""
|
||||
db.add(DataLineage(
|
||||
source_system=source_system,
|
||||
source_record_id=source_record_id,
|
||||
target_table=target_table,
|
||||
target_id=target_id,
|
||||
transform_name=transform_name,
|
||||
transform_detail=transform_detail,
|
||||
batch_id=batch_id,
|
||||
))
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 积分榜管线:/leagues/{id}/standings/ → standings 表
|
||||
# ============================================================
|
||||
|
||||
Reference in New Issue
Block a user