feat: 数据采集管线基础设施接线 + 数据管线管理页
后端: - bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档) - 采集失败写入 IngestFailure(死信队列,支持重试) - 写入 DataLineage(ETL 血缘追踪) - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发) - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛) - lifespan 启动 quality_scheduler 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 失败记录重试按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
co-authored by
new-provider/LongCat-2.0 <
new-provider/LongCat-2.0 <
parent
b6e367a640
commit
ae89d0f04f
@@ -91,7 +91,8 @@ function Icon({ name }: { name: string }) {
|
||||
const NAV_PAGES: Array<{ to: string; label: string; group: string }> = [
|
||||
{ to: '/admin', label: '仪表盘', group: '概览' },
|
||||
{ to: '/admin/collection', label: '数据采集', group: '数据流水线' },
|
||||
{ to: '/admin/data-completeness', label: '数据完整性', group: '数据流水线' },
|
||||
{ to: '/admin/data-completeness', label: '数据完整性', group: '数据流水线' },
|
||||
{ to: '/admin/data-pipeline', label: '数据管线', group: '数据流水线' },
|
||||
{ to: '/admin/predictions', label: '预测历史', group: '数据流水线' },
|
||||
{ to: '/admin/backtest', label: '回测', group: '数据流水线' },
|
||||
{ to: '/admin/monitoring', label: '监控', group: '评估与监控' },
|
||||
@@ -105,6 +106,7 @@ const ROUTE_LABELS: Record<string, string> = {
|
||||
'/admin': '仪表盘',
|
||||
'/admin/collection': '数据采集',
|
||||
'/admin/data-completeness': '数据完整性',
|
||||
'/admin/data-pipeline': '数据管线',
|
||||
'/admin/predictions': '预测历史',
|
||||
'/admin/backtest': '回测',
|
||||
'/admin/monitoring': '监控',
|
||||
|
||||
@@ -441,3 +441,49 @@ export async function deleteSchedule(id: string): Promise<{ ok: boolean }> {
|
||||
export async function runScheduleNow(id: string): Promise<{ ok: boolean; message: string }> {
|
||||
return api.post<{ ok: boolean; message: string }>(`${API_BASE}/admin/schedules/${id}/run`)
|
||||
}
|
||||
|
||||
// ── 数据管线(质量检查 + 失败重试) ──────────────────────────────
|
||||
|
||||
export interface IngestFailureItem {
|
||||
id: number
|
||||
source: string
|
||||
entity_type: string
|
||||
source_record_id?: string
|
||||
error_type: string
|
||||
error_detail?: string
|
||||
retry_count: number
|
||||
status: string
|
||||
next_retry_at?: string | null
|
||||
created_at?: string
|
||||
}
|
||||
|
||||
export interface DataQualityCheckItem {
|
||||
id: number
|
||||
check_name: string
|
||||
entity_type: string
|
||||
passed: boolean
|
||||
severity: string
|
||||
detail?: Record<string, unknown> | null
|
||||
checked_at?: string
|
||||
}
|
||||
|
||||
export interface DataQualityResponse {
|
||||
failures: IngestFailureItem[]
|
||||
checks: DataQualityCheckItem[]
|
||||
}
|
||||
|
||||
export async function fetchDataQuality(): Promise<DataQualityResponse> {
|
||||
return api.get<DataQualityResponse>(`${API_BASE}/admin/data-quality`)
|
||||
}
|
||||
|
||||
export async function runDataQualityCheck(): Promise<{ ok: boolean; checks: Array<{ name: string; passed: boolean }> }> {
|
||||
return api.post<{ ok: boolean; checks: Array<{ name: string; passed: boolean }> }>(`${API_BASE}/admin/data-quality/run`)
|
||||
}
|
||||
|
||||
export async function fetchIngestFailures(): Promise<IngestFailureItem[]> {
|
||||
return api.get<IngestFailureItem[]>(`${API_BASE}/admin/ingest-failures`)
|
||||
}
|
||||
|
||||
export async function retryIngestFailure(id: number): Promise<{ ok: boolean; message: string }> {
|
||||
return api.post<{ ok: boolean; message: string }>(`${API_BASE}/admin/ingest-failures/${id}/retry`)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
/**
|
||||
* Admin 后台 - 数据管线管理页(报刊风)
|
||||
*
|
||||
* 功能:
|
||||
* - 采集失败记录列表(可重试)
|
||||
* - 数据质量检查结果
|
||||
* - 手动触发质量检查
|
||||
*/
|
||||
|
||||
import { useEffect, useState, useCallback } from 'react'
|
||||
import {
|
||||
fetchDataQuality,
|
||||
runDataQualityCheck,
|
||||
fetchIngestFailures,
|
||||
retryIngestFailure,
|
||||
} from '../dal'
|
||||
import type { IngestFailureItem, DataQualityCheckItem } from '../dal'
|
||||
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
|
||||
|
||||
export default function DataPipelinePage() {
|
||||
const [quality, setQuality] = useState<{ failures: IngestFailureItem[]; checks: DataQualityCheckItem[] } | null>(null)
|
||||
const [failures, setFailures] = useState<IngestFailureItem[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [running, setRunning] = useState(false)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
const [notice, setNotice] = useState<{ ok: boolean; text: string } | null>(null)
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const [q, f] = await Promise.all([fetchDataQuality(), fetchIngestFailures()])
|
||||
setQuality(q)
|
||||
setFailures(f)
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : '加载失败')
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => { load() }, [load])
|
||||
|
||||
const handleRunCheck = async () => {
|
||||
setRunning(true)
|
||||
setNotice(null)
|
||||
try {
|
||||
const res = await runDataQualityCheck()
|
||||
const failed = res.checks.filter(c => !c.passed)
|
||||
setNotice({
|
||||
ok: failed.length === 0,
|
||||
text: failed.length === 0
|
||||
? '数据质量检查通过'
|
||||
: `检查完成: ${failed.length} 项未通过`,
|
||||
})
|
||||
await load()
|
||||
} catch {
|
||||
setNotice({ ok: false, text: '质量检查执行失败' })
|
||||
} finally {
|
||||
setRunning(false)
|
||||
}
|
||||
}
|
||||
|
||||
const handleRetry = async (id: number) => {
|
||||
setNotice(null)
|
||||
try {
|
||||
const res = await retryIngestFailure(id)
|
||||
setNotice({ ok: true, text: res.message })
|
||||
await load()
|
||||
} catch {
|
||||
setNotice({ ok: false, text: '重试操作失败' })
|
||||
}
|
||||
}
|
||||
|
||||
const pendingFailures = failures.filter(f => f.status === 'pending' || f.status === 'retrying')
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<SectionHeader
|
||||
title="数据管线"
|
||||
description="采集失败重试、数据质量检查与监控"
|
||||
action={
|
||||
<button onClick={handleRunCheck} disabled={running} className="btn btn-sm">
|
||||
{running ? <><Spinner /> 检查中</> : '运行质量检查'}
|
||||
</button>
|
||||
}
|
||||
/>
|
||||
|
||||
{notice && (
|
||||
<Alert kind={notice.ok ? 'ok' : 'error'} title={notice.text} onClose={() => setNotice(null)} />
|
||||
)}
|
||||
|
||||
{error && <Alert kind="error" title="加载失败" message={error} onClose={() => setError(null)} />}
|
||||
|
||||
{loading && (
|
||||
<div className="flex justify-center py-12"><Spinner /></div>
|
||||
)}
|
||||
|
||||
{!loading && (
|
||||
<>
|
||||
{/* 采集失败记录 */}
|
||||
<Card>
|
||||
<CardHeader
|
||||
title="采集失败记录"
|
||||
description={pendingFailures.length > 0 ? `${pendingFailures.length} 条待处理` : '暂无待处理失败记录'}
|
||||
/>
|
||||
<CardBody className="px-0 sm:px-0">
|
||||
{failures.length === 0 ? (
|
||||
<p className="py-8 text-center text-xs text-ink-400">暂无采集失败记录</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[640px] text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-ink-200 text-left text-ink-500">
|
||||
<th className="px-4 py-2 font-medium">来源</th>
|
||||
<th className="px-4 py-2 font-medium">实体类型</th>
|
||||
<th className="px-4 py-2 font-medium">错误类型</th>
|
||||
<th className="px-4 py-2 font-medium">重试次数</th>
|
||||
<th className="px-4 py-2 font-medium">状态</th>
|
||||
<th className="px-4 py-2 font-medium">操作</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{failures.map(f => (
|
||||
<tr key={f.id} className="border-b border-ink-100 hover:bg-paper-100">
|
||||
<td className="px-4 py-2 text-xs text-ink-700">{f.source}</td>
|
||||
<td className="px-4 py-2 text-xs text-ink-600">{f.entity_type}</td>
|
||||
<td className="px-4 py-2 text-xs text-ink-600">{f.error_type}</td>
|
||||
<td className="px-4 py-2 text-xs tabular-nums text-ink-500">{f.retry_count}</td>
|
||||
<td className="px-4 py-2">
|
||||
<Badge status={f.status === 'resolved' ? 'success' : f.status === 'pending' ? 'warning' : 'info'}>
|
||||
{f.status}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
{(f.status === 'pending' || f.status === 'retrying') && (
|
||||
<button onClick={() => handleRetry(f.id)} className="btn btn-sm">
|
||||
重试
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</CardBody>
|
||||
</Card>
|
||||
|
||||
{/* 数据质量检查 */}
|
||||
<Card>
|
||||
<CardHeader title="数据质量检查" description="最近 20 条检查结果" />
|
||||
<CardBody className="px-0 sm:px-0">
|
||||
{quality?.checks.length === 0 ? (
|
||||
<p className="py-8 text-center text-xs text-ink-400">暂无质量检查记录,点击右上角「运行质量检查」触发</p>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full min-w-[560px] text-sm">
|
||||
<thead>
|
||||
<tr className="border-b border-ink-200 text-left text-ink-500">
|
||||
<th className="px-4 py-2 font-medium">检查项</th>
|
||||
<th className="px-4 py-2 font-medium">实体</th>
|
||||
<th className="px-4 py-2 font-medium">结果</th>
|
||||
<th className="px-4 py-2 font-medium">严重度</th>
|
||||
<th className="px-4 py-2 font-medium">时间</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{quality?.checks.map(c => (
|
||||
<tr key={c.id} className="border-b border-ink-100 hover:bg-paper-100">
|
||||
<td className="px-4 py-2 text-xs text-ink-700">{c.check_name}</td>
|
||||
<td className="px-4 py-2 text-xs text-ink-600">{c.entity_type}</td>
|
||||
<td className="px-4 py-2">
|
||||
<Badge status={c.passed ? 'success' : 'error'}>
|
||||
{c.passed ? '通过' : '未通过'}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="px-4 py-2">
|
||||
<Badge status={c.severity === 'warning' ? 'warning' : c.severity === 'error' ? 'error' : 'info'}>
|
||||
{c.severity}
|
||||
</Badge>
|
||||
</td>
|
||||
<td className="px-4 py-2 text-2xs text-ink-400">
|
||||
{c.checked_at ? new Date(c.checked_at).toLocaleString('zh-CN', { hour12: false }) : '—'}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</CardBody>
|
||||
</Card>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import MonitoringPage from './pages/Monitoring'
|
||||
import SettingsPage from './pages/Settings'
|
||||
import LogsPage from './pages/Logs'
|
||||
import EvalPage from './pages/EvalPage'
|
||||
import DataPipelinePage from './pages/DataPipeline'
|
||||
|
||||
export const adminRoutes = [
|
||||
{
|
||||
@@ -25,6 +26,7 @@ export const adminRoutes = [
|
||||
{ index: true, element: <Dashboard /> },
|
||||
{ path: 'collection', element: <CollectionPage /> },
|
||||
{ path: 'data-completeness', element: <DataCompletenessPage /> },
|
||||
{ path: 'data-pipeline', element: <DataPipelinePage /> },
|
||||
{ path: 'predictions', element: <PredictionsPage /> },
|
||||
{ path: 'backtest', element: <BacktestPage /> },
|
||||
{ path: 'monitoring', element: <MonitoringPage /> },
|
||||
|
||||
Reference in New Issue
Block a user