Files
Profeto/frontend/src/admin/pages/DataPipeline.tsx
T
shangfangjianandnew-provider/LongCat-2.0 < ae89d0f04f feat: 数据采集管线基础设施接线 + 数据管线管理页
后端:
- bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档)
- 采集失败写入 IngestFailure(死信队列,支持重试)
- 写入 DataLineage(ETL 血缘追踪)
- 新增 DataQualityScheduler(每小时自动质量检查)
- 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发)
- 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试)
- 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛)
- lifespan 启动 quality_scheduler

前端:
- 新增「数据管线」管理页(/admin/data-pipeline)
- 采集失败记录列表(状态/重试次数/错误类型)
- 数据质量检查结果(通过/未通过/严重度)
- 手动触发质量检查按钮
- 失败记录重试按钮
- 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充)
- 导航统一为 React Router Link
- AdminStats 类型扩展(matches/stats/standings 真实计数)

Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
2026-09-21 10:04:26 +08:00

200 lines
7.9 KiB
TypeScript

/**
* Admin 后台 - 数据管线管理页(报刊风)
*
* 功能:
* - 采集失败记录列表(可重试)
* - 数据质量检查结果
* - 手动触发质量检查
*/
import { useEffect, useState, useCallback } from 'react'
import {
fetchDataQuality,
runDataQualityCheck,
fetchIngestFailures,
retryIngestFailure,
} from '../dal'
import type { IngestFailureItem, DataQualityCheckItem } from '../dal'
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
export default function DataPipelinePage() {
const [quality, setQuality] = useState<{ failures: IngestFailureItem[]; checks: DataQualityCheckItem[] } | null>(null)
const [failures, setFailures] = useState<IngestFailureItem[]>([])
const [loading, setLoading] = useState(true)
const [running, setRunning] = useState(false)
const [error, setError] = useState<string | null>(null)
const [notice, setNotice] = useState<{ ok: boolean; text: string } | null>(null)
const load = useCallback(async () => {
setLoading(true)
setError(null)
try {
const [q, f] = await Promise.all([fetchDataQuality(), fetchIngestFailures()])
setQuality(q)
setFailures(f)
} catch (err) {
setError(err instanceof Error ? err.message : '加载失败')
} finally {
setLoading(false)
}
}, [])
useEffect(() => { load() }, [load])
const handleRunCheck = async () => {
setRunning(true)
setNotice(null)
try {
const res = await runDataQualityCheck()
const failed = res.checks.filter(c => !c.passed)
setNotice({
ok: failed.length === 0,
text: failed.length === 0
? '数据质量检查通过'
: `检查完成: ${failed.length} 项未通过`,
})
await load()
} catch {
setNotice({ ok: false, text: '质量检查执行失败' })
} finally {
setRunning(false)
}
}
const handleRetry = async (id: number) => {
setNotice(null)
try {
const res = await retryIngestFailure(id)
setNotice({ ok: true, text: res.message })
await load()
} catch {
setNotice({ ok: false, text: '重试操作失败' })
}
}
const pendingFailures = failures.filter(f => f.status === 'pending' || f.status === 'retrying')
return (
<div className="space-y-6">
<SectionHeader
title="数据管线"
description="采集失败重试、数据质量检查与监控"
action={
<button onClick={handleRunCheck} disabled={running} className="btn btn-sm">
{running ? <><Spinner /> 检查中</> : '运行质量检查'}
</button>
}
/>
{notice && (
<Alert kind={notice.ok ? 'ok' : 'error'} title={notice.text} onClose={() => setNotice(null)} />
)}
{error && <Alert kind="error" title="加载失败" message={error} onClose={() => setError(null)} />}
{loading && (
<div className="flex justify-center py-12"><Spinner /></div>
)}
{!loading && (
<>
{/* 采集失败记录 */}
<Card>
<CardHeader
title="采集失败记录"
description={pendingFailures.length > 0 ? `${pendingFailures.length} 条待处理` : '暂无待处理失败记录'}
/>
<CardBody className="px-0 sm:px-0">
{failures.length === 0 ? (
<p className="py-8 text-center text-xs text-ink-400">暂无采集失败记录</p>
) : (
<div className="overflow-x-auto">
<table className="w-full min-w-[640px] text-sm">
<thead>
<tr className="border-b border-ink-200 text-left text-ink-500">
<th className="px-4 py-2 font-medium">来源</th>
<th className="px-4 py-2 font-medium">实体类型</th>
<th className="px-4 py-2 font-medium">错误类型</th>
<th className="px-4 py-2 font-medium">重试次数</th>
<th className="px-4 py-2 font-medium">状态</th>
<th className="px-4 py-2 font-medium">操作</th>
</tr>
</thead>
<tbody>
{failures.map(f => (
<tr key={f.id} className="border-b border-ink-100 hover:bg-paper-100">
<td className="px-4 py-2 text-xs text-ink-700">{f.source}</td>
<td className="px-4 py-2 text-xs text-ink-600">{f.entity_type}</td>
<td className="px-4 py-2 text-xs text-ink-600">{f.error_type}</td>
<td className="px-4 py-2 text-xs tabular-nums text-ink-500">{f.retry_count}</td>
<td className="px-4 py-2">
<Badge status={f.status === 'resolved' ? 'success' : f.status === 'pending' ? 'warning' : 'info'}>
{f.status}
</Badge>
</td>
<td className="px-4 py-2">
{(f.status === 'pending' || f.status === 'retrying') && (
<button onClick={() => handleRetry(f.id)} className="btn btn-sm">
重试
</button>
)}
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</CardBody>
</Card>
{/* 数据质量检查 */}
<Card>
<CardHeader title="数据质量检查" description="最近 20 条检查结果" />
<CardBody className="px-0 sm:px-0">
{quality?.checks.length === 0 ? (
<p className="py-8 text-center text-xs text-ink-400">暂无质量检查记录,点击右上角「运行质量检查」触发</p>
) : (
<div className="overflow-x-auto">
<table className="w-full min-w-[560px] text-sm">
<thead>
<tr className="border-b border-ink-200 text-left text-ink-500">
<th className="px-4 py-2 font-medium">检查项</th>
<th className="px-4 py-2 font-medium">实体</th>
<th className="px-4 py-2 font-medium">结果</th>
<th className="px-4 py-2 font-medium">严重度</th>
<th className="px-4 py-2 font-medium">时间</th>
</tr>
</thead>
<tbody>
{quality?.checks.map(c => (
<tr key={c.id} className="border-b border-ink-100 hover:bg-paper-100">
<td className="px-4 py-2 text-xs text-ink-700">{c.check_name}</td>
<td className="px-4 py-2 text-xs text-ink-600">{c.entity_type}</td>
<td className="px-4 py-2">
<Badge status={c.passed ? 'success' : 'error'}>
{c.passed ? '通过' : '未通过'}
</Badge>
</td>
<td className="px-4 py-2">
<Badge status={c.severity === 'warning' ? 'warning' : c.severity === 'error' ? 'error' : 'info'}>
{c.severity}
</Badge>
</td>
<td className="px-4 py-2 text-2xs text-ink-400">
{c.checked_at ? new Date(c.checked_at).toLocaleString('zh-CN', { hour12: false }) : '—'}
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</CardBody>
</Card>
</>
)}
</div>
)
}