后端: - bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档) - 采集失败写入 IngestFailure(死信队列,支持重试) - 写入 DataLineage(ETL 血缘追踪) - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发) - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛) - lifespan 启动 quality_scheduler 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 失败记录重试按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
200 lines
7.9 KiB
TypeScript
200 lines
7.9 KiB
TypeScript
/**
|
|
* Admin 后台 - 数据管线管理页(报刊风)
|
|
*
|
|
* 功能:
|
|
* - 采集失败记录列表(可重试)
|
|
* - 数据质量检查结果
|
|
* - 手动触发质量检查
|
|
*/
|
|
|
|
import { useEffect, useState, useCallback } from 'react'
|
|
import {
|
|
fetchDataQuality,
|
|
runDataQualityCheck,
|
|
fetchIngestFailures,
|
|
retryIngestFailure,
|
|
} from '../dal'
|
|
import type { IngestFailureItem, DataQualityCheckItem } from '../dal'
|
|
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
|
|
|
|
export default function DataPipelinePage() {
|
|
const [quality, setQuality] = useState<{ failures: IngestFailureItem[]; checks: DataQualityCheckItem[] } | null>(null)
|
|
const [failures, setFailures] = useState<IngestFailureItem[]>([])
|
|
const [loading, setLoading] = useState(true)
|
|
const [running, setRunning] = useState(false)
|
|
const [error, setError] = useState<string | null>(null)
|
|
const [notice, setNotice] = useState<{ ok: boolean; text: string } | null>(null)
|
|
|
|
const load = useCallback(async () => {
|
|
setLoading(true)
|
|
setError(null)
|
|
try {
|
|
const [q, f] = await Promise.all([fetchDataQuality(), fetchIngestFailures()])
|
|
setQuality(q)
|
|
setFailures(f)
|
|
} catch (err) {
|
|
setError(err instanceof Error ? err.message : '加载失败')
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}, [])
|
|
|
|
useEffect(() => { load() }, [load])
|
|
|
|
const handleRunCheck = async () => {
|
|
setRunning(true)
|
|
setNotice(null)
|
|
try {
|
|
const res = await runDataQualityCheck()
|
|
const failed = res.checks.filter(c => !c.passed)
|
|
setNotice({
|
|
ok: failed.length === 0,
|
|
text: failed.length === 0
|
|
? '数据质量检查通过'
|
|
: `检查完成: ${failed.length} 项未通过`,
|
|
})
|
|
await load()
|
|
} catch {
|
|
setNotice({ ok: false, text: '质量检查执行失败' })
|
|
} finally {
|
|
setRunning(false)
|
|
}
|
|
}
|
|
|
|
const handleRetry = async (id: number) => {
|
|
setNotice(null)
|
|
try {
|
|
const res = await retryIngestFailure(id)
|
|
setNotice({ ok: true, text: res.message })
|
|
await load()
|
|
} catch {
|
|
setNotice({ ok: false, text: '重试操作失败' })
|
|
}
|
|
}
|
|
|
|
const pendingFailures = failures.filter(f => f.status === 'pending' || f.status === 'retrying')
|
|
|
|
return (
|
|
<div className="space-y-6">
|
|
<SectionHeader
|
|
title="数据管线"
|
|
description="采集失败重试、数据质量检查与监控"
|
|
action={
|
|
<button onClick={handleRunCheck} disabled={running} className="btn btn-sm">
|
|
{running ? <><Spinner /> 检查中</> : '运行质量检查'}
|
|
</button>
|
|
}
|
|
/>
|
|
|
|
{notice && (
|
|
<Alert kind={notice.ok ? 'ok' : 'error'} title={notice.text} onClose={() => setNotice(null)} />
|
|
)}
|
|
|
|
{error && <Alert kind="error" title="加载失败" message={error} onClose={() => setError(null)} />}
|
|
|
|
{loading && (
|
|
<div className="flex justify-center py-12"><Spinner /></div>
|
|
)}
|
|
|
|
{!loading && (
|
|
<>
|
|
{/* 采集失败记录 */}
|
|
<Card>
|
|
<CardHeader
|
|
title="采集失败记录"
|
|
description={pendingFailures.length > 0 ? `${pendingFailures.length} 条待处理` : '暂无待处理失败记录'}
|
|
/>
|
|
<CardBody className="px-0 sm:px-0">
|
|
{failures.length === 0 ? (
|
|
<p className="py-8 text-center text-xs text-ink-400">暂无采集失败记录</p>
|
|
) : (
|
|
<div className="overflow-x-auto">
|
|
<table className="w-full min-w-[640px] text-sm">
|
|
<thead>
|
|
<tr className="border-b border-ink-200 text-left text-ink-500">
|
|
<th className="px-4 py-2 font-medium">来源</th>
|
|
<th className="px-4 py-2 font-medium">实体类型</th>
|
|
<th className="px-4 py-2 font-medium">错误类型</th>
|
|
<th className="px-4 py-2 font-medium">重试次数</th>
|
|
<th className="px-4 py-2 font-medium">状态</th>
|
|
<th className="px-4 py-2 font-medium">操作</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{failures.map(f => (
|
|
<tr key={f.id} className="border-b border-ink-100 hover:bg-paper-100">
|
|
<td className="px-4 py-2 text-xs text-ink-700">{f.source}</td>
|
|
<td className="px-4 py-2 text-xs text-ink-600">{f.entity_type}</td>
|
|
<td className="px-4 py-2 text-xs text-ink-600">{f.error_type}</td>
|
|
<td className="px-4 py-2 text-xs tabular-nums text-ink-500">{f.retry_count}</td>
|
|
<td className="px-4 py-2">
|
|
<Badge status={f.status === 'resolved' ? 'success' : f.status === 'pending' ? 'warning' : 'info'}>
|
|
{f.status}
|
|
</Badge>
|
|
</td>
|
|
<td className="px-4 py-2">
|
|
{(f.status === 'pending' || f.status === 'retrying') && (
|
|
<button onClick={() => handleRetry(f.id)} className="btn btn-sm">
|
|
重试
|
|
</button>
|
|
)}
|
|
</td>
|
|
</tr>
|
|
))}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
)}
|
|
</CardBody>
|
|
</Card>
|
|
|
|
{/* 数据质量检查 */}
|
|
<Card>
|
|
<CardHeader title="数据质量检查" description="最近 20 条检查结果" />
|
|
<CardBody className="px-0 sm:px-0">
|
|
{quality?.checks.length === 0 ? (
|
|
<p className="py-8 text-center text-xs text-ink-400">暂无质量检查记录,点击右上角「运行质量检查」触发</p>
|
|
) : (
|
|
<div className="overflow-x-auto">
|
|
<table className="w-full min-w-[560px] text-sm">
|
|
<thead>
|
|
<tr className="border-b border-ink-200 text-left text-ink-500">
|
|
<th className="px-4 py-2 font-medium">检查项</th>
|
|
<th className="px-4 py-2 font-medium">实体</th>
|
|
<th className="px-4 py-2 font-medium">结果</th>
|
|
<th className="px-4 py-2 font-medium">严重度</th>
|
|
<th className="px-4 py-2 font-medium">时间</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{quality?.checks.map(c => (
|
|
<tr key={c.id} className="border-b border-ink-100 hover:bg-paper-100">
|
|
<td className="px-4 py-2 text-xs text-ink-700">{c.check_name}</td>
|
|
<td className="px-4 py-2 text-xs text-ink-600">{c.entity_type}</td>
|
|
<td className="px-4 py-2">
|
|
<Badge status={c.passed ? 'success' : 'error'}>
|
|
{c.passed ? '通过' : '未通过'}
|
|
</Badge>
|
|
</td>
|
|
<td className="px-4 py-2">
|
|
<Badge status={c.severity === 'warning' ? 'warning' : c.severity === 'error' ? 'error' : 'info'}>
|
|
{c.severity}
|
|
</Badge>
|
|
</td>
|
|
<td className="px-4 py-2 text-2xs text-ink-400">
|
|
{c.checked_at ? new Date(c.checked_at).toLocaleString('zh-CN', { hour12: false }) : '—'}
|
|
</td>
|
|
</tr>
|
|
))}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
)}
|
|
</CardBody>
|
|
</Card>
|
|
</>
|
|
)}
|
|
</div>
|
|
)
|
|
}
|