- 新表 ingest_jobs(迁移 0019): id UUID/task/params JSONB/
status(pending|running|success|failed,CheckConstraint)/
result JSONB(统计摘要)/error/created_at/started_at/finished_at
- POST /ingest/bzzoiro: 启动后台前创建 pending job,响应返回 job_id;
仍 require_admin。后台 _run_bzzoiro 流转 running→success/failed,
result 按子任务(events/standings/stats)记录摘要(errors 截断 10 条)
- _update_job 尽力而为: 状态更新失败只记日志,绝不拖垮采集主流程;
与 IngestFailure 死信独立(行级 vs 任务级,可同时存在)
- 新增 admin 端点(挂 /api/v1/admin 路由,路由级 require_admin):
GET /admin/ingest/jobs/{job_id} 与 GET /admin/ingest/jobs?limit&status
- 前端采集页: 提交后凭 job_id 3 秒轮询,终态展示结果摘要/失败原因;
无 job_id 时回退旧的 30 秒盲等 + 系统日志提示
- 测试 11 项: 建 job+job_id 契约、非法 task 422、成功/失败/all 流转、
update 失败不拖垮采集、部分失败仍 success、admin 端点 200/404/列表、
结构守护(job 路由在 admin 路由且带 require_admin)
- 禁止项确认: 未动分批 UoW、BzzoiroSource、死信与 Bronze 写入;
docs(01/03/05/07/README)同步 13 张表与端点说明
426 lines
17 KiB
TypeScript
426 lines
17 KiB
TypeScript
/**
|
|
* Admin 后台 - 数据采集页面(bzzoiro 单一数据源)
|
|
*
|
|
* 三个采集任务:
|
|
* events — 比赛日程与比分
|
|
* standings — 联赛积分榜
|
|
* stats — 已完赛比赛详细统计回填
|
|
* all — 依次执行以上三项
|
|
*
|
|
* 响应式布局: 移动端单列,桌面端双列
|
|
*/
|
|
|
|
import { useEffect, useState, useCallback, useRef } from 'react'
|
|
import { triggerCollection, fetchLeagues, fetchIngestStatus, fetchIngestJob } from '../dal'
|
|
import type { IngestSourceStatus } from '../types'
|
|
import type { CollectionRequest, League, IngestJob } from '../types'
|
|
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
|
|
|
|
const TASKS = [
|
|
{ value: 'events', label: '比赛数据', desc: '赛程 / 比分 / 未开赛安排', icon: '⚽' },
|
|
{ value: 'standings', label: '积分榜', desc: '联赛排名 / 积分 / xG差 / 近期走势', icon: '🏆' },
|
|
{ value: 'stats', label: '统计回填', desc: '已完赛比赛的 xG / 射门 / 控球等详细统计', icon: '📊' },
|
|
{ value: 'all', label: '全量采集', desc: '依次采集比赛 + 积分榜 + 统计回填', icon: '⏵⏵' },
|
|
] as const
|
|
|
|
type TaskStatus = 'idle' | 'running' | 'done' | 'error'
|
|
|
|
const JOB_STATUS_LABEL: Record<IngestJob['status'], string> = {
|
|
pending: '等待中',
|
|
running: '执行中',
|
|
success: '已完成',
|
|
failed: '失败',
|
|
}
|
|
|
|
/** job.result 摘要 → 多行文本(错误只显示条数,明细见「采集失败」页与系统日志) */
|
|
function summarizeResult(result: Record<string, any> | null): string[] {
|
|
if (!result) return []
|
|
return Object.entries(result).map(([k, v]) => {
|
|
if (v && typeof v === 'object') {
|
|
const bits: string[] = []
|
|
for (const [kk, vv] of Object.entries(v)) {
|
|
if (kk === 'errors') {
|
|
if (Array.isArray(vv) && vv.length > 0) bits.push(`错误 ${vv.length} 条`)
|
|
} else if (vv !== null && vv !== undefined) {
|
|
bits.push(`${kk} ${vv}`)
|
|
}
|
|
}
|
|
return `${k}: ${bits.length ? bits.join(' · ') : '无变更'}`
|
|
}
|
|
return `${k}: ${String(v)}`
|
|
})
|
|
}
|
|
|
|
export default function CollectionPage() {
|
|
const [leagues, setLeagues] = useState<League[]>([])
|
|
const [task, setTask] = useState<string>('events')
|
|
const [leagueCode, setLeagueCode] = useState('')
|
|
|
|
// 从 URL 查询参数预填充(支持从「数据完整性」页跳转)
|
|
useEffect(() => {
|
|
const sp = new URLSearchParams(window.location.search)
|
|
const taskParam = sp.get('task')
|
|
const leagueParam = sp.get('league')
|
|
if (taskParam && ['events', 'standings', 'stats', 'all'].includes(taskParam)) {
|
|
setTask(taskParam)
|
|
}
|
|
if (leagueParam) {
|
|
setLeagueCode(leagueParam)
|
|
}
|
|
}, [])
|
|
const [dateFrom, setDateFrom] = useState('')
|
|
const [dateTo, setDateTo] = useState('')
|
|
const [season, setSeason] = useState('')
|
|
const [ingestStatus, setIngestStatus] = useState('')
|
|
const [limit, setLimit] = useState(100)
|
|
const [loading, setLoading] = useState(false)
|
|
const [error, setError] = useState<string | null>(null)
|
|
const [result, setResult] = useState<{ title: string; detail: string } | null>(null)
|
|
|
|
// 任务进度反馈
|
|
const [taskStatus, setTaskStatus] = useState<TaskStatus>('idle')
|
|
const [taskStartedAt, setTaskStartedAt] = useState<number | null>(null)
|
|
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
|
const jobPollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
|
const [ingestSnap, setIngestSnap] = useState<IngestSourceStatus | null>(null)
|
|
// 采集任务状态(ingest_jobs):提交后有 job_id 即轮询,替代「30 秒盲等」
|
|
const [job, setJob] = useState<IngestJob | null>(null)
|
|
|
|
const loadLeagues = useCallback(async () => {
|
|
const lg = await fetchLeagues()
|
|
setLeagues(lg)
|
|
}, [])
|
|
|
|
useEffect(() => { loadLeagues() }, [loadLeagues])
|
|
|
|
// 轮询采集状态(任务启动后)
|
|
const startPolling = useCallback(() => {
|
|
if (pollRef.current) clearInterval(pollRef.current)
|
|
pollRef.current = setInterval(async () => {
|
|
try {
|
|
const { sources } = await fetchIngestStatus()
|
|
const bz = sources.find(s => s.name === 'bzzoiro')
|
|
if (bz) setIngestSnap(bz)
|
|
} catch { /* ignore */ }
|
|
}, 5_000)
|
|
}, [])
|
|
|
|
const stopPolling = useCallback(() => {
|
|
if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null }
|
|
}, [])
|
|
|
|
// job 轮询:3 秒一次,终态(success/failed)自动停止;10 分钟兜底防泄漏
|
|
const stopJobPolling = useCallback(() => {
|
|
if (jobPollRef.current) { clearInterval(jobPollRef.current); jobPollRef.current = null }
|
|
}, [])
|
|
|
|
const startJobPolling = useCallback((jobId: string) => {
|
|
stopJobPolling()
|
|
let ticks = 0
|
|
jobPollRef.current = setInterval(async () => {
|
|
ticks += 1
|
|
if (ticks > 200) { stopJobPolling(); setTaskStatus('done'); return }
|
|
try {
|
|
const j = await fetchIngestJob(jobId)
|
|
setJob(j)
|
|
if (j.status === 'success') { stopJobPolling(); setTaskStatus('done') }
|
|
else if (j.status === 'failed') { stopJobPolling(); setTaskStatus('error') }
|
|
} catch { /* 网络抖动忽略,下个周期重试 */ }
|
|
}, 3_000)
|
|
}, [stopJobPolling])
|
|
|
|
useEffect(() => () => { stopPolling(); stopJobPolling() }, [stopPolling, stopJobPolling])
|
|
|
|
const isEventsTask = task === 'events' || task === 'all'
|
|
|
|
async function handleSubmit(e: React.FormEvent) {
|
|
e.preventDefault()
|
|
setError(null)
|
|
setResult(null)
|
|
setJob(null)
|
|
setLoading(true)
|
|
setTaskStatus('running')
|
|
setTaskStartedAt(Date.now())
|
|
|
|
try {
|
|
const body: CollectionRequest = {
|
|
source: 'bzzoiro',
|
|
leagues: leagueCode ? [leagueCode] : undefined,
|
|
task: task as CollectionRequest['task'],
|
|
limit,
|
|
season: season || undefined,
|
|
status: ingestStatus || undefined,
|
|
date_from: isEventsTask ? dateFrom || undefined : undefined,
|
|
date_to: isEventsTask ? dateTo || undefined : undefined,
|
|
}
|
|
const resp = await triggerCollection(body)
|
|
const jobId: string | undefined = resp?.job_id
|
|
if (jobId) {
|
|
// 有 job_id:轮询任务状态直至终态(3 秒/次)
|
|
setResult({
|
|
title: '采集任务已启动',
|
|
detail: `任务 ID: ${jobId}。正在跟踪执行进度,完成后展示结果摘要。`,
|
|
})
|
|
startJobPolling(jobId)
|
|
} else {
|
|
// 兜底:后端未返回 job_id(旧版本),退回「30 秒盲等 + 系统日志」
|
|
setResult({
|
|
title: '采集任务已启动',
|
|
detail: '正在后台执行(上游限速时可能需要数分钟)。完成结果与错误请到「系统日志」页查看(支持自动刷新)。',
|
|
})
|
|
startPolling()
|
|
setTimeout(() => {
|
|
setTaskStatus('done')
|
|
stopPolling()
|
|
}, 30_000)
|
|
}
|
|
} catch (err: unknown) {
|
|
setTaskStatus('error')
|
|
setError(err instanceof Error ? err.message : '采集触发失败')
|
|
stopPolling()
|
|
stopJobPolling()
|
|
} finally {
|
|
setLoading(false)
|
|
}
|
|
}
|
|
|
|
const elapsed = taskStartedAt ? Math.round((Date.now() - taskStartedAt) / 1000) : 0
|
|
|
|
return (
|
|
<div className="space-y-6">
|
|
<SectionHeader
|
|
title="数据采集"
|
|
description="bzzoiro 单一数据源:比赛数据、积分榜、比赛统计三条管线。采集为后台异步执行。"
|
|
/>
|
|
|
|
<div className="grid gap-6 lg:grid-cols-2">
|
|
{/* 采集表单 */}
|
|
<Card>
|
|
<CardHeader title="新建采集任务" />
|
|
<CardBody>
|
|
<form onSubmit={handleSubmit} className="space-y-4">
|
|
{/* 任务类型 */}
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">采集任务</label>
|
|
<div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
|
|
{TASKS.map(t => (
|
|
<button
|
|
key={t.value}
|
|
type="button"
|
|
onClick={() => setTask(t.value)}
|
|
className={`rounded-lg border px-3 py-2 text-left text-xs transition-colors ${
|
|
task === t.value
|
|
? 'border-brand-500 bg-brand-50 text-brand-700'
|
|
: 'border-ink-200 text-ink-600 hover:border-ink-300'
|
|
}`}
|
|
>
|
|
<span className="mr-1">{t.icon}</span>
|
|
<span className="font-medium">{t.label}</span>
|
|
</button>
|
|
))}
|
|
</div>
|
|
</div>
|
|
|
|
{/* 联赛选择 */}
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">联赛</label>
|
|
<select
|
|
value={leagueCode}
|
|
onChange={e => setLeagueCode(e.target.value)}
|
|
className="field w-full"
|
|
>
|
|
<option value="">全部联赛</option>
|
|
{leagues.map(l => (
|
|
<option key={l.code} value={l.code}>{l.name_zh || l.name}</option>
|
|
))}
|
|
</select>
|
|
</div>
|
|
|
|
{/* events/all 任务专用: 比赛状态 + 日期 */}
|
|
{isEventsTask && (
|
|
<>
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">比赛状态</label>
|
|
<select
|
|
value={ingestStatus}
|
|
onChange={e => setIngestStatus(e.target.value)}
|
|
className="field w-full"
|
|
>
|
|
<option value="">全部(已完赛 + 未开赛)</option>
|
|
<option value="finished">仅已完赛</option>
|
|
<option value="scheduled">仅未开赛</option>
|
|
</select>
|
|
</div>
|
|
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">起始日期</label>
|
|
<input
|
|
type="date"
|
|
value={dateFrom}
|
|
onChange={e => setDateFrom(e.target.value)}
|
|
className="field w-full"
|
|
/>
|
|
</div>
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">结束日期</label>
|
|
<input
|
|
type="date"
|
|
value={dateTo}
|
|
onChange={e => setDateTo(e.target.value)}
|
|
className="field w-full"
|
|
/>
|
|
</div>
|
|
</div>
|
|
</>
|
|
)}
|
|
|
|
{/* standings 任务专用: 赛季 */}
|
|
{(task === 'standings') && (
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">赛季(留空取当前赛季)</label>
|
|
<input
|
|
type="text"
|
|
value={season}
|
|
onChange={e => setSeason(e.target.value)}
|
|
placeholder="如 2026-2027"
|
|
className="field w-full"
|
|
/>
|
|
</div>
|
|
)}
|
|
|
|
{/* stats 任务专用: 回填数量 */}
|
|
{(task === 'stats') && (
|
|
<div>
|
|
<label className="mb-1.5 block text-xs text-ink-500">单次最大回填比赛数(1-500)</label>
|
|
<input
|
|
type="number"
|
|
min={1}
|
|
max={500}
|
|
value={limit}
|
|
onChange={e => setLimit(parseInt(e.target.value) || 100)}
|
|
className="field w-full"
|
|
/>
|
|
<p className="mt-1 text-2xs text-ink-400">
|
|
仅回填已有 source_event_id 且无统计的比赛(增量),上游限速约 1.2 秒/次。
|
|
</p>
|
|
</div>
|
|
)}
|
|
|
|
{/* 消息提示 */}
|
|
{error && <Alert kind="error" title="采集失败" message={error} onClose={() => setError(null)} />}
|
|
{result && (
|
|
<Alert
|
|
kind="ok"
|
|
title={result.title}
|
|
message={result.detail || undefined}
|
|
onClose={() => setResult(null)}
|
|
/>
|
|
)}
|
|
|
|
{/* 提交按钮 */}
|
|
<button type="submit" disabled={loading} className="btn btn-solid w-full">
|
|
{loading ? (<><Spinner /> 采集中</>) : '触发采集'}
|
|
</button>
|
|
</form>
|
|
</CardBody>
|
|
</Card>
|
|
|
|
{/* 任务状态 + 数据源说明 */}
|
|
<div className="space-y-6">
|
|
{/* 任务进度 */}
|
|
<Card>
|
|
<CardHeader title="任务状态" />
|
|
<CardBody>
|
|
{taskStatus === 'idle' && (
|
|
<p className="text-xs text-ink-400">尚未触发任务。</p>
|
|
)}
|
|
{taskStatus === 'running' && (
|
|
<div className="space-y-3">
|
|
<div className="flex items-center gap-2 text-xs text-ink-700">
|
|
<Spinner />
|
|
<span>
|
|
{job
|
|
? `${JOB_STATUS_LABEL[job.status]}(轮询中),已运行 ${elapsed}s…`
|
|
: `任务执行中,已运行 ${elapsed}s…`}
|
|
</span>
|
|
</div>
|
|
<p className="text-2xs text-ink-400">
|
|
后台异步执行,关闭页面不影响结果。可稍后查看「系统日志」确认完成。
|
|
</p>
|
|
</div>
|
|
)}
|
|
{taskStatus === 'done' && (
|
|
<div className="space-y-2">
|
|
<div className="flex items-center gap-2 text-xs text-emerald-700">
|
|
<span className="inline-block h-2 w-2 rounded-full bg-emerald-500" />
|
|
<span>
|
|
{job
|
|
? `采集完成(${JOB_STATUS_LABEL[job.status]})`
|
|
: '任务已提交,后台执行中(可能尚未完成)'}
|
|
</span>
|
|
</div>
|
|
{job?.result && summarizeResult(job.result).length > 0 && (
|
|
<div className="rounded-md bg-ink-50 px-3 py-2">
|
|
{summarizeResult(job.result).map((line, i) => (
|
|
<p key={i} className="font-mono text-2xs leading-relaxed text-ink-600">{line}</p>
|
|
))}
|
|
</div>
|
|
)}
|
|
{!job && (
|
|
<p className="text-2xs text-ink-400">
|
|
采集耗时取决于数据量。请到「系统日志」页查看最终结果。
|
|
</p>
|
|
)}
|
|
</div>
|
|
)}
|
|
{taskStatus === 'error' && (
|
|
<div className="space-y-2">
|
|
<p className="text-xs text-press">
|
|
{job?.status === 'failed' ? '采集任务失败' : '任务触发失败,请检查配置或网络。'}
|
|
</p>
|
|
{job?.error && (
|
|
<p className="break-all rounded-md bg-red-50 px-3 py-2 font-mono text-2xs text-press">
|
|
{job.error}
|
|
</p>
|
|
)}
|
|
</div>
|
|
)}
|
|
{ingestSnap?.last_success_at && (
|
|
<div className="mt-3 border-t border-ink-100 pt-3">
|
|
<p className="text-2xs text-ink-400">
|
|
bzzoiro 最近一次采集: {new Date(ingestSnap.last_success_at).toLocaleString('zh-CN', { hour12: false })}
|
|
</p>
|
|
</div>
|
|
)}
|
|
</CardBody>
|
|
</Card>
|
|
|
|
{/* 数据源说明 */}
|
|
<Card>
|
|
<CardHeader title="采集任务说明" />
|
|
<CardBody>
|
|
<div className="space-y-3">
|
|
{TASKS.map(t => (
|
|
<div key={t.value} className="border-b border-ink-200 px-1 py-3 last:border-b-0">
|
|
<div className="flex items-center gap-3">
|
|
<span>{t.icon}</span>
|
|
<Badge status="info">{t.label}</Badge>
|
|
<p className="text-xs text-ink-600">{t.desc}</p>
|
|
</div>
|
|
</div>
|
|
))}
|
|
</div>
|
|
|
|
<p className="mt-4 border-l-2 border-ink-300 pl-3 text-2xs leading-relaxed text-ink-500">
|
|
采集接口需要管理员登录(401 表示登录已过期)。
|
|
各管线基于 bzzoiro 单一数据源(Understat / injuries 已移除)。
|
|
「统计回填」依赖「比赛数据」管线写入的 source_event_id,请先完成比赛采集。
|
|
</p>
|
|
</CardBody>
|
|
</Card>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
)
|
|
}
|