feat(ingest): 采集任务状态跟踪(ingest_jobs),解决 fire-and-forget 不可观测

- 新表 ingest_jobs(迁移 0019): id UUID/task/params JSONB/ status(pending|running|success|failed,CheckConstraint)/ result JSONB(统计摘要)/error/created_at/started_at/finished_at - POST /ingest/bzzoiro: 启动后台前创建 pending job,响应返回 job_id; 仍 require_admin。后台 _run_bzzoiro 流转 running→success/failed, result 按子任务(events/standings/stats)记录摘要(errors 截断 10 条) - _update_job 尽力而为: 状态更新失败只记日志,绝不拖垮采集主流程; 与 IngestFailure 死信独立(行级 vs 任务级,可同时存在) - 新增 admin 端点(挂 /api/v1/admin 路由,路由级 require_admin): GET /admin/ingest/jobs/{job_id} 与 GET /admin/ingest/jobs?limit&status - 前端采集页: 提交后凭 job_id 3 秒轮询,终态展示结果摘要/失败原因; 无 job_id 时回退旧的 30 秒盲等 + 系统日志提示 - 测试 11 项: 建 job+job_id 契约、非法 task 422、成功/失败/all 流转、 update 失败不拖垮采集、部分失败仍 success、admin 端点 200/404/列表、结构守护(job 路由在 admin 路由且带 require_admin) - 禁止项确认: 未动分批 UoW、BzzoiroSource、死信与 Bronze 写入; docs(01/03/05/07/README)同步 13 张表与端点说明
2026-09-21 21:19:21 +08:00
parent 6bdb1f8ae6
commit 983363dab7
13 changed files with 666 additions and 36 deletions
@@ -21,6 +21,7 @@ import type {
  LLMAgentConfig,
  LogEntry,
  IngestSourceStatus,
+  IngestJob,
  MatchDetailOut,
  MatchContextOut,
  AdminStats,
@@ -71,6 +72,11 @@ export async function triggerCollection(req: CollectionRequest): Promise<any> {
  return api.post(`${API_BASE}/ingest/bzzoiro`, body)
 }

+// 查询采集任务状态(ingest_jobs;提交采集返回 job_id 后轮询用)
+export async function fetchIngestJob(jobId: string): Promise<IngestJob> {
+  return api.get<IngestJob>(`${API_BASE}/admin/ingest/jobs/${jobId}`)
+}
+
 // ── 预测管理 ────────────────────────────────────────────────────

 export async function triggerPrediction(req: { match_id: number; mode?: string }): Promise<any> {
@@ -11,9 +11,9 @@
 */

 import { useEffect, useState, useCallback, useRef } from 'react'
-import { triggerCollection, fetchLeagues, fetchIngestStatus } from '../dal'
+import { triggerCollection, fetchLeagues, fetchIngestStatus, fetchIngestJob } from '../dal'
 import type { IngestSourceStatus } from '../types'
-import type { CollectionRequest, League } from '../types'
+import type { CollectionRequest, League, IngestJob } from '../types'
 import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'

 const TASKS = [
@@ -25,6 +25,32 @@ const TASKS = [

 type TaskStatus = 'idle' | 'running' | 'done' | 'error'

+const JOB_STATUS_LABEL: Record<IngestJob['status'], string> = {
+  pending: '等待中',
+  running: '执行中',
+  success: '已完成',
+  failed: '失败',
+}
+
+/** job.result 摘要 → 多行文本(错误只显示条数,明细见「采集失败」页与系统日志) */
+function summarizeResult(result: Record<string, any> | null): string[] {
+  if (!result) return []
+  return Object.entries(result).map(([k, v]) => {
+    if (v && typeof v === 'object') {
+      const bits: string[] = []
+      for (const [kk, vv] of Object.entries(v)) {
+        if (kk === 'errors') {
+          if (Array.isArray(vv) && vv.length > 0) bits.push(`错误 ${vv.length} 条`)
+        } else if (vv !== null && vv !== undefined) {
+          bits.push(`${kk} ${vv}`)
+        }
+      }
+      return `${k}: ${bits.length ? bits.join(' · ') : '无变更'}`
+    }
+    return `${k}: ${String(v)}`
+  })
+}
+
 export default function CollectionPage() {
  const [leagues, setLeagues] = useState<League[]>([])
  const [task, setTask] = useState<string>('events')
@@ -55,7 +81,10 @@ export default function CollectionPage() {
  const [taskStatus, setTaskStatus] = useState<TaskStatus>('idle')
  const [taskStartedAt, setTaskStartedAt] = useState<number | null>(null)
  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
+  const jobPollRef = useRef<ReturnType<typeof setInterval> | null>(null)
  const [ingestSnap, setIngestSnap] = useState<IngestSourceStatus | null>(null)
+  // 采集任务状态(ingest_jobs):提交后有 job_id 即轮询,替代「30 秒盲等」
+  const [job, setJob] = useState<IngestJob | null>(null)

  const loadLeagues = useCallback(async () => {
    const lg = await fetchLeagues()
@@ -80,7 +109,27 @@ export default function CollectionPage() {
    if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null }
  }, [])

-  useEffect(() => () => stopPolling(), [stopPolling])
+  // job 轮询:3 秒一次,终态(success/failed)自动停止;10 分钟兜底防泄漏
+  const stopJobPolling = useCallback(() => {
+    if (jobPollRef.current) { clearInterval(jobPollRef.current); jobPollRef.current = null }
+  }, [])
+
+  const startJobPolling = useCallback((jobId: string) => {
+    stopJobPolling()
+    let ticks = 0
+    jobPollRef.current = setInterval(async () => {
+      ticks += 1
+      if (ticks > 200) { stopJobPolling(); setTaskStatus('done'); return }
+      try {
+        const j = await fetchIngestJob(jobId)
+        setJob(j)
+        if (j.status === 'success') { stopJobPolling(); setTaskStatus('done') }
+        else if (j.status === 'failed') { stopJobPolling(); setTaskStatus('error') }
+      } catch { /* 网络抖动忽略,下个周期重试 */ }
+    }, 3_000)
+  }, [stopJobPolling])
+
+  useEffect(() => () => { stopPolling(); stopJobPolling() }, [stopPolling, stopJobPolling])

  const isEventsTask = task === 'events' || task === 'all'

@@ -88,6 +137,7 @@ export default function CollectionPage() {
    e.preventDefault()
    setError(null)
    setResult(null)
+    setJob(null)
    setLoading(true)
    setTaskStatus('running')
    setTaskStartedAt(Date.now())
@@ -103,22 +153,32 @@ export default function CollectionPage() {
        date_from: isEventsTask ? dateFrom || undefined : undefined,
        date_to: isEventsTask ? dateTo || undefined : undefined,
      }
-      await triggerCollection(body)
-      setResult({
-        title: '采集任务已启动',
-        detail: '正在后台执行(上游限速时可能需要数分钟)。完成结果与错误请到「系统日志」页查看(支持自动刷新)。',
-      })
-      // 启动轮询,跟踪状态
-      startPolling()
-      // 30 秒后自动停止轮询并标记完成
-      setTimeout(() => {
-        setTaskStatus('done')
-        stopPolling()
-      }, 30_000)
+      const resp = await triggerCollection(body)
+      const jobId: string | undefined = resp?.job_id
+      if (jobId) {
+        // 有 job_id:轮询任务状态直至终态(3 秒/次)
+        setResult({
+          title: '采集任务已启动',
+          detail: `任务 ID: ${jobId}。正在跟踪执行进度,完成后展示结果摘要。`,
+        })
+        startJobPolling(jobId)
+      } else {
+        // 兜底:后端未返回 job_id(旧版本),退回「30 秒盲等 + 系统日志」
+        setResult({
+          title: '采集任务已启动',
+          detail: '正在后台执行(上游限速时可能需要数分钟)。完成结果与错误请到「系统日志」页查看(支持自动刷新)。',
+        })
+        startPolling()
+        setTimeout(() => {
+          setTaskStatus('done')
+          stopPolling()
+        }, 30_000)
+      }
    } catch (err: unknown) {
      setTaskStatus('error')
      setError(err instanceof Error ? err.message : '采集触发失败')
      stopPolling()
+      stopJobPolling()
    } finally {
      setLoading(false)
    }
@@ -278,7 +338,11 @@ export default function CollectionPage() {
                <div className="space-y-3">
                  <div className="flex items-center gap-2 text-xs text-ink-700">
                    <Spinner />
-                    <span>任务执行中,已运行 {elapsed}s…</span>
+                    <span>
+                      {job
+                        ? `${JOB_STATUS_LABEL[job.status]}(轮询中),已运行 ${elapsed}s…`
+                        : `任务执行中,已运行 ${elapsed}s…`}
+                    </span>
                  </div>
                  <p className="text-2xs text-ink-400">
                    后台异步执行,关闭页面不影响结果。可稍后查看「系统日志」确认完成。
@@ -289,15 +353,37 @@ export default function CollectionPage() {
                <div className="space-y-2">
                  <div className="flex items-center gap-2 text-xs text-emerald-700">
                    <span className="inline-block h-2 w-2 rounded-full bg-emerald-500" />
-                    <span>任务已提交,后台执行中(可能尚未完成)</span>
+                    <span>
+                      {job
+                        ? `采集完成(${JOB_STATUS_LABEL[job.status]})`
+                        : '任务已提交,后台执行中(可能尚未完成)'}
+                    </span>
                  </div>
-                  <p className="text-2xs text-ink-400">
-                    采集耗时取决于数据量。请到「系统日志」页查看最终结果。
-                  </p>
+                  {job?.result && summarizeResult(job.result).length > 0 && (
+                    <div className="rounded-md bg-ink-50 px-3 py-2">
+                      {summarizeResult(job.result).map((line, i) => (
+                        <p key={i} className="font-mono text-2xs leading-relaxed text-ink-600">{line}</p>
+                      ))}
+                    </div>
+                  )}
+                  {!job && (
+                    <p className="text-2xs text-ink-400">
+                      采集耗时取决于数据量。请到「系统日志」页查看最终结果。
+                    </p>
+                  )}
                </div>
              )}
              {taskStatus === 'error' && (
-                <p className="text-xs text-press">任务触发失败,请检查配置或网络。</p>
+                <div className="space-y-2">
+                  <p className="text-xs text-press">
+                    {job?.status === 'failed' ? '采集任务失败' : '任务触发失败,请检查配置或网络。'}
+                  </p>
+                  {job?.error && (
+                    <p className="break-all rounded-md bg-red-50 px-3 py-2 font-mono text-2xs text-press">
+                      {job.error}
+                    </p>
+                  )}
+                </div>
              )}
              {ingestSnap?.last_success_at && (
                <div className="mt-3 border-t border-ink-100 pt-3">
@@ -103,6 +103,19 @@ export interface CollectionRequest {
  date_to?: string
 }

+// 采集任务状态(ingest_jobs,提交采集后轮询)
+export interface IngestJob {
+  id: string
+  task: 'events' | 'standings' | 'stats' | 'all'
+  params: Record<string, unknown>
+  status: 'pending' | 'running' | 'success' | 'failed'
+  result: Record<string, any> | null
+  error: string | null
+  created_at: string | null
+  started_at: string | null
+  finished_at: string | null
+}
+
 // ── 评估 & 回测 ─────────────────────────────────────────────────

 export interface EvalCalibrationBucket {