feat: 采集任务状态跟踪 ingest_jobs

新增 ingest_jobs 表(UUID/task/params/status/result/error/时间戳),
POST /ingest/bzzoiro 启动前插入 job(pending)→ 后台 running → success/failed,
响应新增 job_id(兼容原 message)。

Admin GET /admin/ingest/jobs/{id} 与 /admin/ingest/jobs?limit= 只读查询;
Collection 页提交后轮询 job 至终态,展示真实 result/error 汇总。
迁移 0019_ingest_jobs + 分批 get_uow/BzzoiroSource/IngestFailure 不变。

全量测试 270 通过。
This commit is contained in:
shangfangjian
2026-09-22 01:34:55 +08:00
parent 44816794d3
commit 92f50fa5d4
6 changed files with 286 additions and 61 deletions
+42
View File
@@ -0,0 +1,42 @@
"""采集任务状态表 ingest_jobs
Revision ID: 0019_ingest_jobs
Revises: 0018_match_checks
Create Date: 2026-09-22
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = '0019_ingest_jobs'
down_revision: Union[str, None] = '0018_match_checks'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_table(
'ingest_jobs',
sa.Column('id', sa.String(36), primary_key=True),
sa.Column('task', sa.String(20), nullable=False),
sa.Column('params', sa.JSON(), nullable=False, server_default='{}'),
sa.Column('status', sa.String(20), nullable=False, server_default='pending'),
sa.Column('result', sa.JSON(), nullable=True),
sa.Column('error', sa.Text(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now()),
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
)
op.create_index('ix_ingest_job_status_created', 'ingest_jobs', ['status', 'created_at'])
op.create_check_constraint(
'ck_ingest_job_status', 'ingest_jobs',
"status IN ('pending', 'running', 'success', 'failed')",
)
def downgrade() -> None:
op.drop_constraint('ck_ingest_job_status', 'ingest_jobs', type_='check')
op.drop_index('ix_ingest_job_status_created', table_name='ingest_jobs')
op.drop_table('ingest_jobs')
+15
View File
@@ -21,6 +21,7 @@ import type {
LLMAgentConfig, LLMAgentConfig,
LogEntry, LogEntry,
IngestSourceStatus, IngestSourceStatus,
IngestJob,
MatchDetailOut, MatchDetailOut,
MatchContextOut, MatchContextOut,
AdminStats, AdminStats,
@@ -362,6 +363,20 @@ export function fetchIngestStatus(): Promise<{ sources: IngestSourceStatus[] }>
return api.get<{ sources: IngestSourceStatus[] }>(`${API_BASE}/admin/ingest/status`) return api.get<{ sources: IngestSourceStatus[] }>(`${API_BASE}/admin/ingest/status`)
} }
/**
* 采集任务状态轮询(单任务)
*/
export function fetchIngestJob(jobId: string): Promise<IngestJob> {
return api.get<IngestJob>(`${API_BASE}/admin/ingest/jobs/${jobId}`)
}
/**
* 最近采集任务列表(最新在前)
*/
export function fetchIngestJobs(limit = 20): Promise<IngestJob[]> {
return api.get<IngestJob[]>(`${API_BASE}/admin/ingest/jobs?limit=${limit}`)
}
/** /**
* 比赛详情(含最近预测摘要) * 比赛详情(含最近预测摘要)
*/ */
+87 -51
View File
@@ -11,9 +11,9 @@
*/ */
import { useEffect, useState, useCallback, useRef } from 'react' import { useEffect, useState, useCallback, useRef } from 'react'
import { triggerCollection, fetchLeagues, fetchIngestStatus } from '../dal' import { triggerCollection, fetchLeagues, fetchIngestJob } from '../dal'
import type { IngestSourceStatus } from '../types' import type { IngestJob, League } from '../types'
import type { CollectionRequest, League } from '../types' import type { CollectionRequest } from '../types'
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components' import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
const TASKS = [ const TASKS = [
@@ -23,7 +23,9 @@ const TASKS = [
{ value: 'all', label: '全量采集', desc: '依次采集比赛 + 积分榜 + 统计回填', icon: '⏵⏵' }, { value: 'all', label: '全量采集', desc: '依次采集比赛 + 积分榜 + 统计回填', icon: '⏵⏵' },
] as const ] as const
type TaskStatus = 'idle' | 'running' | 'done' | 'error' type TaskUIStatus = 'idle' | 'running' | 'done' | 'error'
const TERMINAL_STATUSES: ReadonlySet<string> = new Set(['success', 'failed'])
export default function CollectionPage() { export default function CollectionPage() {
const [leagues, setLeagues] = useState<League[]>([]) const [leagues, setLeagues] = useState<League[]>([])
@@ -49,13 +51,13 @@ export default function CollectionPage() {
const [limit, setLimit] = useState(100) const [limit, setLimit] = useState(100)
const [loading, setLoading] = useState(false) const [loading, setLoading] = useState(false)
const [error, setError] = useState<string | null>(null) const [error, setError] = useState<string | null>(null)
const [result, setResult] = useState<{ title: string; detail: string } | null>(null)
// 任务进度反馈 // 任务进度反馈:跟踪真实 ingest_job 状态
const [taskStatus, setTaskStatus] = useState<TaskStatus>('idle') const [taskStatus, setTaskStatus] = useState<TaskUIStatus>('idle')
const [jobId, setJobId] = useState<string | null>(null)
const [jobInfo, setJobInfo] = useState<IngestJob | null>(null)
const [taskStartedAt, setTaskStartedAt] = useState<number | null>(null) const [taskStartedAt, setTaskStartedAt] = useState<number | null>(null)
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null) const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
const [ingestSnap, setIngestSnap] = useState<IngestSourceStatus | null>(null)
const loadLeagues = useCallback(async () => { const loadLeagues = useCallback(async () => {
const lg = await fetchLeagues() const lg = await fetchLeagues()
@@ -64,30 +66,65 @@ export default function CollectionPage() {
useEffect(() => { loadLeagues() }, [loadLeagues]) useEffect(() => { loadLeagues() }, [loadLeagues])
// 轮询采集状态(任务启动后) // 轮询采集 job 直到终态(success/failed)
const startPolling = useCallback(() => {
if (pollRef.current) clearInterval(pollRef.current)
pollRef.current = setInterval(async () => {
try {
const { sources } = await fetchIngestStatus()
const bz = sources.find(s => s.name === 'bzzoiro')
if (bz) setIngestSnap(bz)
} catch { /* ignore */ }
}, 5_000)
}, [])
const stopPolling = useCallback(() => { const stopPolling = useCallback(() => {
if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null } if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null }
}, []) }, [])
useEffect(() => () => stopPolling(), [stopPolling]) useEffect(() => () => stopPolling(), [stopPolling])
const startJobPolling = useCallback((id: string) => {
stopPolling()
const tick = async () => {
try {
const job = await fetchIngestJob(id)
setJobInfo(job)
if (TERMINAL_STATUSES.has(job.status)) {
setTaskStatus(job.status === 'success' ? 'done' : 'error')
stopPolling()
}
} catch { /* 单次轮询失败不影响后续 */ }
}
tick()
pollRef.current = setInterval(tick, 3_000)
}, [stopPolling])
const isEventsTask = task === 'events' || task === 'all' const isEventsTask = task === 'events' || task === 'all'
// 友好汇总 job.result
const jobSummary = (j: IngestJob | null): { title: string; detail: string } | null => {
if (!j) return null
if (j.status === 'failed') {
return { title: '采集失败', detail: j.error || '采集任务异常终止,请到「系统日志」查看详细堆栈。' }
}
if (j.status !== 'success') return null
const r = j.result as Record<string, unknown> | null
if (!r) return { title: '采集完成', detail: '任务成功(无汇总数据)。' }
const ev = r.events as Record<string, unknown> | undefined
const evTotal = ev ? (ev.total_inserted as number ?? 0) + (ev.total_updated as number ?? 0) : 0
const st = r.standings as Record<string, unknown> | undefined
const stTotal = st ? (st.total_upserted as number ?? 0) : 0
const stats = r.stats as Record<string, unknown> | undefined
const statsTotal = stats ? (stats.created as number ?? 0) + (stats.updated as number ?? 0) : 0
const evErr = (ev?.errors as string[] | undefined)?.length ?? 0
const stErr = (st?.errors as string[] | undefined)?.length ?? 0
const statsErr = (stats?.errors as string[] | undefined)?.length ?? 0
const totalErr = evErr + stErr + statsErr
const parts: string[] = []
if (ev) parts.push(`比赛 +${evTotal}`)
if (st) parts.push(`积分榜 +${stTotal}`)
if (stats) parts.push(`统计 +${statsTotal}`)
const detail = parts.length
? `共更新: ${parts.join(' / ')}${totalErr ? `,错误 ${totalErr} 条(见日志)` : ''}`
: '任务成功'
return { title: '采集完成', detail }
}
async function handleSubmit(e: React.FormEvent) { async function handleSubmit(e: React.FormEvent) {
e.preventDefault() e.preventDefault()
setError(null) setError(null)
setResult(null) setJobInfo(null)
setJobId(null)
setLoading(true) setLoading(true)
setTaskStatus('running') setTaskStatus('running')
setTaskStartedAt(Date.now()) setTaskStartedAt(Date.now())
@@ -103,18 +140,15 @@ export default function CollectionPage() {
date_from: isEventsTask ? dateFrom || undefined : undefined, date_from: isEventsTask ? dateFrom || undefined : undefined,
date_to: isEventsTask ? dateTo || undefined : undefined, date_to: isEventsTask ? dateTo || undefined : undefined,
} }
await triggerCollection(body) const res = await triggerCollection(body)
setResult({ const id: string | undefined = res?.job_id
title: '采集任务已启动', if (id) {
detail: '正在后台执行(上游限速时可能需要数分钟)。完成结果与错误请到「系统日志」页查看(支持自动刷新)。', setJobId(id)
}) startJobPolling(id)
// 启动轮询,跟踪状态 } else {
startPolling() // 后端未返回 job_id(旧版兼容):退化为原逻辑
// 30 秒后自动停止轮询并标记完成 setTimeout(() => { setTaskStatus('done'); }, 30_000)
setTimeout(() => { }
setTaskStatus('done')
stopPolling()
}, 30_000)
} catch (err: unknown) { } catch (err: unknown) {
setTaskStatus('error') setTaskStatus('error')
setError(err instanceof Error ? err.message : '采集触发失败') setError(err instanceof Error ? err.message : '采集触发失败')
@@ -125,6 +159,7 @@ export default function CollectionPage() {
} }
const elapsed = taskStartedAt ? Math.round((Date.now() - taskStartedAt) / 1000) : 0 const elapsed = taskStartedAt ? Math.round((Date.now() - taskStartedAt) / 1000) : 0
const summary = jobInfo ? jobSummary(jobInfo) : null
return ( return (
<div className="space-y-6"> <div className="space-y-6">
@@ -248,12 +283,11 @@ export default function CollectionPage() {
{/* 消息提示 */} {/* 消息提示 */}
{error && <Alert kind="error" title="采集失败" message={error} onClose={() => setError(null)} />} {error && <Alert kind="error" title="采集失败" message={error} onClose={() => setError(null)} />}
{result && ( {summary && (
<Alert <Alert
kind="ok" kind={jobInfo?.status === 'success' ? 'ok' : 'error'}
title={result.title} title={summary.title}
message={result.detail || undefined} message={summary.detail}
onClose={() => setResult(null)}
/> />
)} )}
@@ -278,10 +312,10 @@ export default function CollectionPage() {
<div className="space-y-3"> <div className="space-y-3">
<div className="flex items-center gap-2 text-xs text-ink-700"> <div className="flex items-center gap-2 text-xs text-ink-700">
<Spinner /> <Spinner />
<span>, {elapsed}s</span> <span>{jobId ? `(job ${jobId.slice(0, 8)}…)` : ''}, {elapsed}s</span>
</div> </div>
<p className="text-2xs text-ink-400"> <p className="text-2xs text-ink-400">
, , 3
</p> </p>
</div> </div>
)} )}
@@ -289,23 +323,25 @@ export default function CollectionPage() {
<div className="space-y-2"> <div className="space-y-2">
<div className="flex items-center gap-2 text-xs text-emerald-700"> <div className="flex items-center gap-2 text-xs text-emerald-700">
<span className="inline-block h-2 w-2 rounded-full bg-emerald-500" /> <span className="inline-block h-2 w-2 rounded-full bg-emerald-500" />
<span>,()</span> <span>{jobId ? `(job ${jobId.slice(0, 8)}…)` : ''}</span>
</div> </div>
<p className="text-2xs text-ink-400"> {summary && <p className="text-2xs text-ink-500">{summary.detail}</p>}
</p>
</div> </div>
)} )}
{taskStatus === 'error' && ( {taskStatus === 'error' && (
<p className="text-xs text-press">,</p> <div className="space-y-1">
)} <p className="text-xs text-press">{jobId ? `(job ${jobId.slice(0, 8)}…)` : ''}</p>
{ingestSnap?.last_success_at && ( {jobInfo?.error && (
<div className="mt-3 border-t border-ink-100 pt-3"> <p className="text-2xs text-ink-500">{jobInfo.error.slice(0, 200)}</p>
<p className="text-2xs text-ink-400"> )}
bzzoiro : {new Date(ingestSnap.last_success_at).toLocaleString('zh-CN', { hour12: false })}
</p>
</div> </div>
)} )}
{jobInfo?.created_at && (
<p className="mt-2 text-2xs text-ink-400">
{new Date(jobInfo.created_at).toLocaleString('zh-CN', { hour12: false })}
{jobInfo.finished_at && ` · 完成于 ${new Date(jobInfo.finished_at).toLocaleString('zh-CN', { hour12: false })}`}
</p>
)}
</CardBody> </CardBody>
</Card> </Card>
+14
View File
@@ -283,6 +283,20 @@ export interface IngestSourceStatus {
last_failure: IngestLastFailure | null last_failure: IngestLastFailure | null
} }
// ── 采集任务状态 ──────────────────────────────────────────────
export interface IngestJob {
id: string
task: string
params: Record<string, unknown>
status: 'pending' | 'running' | 'success' | 'failed'
result: Record<string, unknown> | null
error: string | null
created_at: string | null
started_at: string | null
finished_at: string | null
}
// ── 比赛详情 ───────────────────────────────────────────────────── // ── 比赛详情 ─────────────────────────────────────────────────────
export interface MatchRecentPrediction { export interface MatchRecentPrediction {
+55
View File
@@ -0,0 +1,55 @@
"""后台管理:采集任务状态查询(只读)。
GET /api/v1/admin/ingest/jobs/{job_id} — 单任务详情
GET /api/v1/admin/ingest/jobs?limit=N — 最近任务列表(默认 20)
"""
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import desc, select
from src.api.deps import require_admin
from src.api.schemas import IngestJobOut
from src.db.base import AsyncSession, get_db_read
from src.db.models import IngestJob
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
@router.get("/ingest/jobs/{job_id}", response_model=IngestJobOut)
async def get_ingest_job(job_id: str, db: AsyncSession = Depends(get_db_read)):
"""查询单个采集任务状态。"""
job = await db.get(IngestJob, job_id)
if job is None:
raise HTTPException(404, f"采集任务不存在: {job_id}")
return _job_to_out(job)
@router.get("/ingest/jobs", response_model=list[IngestJobOut])
async def list_ingest_jobs(
limit: int = Query(20, ge=1, le=100, description="返回条数"),
db: AsyncSession = Depends(get_db_read),
):
"""查询最近采集任务(最新在前)。"""
rows = (
await db.execute(select(IngestJob).order_by(desc(IngestJob.created_at)).limit(limit))
).scalars().all()
return [_job_to_out(j) for j in rows]
def _job_to_out(job: IngestJob) -> IngestJobOut:
return IngestJobOut(
id=job.id,
task=job.task,
params=job.params or {},
status=job.status,
result=job.result,
error=job.error,
created_at=job.created_at,
started_at=job.started_at,
finished_at=job.finished_at,
)
+73 -10
View File
@@ -10,11 +10,13 @@ from __future__ import annotations
import asyncio import asyncio
import logging import logging
import uuid
from datetime import datetime, timezone
from fastapi import APIRouter, Depends, HTTPException from fastapi import APIRouter, Depends, HTTPException
from src.api.deps import require_admin from src.api.deps import require_admin
from src.api.schemas import IngestBzzoiroRequest from src.api.schemas import IngestBzzoiroRequest, IngestBzzoiroResponse
from src.data.config import BZZOIRO_LEAGUE_IDS from src.data.config import BZZOIRO_LEAGUE_IDS
from src.data.bzzoiro_standings import ingest_bzzoiro_standings from src.data.bzzoiro_standings import ingest_bzzoiro_standings
from src.data.bzzoiro_stats import ingest_bzzoiro_event_stats from src.data.bzzoiro_stats import ingest_bzzoiro_event_stats
@@ -38,22 +40,58 @@ def _spawn(coro) -> None:
task.add_done_callback(_background_tasks.discard) task.add_done_callback(_background_tasks.discard)
@router.post("/ingest/bzzoiro", dependencies=[Depends(require_admin)]) @router.post("/ingest/bzzoiro", response_model=IngestBzzoiroResponse, dependencies=[Depends(require_admin)])
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest): async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
"""触发 bzzoiro 采集(events / standings / stats / all)。""" """触发 bzzoiro 采集(events / standings / stats / all)。
启动后台任务前写入 ingest_jobs(pending),响应返回 job_id 供前端轮询。
兼容原 message 字段(仍返回)。
"""
if req.task not in VALID_TASKS: if req.task not in VALID_TASKS:
raise HTTPException(status_code=422, detail=f"未知任务类型: {req.task}(可选: {', '.join(sorted(VALID_TASKS))})") raise HTTPException(status_code=422, detail=f"未知任务类型: {req.task}(可选: {', '.join(sorted(VALID_TASKS))})")
leagues = req.leagues or list(BZZOIRO_LEAGUE_IDS.keys()) leagues = req.leagues or list(BZZOIRO_LEAGUE_IDS.keys())
task_label = {"events": "比赛数据", "standings": "积分榜", "stats": "统计回填", "all": "全量(比赛+积分榜+统计)"}[req.task] task_label = {"events": "比赛数据", "standings": "积分榜", "stats": "统计回填", "all": "全量(比赛+积分榜+统计)"}[req.task]
_spawn(_run_bzzoiro(req.task, leagues, req))
return { job_id = await _create_ingest_job(req.task, leagues, req)
"ok": True, _spawn(_run_bzzoiro(job_id, req.task, leagues, req))
"message": f"采集任务已启动(后台执行,任务: {task_label}),请在「系统日志」查看进度与结果",
return IngestBzzoiroResponse(
ok=True,
job_id=job_id,
message=f"采集任务已启动(后台执行,任务: {task_label}),请到「数据采集」页跟踪进度",
)
async def _create_ingest_job(task: str, leagues: list[str], req: IngestBzzoiroRequest) -> str:
"""写入一条 ingest_jobs(pending),返回 job_id。"""
from src.db.models import IngestJob
job_id = str(uuid.uuid4())
params = {
"leagues": leagues,
"date_from": req.date_from,
"date_to": req.date_to,
"status": req.status,
"task": task,
"limit": req.limit,
"season": req.season,
} }
async with get_uow() as session:
job = IngestJob(id=job_id, task=task, params=params, status="pending")
session.add(job)
logger.info("ingest_jobs 创建: job=%s task=%s leagues=%s", job_id, task, leagues)
return job_id
async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None: async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None:
"""后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。""" """后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。
状态流转: pending → running → (success|failed)。
"""
from src.db.models import IngestJob
await _update_job(job_id, status="running", started_at=datetime.now(timezone.utc))
result: dict = {}
try: try:
if task in ("events", "all"): if task in ("events", "all"):
statuses = [req.status] if req.status else ["finished", "scheduled"] statuses = [req.status] if req.status else ["finished", "scheduled"]
@@ -80,6 +118,7 @@ async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest)
) )
if merged["errors"]: if merged["errors"]:
logger.warning("bzzoiro 比赛采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3]) logger.warning("bzzoiro 比赛采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3])
result["events"] = merged
if task in ("standings", "all"): if task in ("standings", "all"):
async with get_uow() as session: async with get_uow() as session:
@@ -88,6 +127,7 @@ async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest)
logger.warning("bzzoiro 积分榜采集部分失败: %s", r["errors"][:3]) logger.warning("bzzoiro 积分榜采集部分失败: %s", r["errors"][:3])
else: else:
logger.info("bzzoiro 积分榜采集完成: upsert %d", r["total_upserted"]) logger.info("bzzoiro 积分榜采集完成: upsert %d", r["total_upserted"])
result["standings"] = r
if task in ("stats", "all"): if task in ("stats", "all"):
async with get_uow() as session: async with get_uow() as session:
@@ -96,5 +136,28 @@ async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest)
) )
if r["errors"]: if r["errors"]:
logger.warning("bzzoiro 统计回填错误 %d 条: %s", len(r["errors"]), r["errors"][:3]) logger.warning("bzzoiro 统计回填错误 %d 条: %s", len(r["errors"]), r["errors"][:3])
except Exception: result["stats"] = r
await _update_job(job_id, status="success", result=result, finished_at=datetime.now(timezone.utc))
logger.info("ingest_jobs 完成: job=%s task=%s", job_id, task)
except Exception as e:
logger.exception("bzzoiro 采集任务失败(task=%s)", task) logger.exception("bzzoiro 采集任务失败(task=%s)", task)
await _update_job(
job_id, status="failed", error=str(e), finished_at=datetime.now(timezone.utc),
)
async def _update_job(job_id: str, **fields) -> None:
"""更新 ingest_jobs 单行;失败仅记日志,绝不抛异常(避免干扰采集主流程)。"""
from src.db.models import IngestJob
try:
async with get_uow() as session:
job = await session.get(IngestJob, job_id)
if job is None:
logger.warning("ingest_jobs 更新失败: job=%s 不存在", job_id)
return
for k, v in fields.items():
setattr(job, k, v)
except Exception:
logger.warning("ingest_jobs 更新异常: job=%s fields=%s", job_id, list(fields.keys()))