diff --git a/frontend/src/admin/dal.ts b/frontend/src/admin/dal.ts index d4cf05c..f605f2b 100644 --- a/frontend/src/admin/dal.ts +++ b/frontend/src/admin/dal.ts @@ -363,6 +363,19 @@ export function fetchIngestStatus(): Promise<{ sources: IngestSourceStatus[] }> return api.get<{ sources: IngestSourceStatus[] }>(`${API_BASE}/admin/ingest/status`) } +/** + * 上游数据源(bzzoiro)可达性探针:轻量 GET,不携带 Key、不消耗配额 + */ +export function fetchUpstreamProbe(): Promise<{ + ok: boolean + status_code?: number + latency_ms: number + endpoint: string + error?: string +}> { + return api.get(`${API_BASE}/admin/monitoring/upstream`) +} + /** * 采集任务状态轮询(单任务) */ diff --git a/frontend/src/admin/pages/Monitoring.tsx b/frontend/src/admin/pages/Monitoring.tsx index 4e428e6..82047c4 100644 --- a/frontend/src/admin/pages/Monitoring.tsx +++ b/frontend/src/admin/pages/Monitoring.tsx @@ -1,20 +1,49 @@ /** - * Admin 后台 - 监控面板(报刊风) + * Admin 后台 - 监控页(报刊风·三分区) * - * 功能: - * - /health 存活检查(自动:30 秒一轮;可手动刷新) - * - /health/ready 数据库就绪检查 - * - 服务名 / 版本 / 运行时间 / 检查项(后端返回什么就展示什么) + * 此前只有存活/DB 两个布尔,与顶栏健康点重复,信息丰度不足。 + * 现在聚合全站已有信号,回答三个问题: + * 1. 基础设施活着吗 —— 服务存活 / DB 就绪 / 版本 / 运行时间 / 上游 bzzoiro 可达性 + * 2. 数据管线健康吗 —— 最近采集成功率 / 死信 / 数据缺口 + * 3. LLM 服务正常吗 —— 预测成功率 / 平均延迟 + * 顶部「需要关注」聚合条:任何一项异常即亮红并直达处理页。 + * 30s 自动巡检 + 手动巡检。 */ -import { useCallback, useEffect, useState } from 'react' -import { fetchHealth } from '../dal' +import { useCallback, useEffect, useMemo, useState } from 'react' +import { Link } from 'react-router-dom' +import { fetchHealth, fetchUpstreamProbe, fetchIngestJobs, fetchIngestFailures, fetchDataCompleteness, fetchLLMUsageStats } from '../dal' +import type { DataCompletenessResponse, IngestFailureItem } from '../dal' +import type { IngestJob } from '../types' import { api } from '../api' -import { Card, CardBody, CardHeader, SectionHeader, Alert, Spinner, Badge } from '../components' +import { Alert, SectionHeader, Spinner } from '../components' + +type UpstreamProbe = { ok: boolean; status_code?: number; latency_ms: number; endpoint: string; error?: string } +type LLMStats = { total_predictions: number; avg_latency_ms: number; success_rate: number } + +interface TodoItem { + key: string + label: string + to: string +} + +function MetricCard({ label, children }: { label: string; children: React.ReactNode }) { + return ( +
+
{label}
+
{children}
+
+ ) +} export default function MonitoringPage() { - const [health, setHealth] = useState(null) + const [health, setHealth] = useState | null>(null) const [ready, setReady] = useState<'ready' | 'not_ready' | null>(null) + const [upstream, setUpstream] = useState(null) + const [recentJobs, setRecentJobs] = useState(null) + const [failures, setFailures] = useState([]) + const [completeness, setCompleteness] = useState(null) + const [llmStats, setLlmStats] = useState(null) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) const [lastCheck, setLastCheck] = useState('') @@ -22,20 +51,28 @@ export default function MonitoringPage() { const refresh = useCallback(async () => { setLoading(true) setError(null) - try { - const [h, r] = await Promise.allSettled([ - fetchHealth(), - api.get<{ status: string }>('/health/ready'), - ]) - setHealth(h.status === 'fulfilled' ? h.value : null) - setReady(r.status === 'fulfilled' ? (r.value?.status as 'ready' | 'not_ready') : null) - if (h.status === 'rejected') { - setError(h.reason instanceof Error ? h.reason.message : '无法连接到后端') - } - setLastCheck(new Date().toLocaleTimeString('zh-CN', { hour12: false })) - } finally { - setLoading(false) + // 各信号独立容错:单项失败只降级对应卡片 + const [h, r, u, j, f, c, l] = await Promise.allSettled([ + fetchHealth(), + api.get<{ status: string }>('/health/ready'), + fetchUpstreamProbe(), + fetchIngestJobs({ limit: 10 }), + fetchIngestFailures(), + fetchDataCompleteness(), + fetchLLMUsageStats() as Promise, + ]) + setHealth(h.status === 'fulfilled' ? h.value : null) + setReady(r.status === 'fulfilled' ? (r.value?.status as 'ready' | 'not_ready') : null) + setUpstream(u.status === 'fulfilled' ? u.value : null) + setRecentJobs(j.status === 'fulfilled' && Array.isArray(j.value) ? j.value : null) + setFailures(f.status === 'fulfilled' ? f.value : []) + setCompleteness(c.status === 'fulfilled' ? c.value : null) + setLlmStats(l.status === 'fulfilled' ? l.value : null) + if (h.status === 'rejected') { + setError(h.reason instanceof Error ? h.reason.message : '无法连接到后端') } + setLastCheck(new Date().toLocaleTimeString('zh-CN', { hour12: false })) + setLoading(false) }, []) useEffect(() => { @@ -45,108 +82,188 @@ export default function MonitoringPage() { }, [refresh]) const alive = health?.status === 'healthy' || health?.status === 'ok' + const deadLetterCount = failures.filter(f => f.status !== 'resolved').length + const missingStatsLeagues = completeness?.leagues.filter( + l => l.matches.finished > 0 && l.stats.rows === 0, + ).length ?? 0 + const recentFailedJobs = useMemo( + () => (recentJobs ?? []).filter(j => j.status === 'failed').length, + [recentJobs], + ) + + // ── 「需要关注」聚合:任何一项异常即亮红 ── + const todos: TodoItem[] = [ + !alive && { key: 'alive', label: '服务存活异常', to: '/admin/logs' }, + ready === 'not_ready' && { key: 'db', label: '数据库未就绪', to: '/admin/logs' }, + upstream && !upstream.ok && { key: 'upstream', label: '上游 bzzoiro 不可达', to: '/admin/logs' }, + recentFailedJobs > 0 && { key: 'jobs', label: `最近采集失败 ${recentFailedJobs} 次`, to: '/admin/collection' }, + deadLetterCount > 0 && { key: 'deadletter', label: `死信待处理 ${deadLetterCount} 条`, to: '/admin/data-pipeline' }, + missingStatsLeagues > 0 && { key: 'missing', label: `${missingStatsLeagues} 个联赛缺统计`, to: '/admin/data-completeness' }, + ].filter((t): t is TodoItem => t !== false) + + const okJobs = (recentJobs ?? []).filter(j => j.status === 'success').length + const runJobs = (recentJobs ?? []).filter(j => j.status === 'success' || j.status === 'failed').length + const lastJobTime = recentJobs?.[0]?.created_at return (
+ {lastCheck && `最近巡检 ${lastCheck}`} + +
+ } /> -
- - {lastCheck && `最近巡检 ${lastCheck}`} - - -
- {error && ( - + )} -
- {/* 存活状态 */} -
-
存活状态
-
-
-
- - {/* 数据库就绪 */} -
-
数据库就绪
-
-
+ ) : ( + !loading && ( +

+

+ ) + )} - {/* 服务名 */} -
-
服务
-
- {health?.service || 'profeto'} -
-
- - {/* 版本 */} - {health?.version && ( -
-
版本
-
- {health.version} + {/* ── 基础设施 ── */} +
+

基础设施

+
+ +
+
-
- )} + - {/* 运行时间 */} - {health?.uptime_seconds != null && ( -
-
运行时间
-
- {Math.floor(health.uptime_seconds / 3600)}h{' '} - {Math.floor((health.uptime_seconds % 3600) / 60)}m + +
+
-
- )} + - {/* 检查项 */} - {health?.checks && Object.keys(health.checks).length > 0 && ( -
-
健康检查
-
- {Object.entries(health.checks).map(([key, val]) => ( -
- {key} - - {String(val)} - + +
+ {health?.version ? String(health.version) : '—'} +
+
+ {health?.uptime_seconds != null + ? `已运行 ${Math.floor(Number(health.uptime_seconds) / 3600)}h ${Math.floor((Number(health.uptime_seconds) % 3600) / 60)}m` + : '—'} +
+ + + + {upstream ? ( +
+
+
- ))} +
+ {upstream.ok + ? `HTTP ${upstream.status_code} · ${upstream.latency_ms}ms` + : upstream.error?.slice(0, 40) || `HTTP ${upstream.status_code}`} +
+
+ ) : ( + — + )} + +
+
+ + {/* ── 数据管线 ── */} +
+

数据管线

+
+ + {recentJobs && runJobs > 0 ? ( +
+ {okJobs}/{runJobs} 成功 +
+ ) : ( +
—
+ )} +
+ {lastJobTime ? `最近 ${new Date(lastJobTime).toLocaleString('zh-CN', { month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit', hour12: false })}` : '暂无记录'}
-
- )} -
+ + + +
0 ? 'text-press' : 'text-ink-900'}`}> + {deadLetterCount} +
+
失败记录,可重试
+ + + +
0 ? 'text-press' : 'text-ink-900'}`}> + {missingStatsLeagues} +
+
联赛有完赛缺统计
+ + + +
+ {completeness ? `${completeness.totals.stats_coverage_pct}%` : '—'} +
+
有统计 / 已完赛
+ +
+ + + {/* ── LLM ── */} +
+

LLM 服务

+
+ +
+ {llmStats ? llmStats.total_predictions : '—'} +
+ + +
+ {llmStats && llmStats.avg_latency_ms > 0 ? `${(llmStats.avg_latency_ms / 1000).toFixed(1)}s` : '—'} +
+ + +
+ {llmStats ? `${llmStats.success_rate.toFixed(0)}%` : '—'} +
+ +
+
) } diff --git a/src/api/app.py b/src/api/app.py index 84e2b44..91f6b47 100644 --- a/src/api/app.py +++ b/src/api/app.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import os +import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -13,6 +14,9 @@ from src.core.config import settings logger = logging.getLogger(__name__) +# 进程启动时间(monotonic,不受系统时钟跳变影响):/health 的 uptime 来源 +_PROCESS_STARTED_MONOTONIC = time.monotonic() + async def _fail_stale_ingest_jobs() -> None: """P1-E: 启动时将上次遗留的 pending/running ingest_jobs 标 failed。 @@ -156,6 +160,7 @@ def create_app() -> FastAPI: from src.api.routes.auth import router as auth_router from src.api.routes.admin_settings import router as admin_settings_router from src.api.routes.schedules import router as schedules_router + from src.api.routes.admin_monitoring import router as admin_monitoring_router app.include_router(matches_router) app.include_router(predict_router) @@ -165,10 +170,25 @@ def create_app() -> FastAPI: app.include_router(auth_router) app.include_router(admin_settings_router) app.include_router(schedules_router) + app.include_router(admin_monitoring_router) @app.get("/health") async def health(): - return {"status": "healthy", "service": "profeto"} + """存活检查。version/uptime_seconds 供管理端监控页展示。""" + version = None + try: + from importlib.metadata import version as _pkg_version + + version = _pkg_version("profeto") + except Exception: + # 包元数据缺失时返回 None,前端降级隐藏版本卡片 + version = None + return { + "status": "healthy", + "service": "profeto", + "version": version, + "uptime_seconds": round(time.monotonic() - _PROCESS_STARTED_MONOTONIC), + } @app.get("/health/ready") diff --git a/src/api/routes/admin_monitoring.py b/src/api/routes/admin_monitoring.py new file mode 100644 index 0000000..1f587be --- /dev/null +++ b/src/api/routes/admin_monitoring.py @@ -0,0 +1,44 @@ +"""后台管理:监控增强探针(上游可达性)。 + +所有接口需管理员鉴权(require_admin)。路由前缀 /api/v1/admin。 +""" +from __future__ import annotations + +import logging +import time + +import httpx +from fastapi import APIRouter, Depends + +from src.api.deps import require_admin +from src.core.config import settings +from src.core.runtime_config import get_runtime_value + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)]) + + +@router.get("/monitoring/upstream") +async def upstream_probe(): + """上游数据源(bzzoiro)可达性探针。 + + 轻量 GET 根端点:不携带 API Key、不触发采集,不消耗配额。 + HTTP <500 视为可达 —— 404/401 也说明 DNS/网络/TLS 正常,业务语义层的 + 失败(签名、限流)由采集管线自身的死信与日志上报,不在探针职责内。 + """ + base = ((await get_runtime_value("BZZOIRO_BASE")) or settings.BZZOIRO_BASE).rstrip("/") + started = time.perf_counter() + try: + async with httpx.AsyncClient( + timeout=httpx.Timeout(connect=5.0, read=5.0, write=5.0, pool=5.0), + ) as client: + resp = await client.get(base) + latency_ms = round((time.perf_counter() - started) * 1000) + ok = resp.status_code < 500 + logger.info("上游探针 %s → %s (%sms)", base, resp.status_code, latency_ms) + return {"ok": ok, "status_code": resp.status_code, "latency_ms": latency_ms, "endpoint": base} + except Exception as e: + latency_ms = round((time.perf_counter() - started) * 1000) + logger.warning("上游探针失败 %s: %s", base, e) + return {"ok": False, "error": str(e)[:200], "latency_ms": latency_ms, "endpoint": base} diff --git a/tests/test_monitoring_endpoints.py b/tests/test_monitoring_endpoints.py new file mode 100644 index 0000000..2519d72 --- /dev/null +++ b/tests/test_monitoring_endpoints.py @@ -0,0 +1,114 @@ +"""监控增强端点测试:/health 扩展字段 + 上游探针。 + +沿用 test_api_critical.py 的模式:直接调用 handler,不启动完整 app +lifespan(异步 DB 引擎与同步 TestClient 不兼容);探针的外呼用 mock。 +""" +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + + + + +class TestHealthExtended: + async def _get_health(self): + """ASGITransport 走真实路由:/health 无鉴权、不依赖 DB,不触发 lifespan。""" + from httpx import ASGITransport, AsyncClient + + from src.api.app import app + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + return await client.get("/health") + + @pytest.mark.asyncio + async def test_reports_version_and_uptime(self): + resp = await self._get_health() + + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "healthy" + assert isinstance(body["uptime_seconds"], int) + assert body["uptime_seconds"] >= 0 + # 包已随 pip install . 安装,元数据可读;异常时为 None(前端降级隐藏) + assert body["version"] is None or isinstance(body["version"], str) + + @pytest.mark.asyncio + async def test_metadata_failure_degrades_to_none(self): + """包元数据不可读时 version=None,不影响存活判定。""" + from importlib.metadata import PackageNotFoundError + + with patch("importlib.metadata.version", side_effect=PackageNotFoundError("profeto")): + resp = await self._get_health() + + assert resp.status_code == 200 + body = resp.json() + assert body["status"] == "healthy" + assert body["version"] is None + assert isinstance(body["uptime_seconds"], int) + + +class TestUpstreamProbe: + async def _probe_with(self, mock_client_factory): + from src.api.routes import admin_monitoring + + with patch.object(admin_monitoring.httpx, "AsyncClient", mock_client_factory), \ + patch.object( + admin_monitoring, "get_runtime_value", + AsyncMock(return_value="https://sports.bzzoiro.com/api/v2"), + ): + return await admin_monitoring.upstream_probe() + + @pytest.mark.asyncio + async def test_reachable_upstream(self): + """HTTP 200 → ok=True 且带延迟与状态码。""" + resp = MagicMock() + resp.status_code = 200 + client = MagicMock() + client.get = AsyncMock(return_value=resp) + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=False) + factory = MagicMock(return_value=client) + + out = await self._probe_with(factory) + + assert out["ok"] is True + assert out["status_code"] == 200 + assert out["latency_ms"] >= 0 + assert out["endpoint"].endswith("/api/v2") + client.get.assert_awaited_once_with("https://sports.bzzoiro.com/api/v2") + + @pytest.mark.asyncio + async def test_5xx_counts_as_unreachable(self): + resp = MagicMock() + resp.status_code = 502 + client = MagicMock() + client.get = AsyncMock(return_value=resp) + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=False) + factory = MagicMock(return_value=client) + + out = await self._probe_with(factory) + + assert out["ok"] is False + assert out["status_code"] == 502 + + @pytest.mark.asyncio + async def test_network_error_degrades_gracefully(self): + """连接失败不抛 500:返回 ok=False + error 摘要(探针失败不是故障)。""" + + def factory(*_a, **_kw): + client = MagicMock() + client.get = AsyncMock(side_effect=httpx.ConnectError("connection refused")) + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=False) + return client + + out = await self._probe_with(factory) + + assert out["ok"] is False + assert "connection refused" in out["error"] + assert out["latency_ms"] >= 0