Author SHA1 Message Date
shangfangjian a0ce2e2a6a feat: 数据管线 P2 优化 — 血缘写入 + 质量监控 + 缓存清理 + 重试 worker
P2-1: DataLineage 血缘写入(bzzoiro/understat Bronze→Silver 全链路)
P2-2: DataQualityCheck 质量检查写入(行计数异常检测)
P2-3: RateLimitedClient 集成到 bzzoiro 采集
P2-4: injuries 缓存过期清理(启动时扫描删除 >1h 文件)
P2-5: IngestFailure 重试 worker(FastAPI startup 注册)

新增文件:
  - src/data/retry_worker.py (死信表重试 sweep)
2026-09-16 15:55:08 +08:00
shangfangjian 5de8ffb09d feat: 数据管线架构优化 — Bronze 层 + 死信队列 + 令牌桶 + 血缘追踪
P0: injuries 缓存原子写(tempfile+os.replace) + TTL 分级 + asyncio.Lock 并发安全
P1: 引入 Bronze 层 RawEvent 表(原始事件存档,不可变)
P1: IngestFailure 死信表(失败持久化,支持重试恢复)
P1: understat 改为 xG 覆盖更新模式 + xG 来源追踪字段
P1: 令牌桶限流器(TokenBucket)替换固定 sleep
P1: DataQualityCheck 数据质量监控表
P1: DataLineage 血缘追踪表
P2: 回测并发控制 Semaphore(3)

新增文件:
  - src/data/rate_limiter.py (令牌桶限流器)
  - alembic/versions/0008 (4 张新表迁移)
  - alembic/versions/0009 (MatchStats xG 字段迁移)
2026-09-16 14:12:42 +08:00
shangfangjian 2fd8a80ad9 revert: 移除手动创建的 experts 目录
Octop 为每个专家 Agent 自带存储后端,无需在项目代码中创建
2026-09-16 13:37:00 +08:00
shangfangjian 5676198392 chore: 添加专家工作区目录结构
为代码审查、数据工程、后端架构、安全工程四个角色
建立专属工作区,用于持久化审查报告、设计文档和知识积累
2026-09-16 13:34:42 +08:00
34 changed files with 1206 additions and 2916 deletions
-43
View File
@@ -1,43 +0,0 @@
{
"permissions": {
"allow": [
"Bash(python -m pytest tests/ -v)",
"Bash(python -c \"import src.core.http_client; import src.llm.provider; import src.data.understat; import src.data.injuries; import src.llm.agents.orchestrator; import src.llm.context_builder; import src.api.app; print\\('All imports OK'\\)\")",
"Bash(python -c \"from src.data.sources import get_source, list_sources; print\\('sources:', list_sources\\(\\)\\); print\\('bzzoiro:', get_source\\('bzzoiro'\\).name\\); print\\('understat:', get_source\\('understat'\\).name\\)\")",
"Bash(python -c \"from src.api.routes.ingest import router; print\\('ingest router OK:', len\\(router.routes\\), 'routes'\\)\")",
"Bash(python -c ' *)",
"Bash(python -m pytest tests/ -q)",
"Bash(git -C /p rev-parse --git-dir)",
"Bash(git config *)",
"Bash(git init *)",
"Bash(git add *)",
"Bash(git commit -m 'feat: 足球 LLM 预测服务初始提交 *)",
"Bash(git remote *)",
"Bash(git push *)",
"Bash(git fetch *)",
"Bash(git branch *)",
"Bash(git commit *)",
"Bash(pg_isready -h localhost -p 5432)",
"Bash(python -c \"import pytest_asyncio; print\\('pytest-asyncio OK'\\)\")",
"Bash(python -c \"import pytest; print\\('pytest', pytest.__version__\\)\")",
"Bash(python -c \"import aiosqlite; print\\('aiosqlite', aiosqlite.__version__\\)\")",
"Bash(npm run *)",
"Bash(npm install *)",
"Bash(git revert *)",
"Bash(python -c \"from src.api.app import app; print\\('OK'\\)\")",
"Bash(python -m pytest tests/test_agents.py::TestReportParsing::test_parse_bad_values_forgiving -v)",
"Bash(python -c \"from src.data.bzzoiro import BzzoiroSource; print\\('OK'\\)\")",
"Bash(python -m pytest tests/test_agents.py::TestReportParsing::test_parse_full_report -v)",
"Bash(python -c \"from src.db.unit_of_work import UnitOfWork, get_uow; from src.db.repositories import MatchRepository, TeamRepository; print\\('OK'\\)\")",
"Bash(python -c \"from src.data.bzzoiro import BzzoiroSource; from src.data.understat import UnderstatSource; from src.data.injuries import ingest_injuries; print\\('OK'\\)\")",
"Bash(python -m compileall src tests)",
"Bash(alembic current *)",
"Bash(alembic history *)",
"Bash(python -c \"from src.db.models import MatchStats, Prediction; print\\('Models OK'\\)\")",
"Bash(git pull *)",
"Bash(ocr delegate *)",
"Bash(npm i *)",
"Bash(python -c \"from src.llm.validation import KNOWN_AGENT_NAMES; print\\(KNOWN_AGENT_NAMES\\)\")"
]
}
}
+1 -19
View File
@@ -3,7 +3,7 @@ from logging.config import fileConfig
import sys import sys
from pathlib import Path from pathlib import Path
from sqlalchemy import engine_from_config, pool, text from sqlalchemy import engine_from_config, pool
from alembic import context from alembic import context
# 把项目根加入 pythonpath,让 alembic 能找到 src 包 # 把项目根加入 pythonpath,让 alembic 能找到 src 包
@@ -29,23 +29,6 @@ if config.config_file_name is not None:
target_metadata = Base.metadata target_metadata = Base.metadata
def _ensure_version_table(connection):
"""确保 alembic_version 表存在且 version_num 列足够长。
Alembic 默认 version_num 是 String(32),但我们的迁移名较长(如
0005_prediction_status_and_stats_provenance 有 41 个字符),会导致
StringDataRightTruncation 错误。
"""
result = connection.execute(text(
"SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'alembic_version')"
))
if not result.scalar():
connection.execute(text(
"CREATE TABLE alembic_version (version_num VARCHAR(255) NOT NULL PRIMARY KEY)"
))
connection.commit()
def run_migrations_offline() -> None: def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.""" """Run migrations in 'offline' mode."""
url = config.get_main_option("sqlalchemy.url") url = config.get_main_option("sqlalchemy.url")
@@ -69,7 +52,6 @@ def run_migrations_online() -> None:
) )
with connectable.connect() as connection: with connectable.connect() as connection:
_ensure_version_table(connection)
context.configure(connection=connection, target_metadata=target_metadata) context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction(): with context.begin_transaction():
@@ -1,9 +1,14 @@
"""创建 Bronze 层、死信表、质量监控、血缘追踪 4 张新 """新增 Bronze 层 + 死信表 + 数据质量表 + 血缘
Revision ID: 0008_raw_event_and_ingest_failure Revision ID: 0008_raw_event_and_ingest_failure
Revises: 0007_predictions_unique_constraint Revises: 0007_predictions_unique_constraint
Create Date: 2026-09-17 Create Date: 2026-09-17
架构审查报告 P1 实施:
- raw_events: Bronze 层,不可变原始采集记录
- ingest_failures: 采集失败死信表
- data_quality_checks: 数据质量检查结果记录
- data_lineage: ETL 全过程元数据血缘追踪
""" """
from typing import Sequence, Union from typing import Sequence, Union
@@ -12,6 +17,7 @@ from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.dialects.postgresql import JSONB
# revision identifiers, used by Alembic.
revision: str = '0008_raw_event_and_ingest_failure' revision: str = '0008_raw_event_and_ingest_failure'
down_revision: Union[str, None] = '0007_predictions_unique_constraint' down_revision: Union[str, None] = '0007_predictions_unique_constraint'
branch_labels: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None
@@ -19,89 +25,97 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None: def upgrade() -> None:
# 1. RawEvent - Bronze 层原始事件存档 # ── raw_events: Bronze 层原始记录 ──
op.create_table( op.create_table(
'raw_events', "raw_events",
sa.Column('id', sa.BigInteger(), primary_key=True), sa.Column("id", sa.Integer(), primary_key=True),
sa.Column('source_system', sa.String(50), nullable=False), sa.Column("source_system", sa.String(30), nullable=False),
sa.Column('source_record_id', sa.String(100), nullable=False), sa.Column("source_record_id", sa.String(100), nullable=False),
sa.Column('raw_payload', JSONB(), nullable=False), sa.Column("raw_payload", JSONB(), nullable=False),
sa.Column('ingested_at', sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
sa.Column('ingest_batch_id', sa.String(36), nullable=True), sa.Column("ingest_batch_id", sa.String(64), nullable=True),
sa.UniqueConstraint('source_system', 'source_record_id', name='uq_raw_event'),
) )
op.create_index('ix_raw_event_batch', 'raw_events', ['ingest_batch_id']) op.create_index("ix_raw_events_batch", "raw_events", ["ingest_batch_id"])
op.create_index("ix_raw_events_source_ingested", "raw_events", ["source_system", "ingested_at"])
op.create_unique_constraint("uq_raw_events_source_record", "raw_events", ["source_system", "source_record_id"])
# 2. IngestFailure - 采集失败死信表 # ── ingest_failures: 采集失败死信表 ──
op.create_table( op.create_table(
'ingest_failures', "ingest_failures",
sa.Column('id', sa.BigInteger(), primary_key=True), sa.Column("id", sa.Integer(), primary_key=True),
sa.Column('source_system', sa.String(50), nullable=False), sa.Column("source_system", sa.String(30), nullable=False),
sa.Column('entity_type', sa.String(50), nullable=False), sa.Column("entity_type", sa.String(30), nullable=False),
sa.Column('source_record_id', sa.String(100), nullable=True), sa.Column("source_record_id", sa.String(100), nullable=True),
sa.Column('error_type', sa.String(50), nullable=False), sa.Column("error_type", sa.String(50), nullable=False),
sa.Column('error_detail', sa.Text(), nullable=True), sa.Column("error_detail", sa.Text(), nullable=True),
sa.Column('raw_payload', JSONB(), nullable=True), sa.Column("raw_payload", JSONB(), nullable=True),
sa.Column('retry_count', sa.Integer(), server_default='0'), sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
sa.Column('next_retry_at', sa.DateTime(timezone=True), nullable=True), sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True),
sa.Column('status', sa.String(20), server_default='pending'), sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
)
op.create_index('ix_ingest_failure_status', 'ingest_failures', ['status', 'next_retry_at'])
op.create_check_constraint(
'ck_ingest_failure_status',
'ingest_failures',
"status IN ('pending', 'retrying', 'resolved', 'abandoned')",
) )
op.create_index("ix_ingest_failures_status_next_retry", "ingest_failures", ["status", "next_retry_at"])
op.create_index("ix_ingest_failures_source", "ingest_failures", ["source_system", "entity_type"])
op.create_check_constraint("ck_ingest_failures_status", "ingest_failures",
"status IN ('pending', 'retrying', 'resolved', 'abandoned')")
op.create_check_constraint("ck_ingest_failures_retry_nonneg", "ingest_failures", "retry_count >= 0")
# 3. DataQualityCheck - 数据质量监控 # ── data_quality_checks: 数据质量检查记录 ──
op.create_table( op.create_table(
'data_quality_checks', "data_quality_checks",
sa.Column('id', sa.BigInteger(), primary_key=True), sa.Column("id", sa.Integer(), primary_key=True),
sa.Column('check_name', sa.String(100), nullable=False), sa.Column("check_name", sa.String(100), nullable=False),
sa.Column('entity_type', sa.String(50), nullable=False), sa.Column("entity_type", sa.String(30), nullable=False),
sa.Column('entity_id', sa.Integer(), nullable=True), sa.Column("entity_id", sa.String(50), nullable=True),
sa.Column('expected_value', sa.Float(), nullable=True), sa.Column("expected_value", sa.Text(), nullable=True),
sa.Column('actual_value', sa.Float(), nullable=False), sa.Column("actual_value", sa.Text(), nullable=True),
sa.Column('passed', sa.Boolean(), nullable=False), sa.Column("passed", sa.Boolean(), nullable=False),
sa.Column('severity', sa.String(10), server_default='warning'), sa.Column("severity", sa.String(10), nullable=False, server_default="warning"),
sa.Column('detail', JSONB(), nullable=True), sa.Column("detail", sa.Text(), nullable=True),
sa.Column('checked_at', sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False),
) )
op.create_index('ix_dqc_checked_at', 'data_quality_checks', ['checked_at']) op.create_index("ix_dqc_check_time", "data_quality_checks", ["check_name", "checked_at"])
op.create_index('ix_dqc_entity', 'data_quality_checks', ['entity_type', 'entity_id']) op.create_index("ix_dqc_entity", "data_quality_checks", ["entity_type", "entity_id"])
op.create_index("ix_dqc_severity_passed", "data_quality_checks", ["severity", "passed"])
op.create_check_constraint("ck_dqc_severity", "data_quality_checks",
"severity IN ('info', 'warning', 'critical')")
# 4. DataLineage - ETL 血缘追踪 # ── data_lineage: ETL 血缘追踪 ──
op.create_table( op.create_table(
'data_lineage', "data_lineage",
sa.Column('id', sa.BigInteger(), primary_key=True), sa.Column("id", sa.Integer(), primary_key=True),
sa.Column('source_system', sa.String(50), nullable=False), sa.Column("source_system", sa.String(30), nullable=False),
sa.Column('source_record_id', sa.String(100), nullable=False), sa.Column("source_record_id", sa.String(100), nullable=False),
sa.Column('target_table', sa.String(50), nullable=False), sa.Column("target_table", sa.String(50), nullable=False),
sa.Column('target_id', sa.Integer(), nullable=True), sa.Column("target_id", sa.Integer(), nullable=True),
sa.Column('transform_name', sa.String(50), nullable=False), sa.Column("transform_name", sa.String(100), nullable=False),
sa.Column('transform_detail', JSONB(), nullable=True), sa.Column("transform_detail", sa.Text(), nullable=True),
sa.Column('batch_id', sa.String(36), nullable=True), sa.Column("batch_id", sa.String(64), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
) )
op.create_index('ix_lineage_source', 'data_lineage', ['source_system', 'source_record_id']) op.create_index("ix_lineage_source", "data_lineage", ["source_system", "source_record_id"])
op.create_index('ix_lineage_target', 'data_lineage', ['target_table', 'target_id']) op.create_index("ix_lineage_target", "data_lineage", ["target_table", "target_id"])
op.create_index('ix_lineage_batch', 'data_lineage', ['batch_id']) op.create_index("ix_lineage_batch", "data_lineage", ["batch_id"])
def downgrade() -> None: def downgrade() -> None:
op.drop_index('ix_lineage_batch', table_name='data_lineage') # 逆序删除
op.drop_index('ix_lineage_target', table_name='data_lineage') op.drop_index("ix_lineage_batch", table_name="data_lineage")
op.drop_index('ix_lineage_source', table_name='data_lineage') op.drop_index("ix_lineage_target", table_name="data_lineage")
op.drop_table('data_lineage') op.drop_index("ix_lineage_source", table_name="data_lineage")
op.drop_table("data_lineage")
op.drop_index('ix_dqc_entity', table_name='data_quality_checks') op.drop_index("ix_dqc_severity_passed", table_name="data_quality_checks")
op.drop_index('ix_dqc_checked_at', table_name='data_quality_checks') op.drop_index("ix_dqc_entity", table_name="data_quality_checks")
op.drop_table('data_quality_checks') op.drop_index("ix_dqc_check_time", table_name="data_quality_checks")
op.drop_table("data_quality_checks")
op.drop_index('ix_ingest_failure_status', table_name='ingest_failures') op.drop_index("ix_ingest_failures_source", table_name="ingest_failures")
op.drop_table('ingest_failures') op.drop_index("ix_ingest_failures_status_next_retry", table_name="ingest_failures")
op.drop_table("ingest_failures")
op.drop_index('ix_raw_event_batch', table_name='raw_events') op.drop_constraint("uq_raw_events_source_record", table_name="raw_events", type_="unique")
op.drop_table('raw_events') op.drop_index("ix_raw_events_source_ingested", table_name="raw_events")
op.drop_index("ix_raw_events_batch", table_name="raw_events")
op.drop_table("raw_events")
+13 -7
View File
@@ -1,9 +1,12 @@
"""MatchStats 添加 xG 追踪字段 """MatchStats 新增 xG 追踪字段
Revision ID: 0009_match_stats_xg_fields Revision ID: 0009_match_stats_xg_fields
Revises: 0008_raw_event_and_ingest_failure Revises: 0008_raw_event_and_ingest_failure
Create Date: 2026-09-17 Create Date: 2026-09-17
架构审查报告 P1-4 实施:
understat 允许纠正旧 xG 值。新增字段追踪 xG 具体来源和更新时间,
实现全量覆盖模式:当 understat 数据更新时覆盖旧值而非跳过。
""" """
from typing import Sequence, Union from typing import Sequence, Union
@@ -11,6 +14,7 @@ from typing import Sequence, Union
from alembic import op from alembic import op
import sqlalchemy as sa import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = '0009_match_stats_xg_fields' revision: str = '0009_match_stats_xg_fields'
down_revision: Union[str, None] = '0008_raw_event_and_ingest_failure' down_revision: Union[str, None] = '0008_raw_event_and_ingest_failure'
branch_labels: Union[str, Sequence[str], None] = None branch_labels: Union[str, Sequence[str], None] = None
@@ -18,12 +22,14 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None: def upgrade() -> None:
op.add_column('match_stats', sa.Column('xg_source', sa.String(30), nullable=True)) op.add_column("match_stats", sa.Column("xg_source", sa.String(30), nullable=True))
op.add_column('match_stats', sa.Column('xg_updated_at', sa.DateTime(timezone=True), nullable=True)) op.add_column("match_stats", sa.Column("xg_updated_at", sa.DateTime(timezone=True), nullable=True))
op.add_column('match_stats', sa.Column('xg_source_record_id', sa.String(100), nullable=True)) op.add_column("match_stats", sa.Column("xg_source_record_id", sa.String(100), nullable=True))
op.create_index("ix_match_stats_xg_source", "match_stats", ["xg_source", "xg_updated_at"])
def downgrade() -> None: def downgrade() -> None:
op.drop_column('match_stats', 'xg_source_record_id') op.drop_index("ix_match_stats_xg_source", table_name="match_stats")
op.drop_column('match_stats', 'xg_updated_at') op.drop_column("match_stats", "xg_source_record_id")
op.drop_column('match_stats', 'xg_source') op.drop_column("match_stats", "xg_updated_at")
op.drop_column("match_stats", "xg_source")
+1 -13
View File
@@ -6,7 +6,7 @@ services:
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD 未设置} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD 未设置}
POSTGRES_DB: ${POSTGRES_DB:-football} POSTGRES_DB: ${POSTGRES_DB:-football}
ports: ports:
- "${POSTGRES_PORT:-5433}:5432" - "${POSTGRES_PORT:-5432}:5432"
volumes: volumes:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
healthcheck: healthcheck:
@@ -26,18 +26,6 @@ services:
condition: service_healthy condition: service_healthy
volumes: volumes:
- ./src:/app/src - ./src:/app/src
- ./alembic:/app/alembic
- ./alembic.ini:/app/alembic.ini
frontend:
image: nginx:alpine
ports:
- "${FRONTEND_PORT:-3000}:80"
volumes:
- ./frontend/dist:/usr/share/nginx/html
- ./frontend/nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on:
- api
volumes: volumes:
pgdata: pgdata:
-38
View File
@@ -1,38 +0,0 @@
server {
listen 80;
server_name localhost;
# Serve static files
root /usr/share/nginx/html;
index index.html;
# API proxy
location /api/ {
proxy_pass http://api:8000/api/;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_cache_bypass $http_upgrade;
}
# Health check proxy
location /health {
proxy_pass http://api:8000/health;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
# SPA routing - serve index.html for all routes
location / {
try_files $uri $uri/ /index.html;
}
# Gzip compression
gzip on;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
gzip_min_length 1000;
}
+43 -43
View File
@@ -9,8 +9,7 @@
"version": "0.1.0", "version": "0.1.0",
"dependencies": { "dependencies": {
"react": "^18.3.1", "react": "^18.3.1",
"react-dom": "^18.3.1", "react-dom": "^18.3.1"
"react-router-dom": "^6.30.6"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "^18.3.3", "@types/react": "^18.3.3",
@@ -767,6 +766,9 @@
"x64" "x64"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -814,15 +816,6 @@
"node": ">= 8" "node": ">= 8"
} }
}, },
"node_modules/@remix-run/router": {
"version": "1.23.4",
"resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.4.tgz",
"integrity": "sha512-q7j5geK7xs3UJSdm9/iytUNclBnLmYx1EnSeCFXHPeutdqgIMeFeHtUZgS3EhlKxdBEAu8OwtJCwmLrEzpSs7Q==",
"license": "MIT",
"engines": {
"node": ">=14.0.0"
}
},
"node_modules/@rolldown/pluginutils": { "node_modules/@rolldown/pluginutils": {
"version": "1.0.0-beta.27", "version": "1.0.0-beta.27",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz",
@@ -922,6 +915,9 @@
"arm" "arm"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -936,6 +932,9 @@
"arm" "arm"
], ],
"dev": true, "dev": true,
"libc": [
"musl"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -950,6 +949,9 @@
"arm64" "arm64"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -964,6 +966,9 @@
"arm64" "arm64"
], ],
"dev": true, "dev": true,
"libc": [
"musl"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -978,6 +983,9 @@
"loong64" "loong64"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -992,6 +1000,9 @@
"loong64" "loong64"
], ],
"dev": true, "dev": true,
"libc": [
"musl"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1006,6 +1017,9 @@
"ppc64" "ppc64"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1020,6 +1034,9 @@
"ppc64" "ppc64"
], ],
"dev": true, "dev": true,
"libc": [
"musl"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1034,6 +1051,9 @@
"riscv64" "riscv64"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1048,6 +1068,9 @@
"riscv64" "riscv64"
], ],
"dev": true, "dev": true,
"libc": [
"musl"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1062,6 +1085,9 @@
"s390x" "s390x"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1076,6 +1102,9 @@
"x64" "x64"
], ],
"dev": true, "dev": true,
"libc": [
"glibc"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -1090,6 +1119,9 @@
"x64" "x64"
], ],
"dev": true, "dev": true,
"libc": [
"musl"
],
"license": "MIT", "license": "MIT",
"optional": true, "optional": true,
"os": [ "os": [
@@ -2255,38 +2287,6 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/react-router": {
"version": "6.30.6",
"resolved": "https://registry.npmjs.org/react-router/-/react-router-6.30.6.tgz",
"integrity": "sha512-5HfK7k5im7LTOB0EqCQmfvy4C13G92Ssj1VTmouTK3AJvyjKTnFuCV0vcMAD/JS+JC4DvDIBRrlAeJIFjh5VWg==",
"license": "MIT",
"dependencies": {
"@remix-run/router": "1.23.4"
},
"engines": {
"node": ">=14.0.0"
},
"peerDependencies": {
"react": ">=16.8"
}
},
"node_modules/react-router-dom": {
"version": "6.30.6",
"resolved": "https://registry.npmjs.org/react-router-dom/-/react-router-dom-6.30.6.tgz",
"integrity": "sha512-0RHKZz7wwffvkU+2MFVT2NnjK44ssLEV+m0CAJaS2Ksmorrwj7WxH00jO0SOCW26/tINUnJHToXblDs33I38YQ==",
"license": "MIT",
"dependencies": {
"@remix-run/router": "1.23.4",
"react-router": "6.30.6"
},
"engines": {
"node": ">=14.0.0"
},
"peerDependencies": {
"react": ">=16.8",
"react-dom": ">=16.8"
}
},
"node_modules/read-cache": { "node_modules/read-cache": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.2.tgz",
+1 -2
View File
@@ -10,8 +10,7 @@
}, },
"dependencies": { "dependencies": {
"react": "^18.3.1", "react": "^18.3.1",
"react-dom": "^18.3.1", "react-dom": "^18.3.1"
"react-router-dom": "^6.30.6"
}, },
"devDependencies": { "devDependencies": {
"@types/react": "^18.3.3", "@types/react": "^18.3.3",
+8 -43
View File
@@ -1,17 +1,7 @@
/**
* 主应用入口
*
* 整合前台(报纸风格)和后台(暗色管理)的路由。
* - / → 先知(Profeto)主站
* - /admin/* → 管理后台
*/
import { BrowserRouter, Routes, Route, Navigate } from 'react-router-dom'
import { ErrorBoundary } from './components/ErrorBoundary' import { ErrorBoundary } from './components/ErrorBoundary'
import Matches from './pages/Matches' import Matches from './pages/Matches'
import { adminRoutes } from './admin/routes'
/** 报眉日期行 */ /** 报眉日期行:2026年9月15日 星期二 */
function dateLine(): string { function dateLine(): string {
return new Date().toLocaleDateString('zh-CN', { return new Date().toLocaleDateString('zh-CN', {
year: 'numeric', year: 'numeric',
@@ -21,8 +11,9 @@ function dateLine(): string {
}) })
} }
function HomePage() { export default function App() {
return ( return (
<ErrorBoundary>
<div className="min-h-screen bg-paper-50"> <div className="min-h-screen bg-paper-50">
{/* ── 报头:粗线 + 居中刊名 + 报眉 ── */} {/* ── 报头:粗线 + 居中刊名 + 报眉 ── */}
<header className="masthead-rule"> <header className="masthead-rule">
@@ -40,13 +31,10 @@ function HomePage() {
</div> </div>
<div className="flex items-center justify-between border-b border-ink-200 py-2 text-2xs text-ink-500"> <div className="flex items-center justify-between border-b border-ink-200 py-2 text-2xs text-ink-500">
<span>{dateLine()}</span> <span>{dateLine()}</span>
<a <span className="flex items-center gap-1.5">
href="/admin" <span className="inline-block h-1.5 w-1.5 bg-emerald-600" aria-hidden="true" />
className="flex items-center gap-1.5 text-press hover:text-press-dark transition-colors"
> </span>
<span aria-hidden="true"></span>
</a>
</div> </div>
</div> </div>
</header> </header>
@@ -55,36 +43,13 @@ function HomePage() {
<Matches /> <Matches />
</main> </main>
{/* ── 版底 ── */}
<footer className="mx-auto max-w-5xl px-5 pb-10 sm:px-8"> <footer className="mx-auto max-w-5xl px-5 pb-10 sm:px-8">
<div className="border-t border-ink-200 pt-3 text-center text-2xs leading-relaxed text-ink-400"> <div className="border-t border-ink-200 pt-3 text-center text-2xs leading-relaxed text-ink-400">
· · · ·
</div> </div>
</footer> </footer>
</div> </div>
)
}
export default function App() {
return (
<ErrorBoundary>
<BrowserRouter>
<Routes>
<Route path="/" element={<HomePage />} />
{adminRoutes.map(route => (
<Route key={route.path} path={route.path} element={route.element}>
{route.children.map(child => (
<Route
key={child.path ?? 'index'}
index={child.index}
path={child.path}
element={child.element}
/>
))}
</Route>
))}
<Route path="*" element={<Navigate to="/" replace />} />
</Routes>
</BrowserRouter>
</ErrorBoundary> </ErrorBoundary>
) )
} }
-15
View File
@@ -1,15 +0,0 @@
/**
* Admin 后台 - 应用入口
*
* 独立的 Admin 应用入口,用于 createBrowserRouter。
* 也可通过 createHashRouter 直接挂载为独立应用。
*/
import { createBrowserRouter, RouterProvider } from 'react-router-dom'
import { adminRoutes } from './routes'
const router = createBrowserRouter(adminRoutes)
export default function AdminApp() {
return <RouterProvider router={router} />
}
-155
View File
@@ -1,155 +0,0 @@
/**
* Admin 后台 - 布局组件
*
* 响应式布局: 移动端汉堡菜单 + 可折叠侧边栏
* 桌面端: 固定侧边栏 + 内容区
* 暗色主题,参考线性风格设计
*/
import { useState, useEffect, useCallback } from 'react'
import { NavLink, Outlet, useLocation } from 'react-router-dom'
const NAV_ITEMS = [
{ to: '/admin', label: '仪表盘', icon: '◇', end: true },
{ to: '/admin/collection', label: '数据采集', icon: '◈' },
{ to: '/admin/predictions', label: '预测管理', icon: '◆' },
{ to: '/admin/backtest', label: '回测管理', icon: '◉' },
{ to: '/admin/monitoring', label: '监控面板', icon: '◐' },
{ to: '/admin/data-sources', label: '数据源', icon: '◫' },
{ to: '/admin/llm-config', label: 'LLM 配置', icon: '◬' },
{ to: '/admin/config', label: '系统配置', icon: '◑' },
]
export default function AdminLayout() {
const [sidebarOpen, setSidebarOpen] = useState(false)
const location = useLocation()
// 路由变化时关闭移动端菜单
const closeSidebar = useCallback(() => setSidebarOpen(false), [])
useEffect(() => {
closeSidebar()
}, [location.pathname, closeSidebar])
// ESC 键关闭菜单
useEffect(() => {
const handler = (e: KeyboardEvent) => {
if (e.key === 'Escape') setSidebarOpen(false)
}
document.addEventListener('keydown', handler)
return () => document.removeEventListener('keydown', handler)
}, [])
return (
<div className="flex h-screen bg-gray-950 text-gray-200 overflow-hidden">
{/* ── 移动端遮罩层 ── */}
{sidebarOpen && (
<div
className="fixed inset-0 z-40 bg-black/60 backdrop-blur-sm lg:hidden"
onClick={() => setSidebarOpen(false)}
aria-hidden="true"
/>
)}
{/* ── 侧边栏 ── */}
<aside
className={`
fixed inset-y-0 left-0 z-50 flex w-64 flex-shrink-0 flex-col border-r border-gray-800 bg-gray-900
transform transition-transform duration-200 ease-in-out
lg:relative lg:z-auto lg:translate-x-0
${sidebarOpen ? 'translate-x-0' : '-translate-x-full'}
`}
aria-label="主导航"
>
{/* Logo */}
<div className="flex items-center justify-between border-b border-gray-800 px-5 py-4">
<h1 className="font-serif text-lg font-bold tracking-wider text-gray-100">
Profeto
<span className="ml-2 text-xs font-normal tracking-normal text-gray-500">Admin</span>
</h1>
{/* 移动端关闭按钮 */}
<button
onClick={() => setSidebarOpen(false)}
className="rounded-md p-2 text-gray-400 hover:bg-gray-800 hover:text-gray-200 lg:hidden min-h-[44px] min-w-[44px]"
aria-label="关闭菜单"
>
<svg className="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M6 18L18 6M6 6l12 12" />
</svg>
</button>
</div>
{/* 导航 */}
<nav className="flex-1 overflow-y-auto px-3 py-4">
<ul className="space-y-0.5">
{NAV_ITEMS.map(item => (
<li key={item.to}>
<NavLink
to={item.to}
end={item.end}
className={({ isActive }) =>
`flex items-center gap-3 rounded-md px-3 py-2.5 text-sm transition-colors min-h-[44px] ${
isActive
? 'bg-gray-800 text-white font-medium'
: 'text-gray-400 hover:bg-gray-800/60 hover:text-gray-200'
}`
}
>
<span className="text-base leading-none opacity-60" aria-hidden="true">
{item.icon}
</span>
{item.label}
</NavLink>
</li>
))}
</ul>
</nav>
{/* 底部 */}
<div className="border-t border-gray-800 px-4 py-3">
<a
href="/"
className="flex items-center gap-2 text-xs text-gray-500 transition-colors hover:text-gray-300 min-h-[44px]"
>
<span aria-hidden="true"></span>
</a>
</div>
</aside>
{/* ── 主内容区 ── */}
<div className="flex flex-1 flex-col overflow-hidden">
{/* 顶栏 */}
<header className="flex h-12 flex-shrink-0 items-center justify-between border-b border-gray-800 bg-gray-900 px-4 lg:px-6">
{/* 移动端菜单按钮 */}
<button
onClick={() => setSidebarOpen(true)}
className="rounded-md p-2 text-gray-400 hover:bg-gray-800 hover:text-gray-200 lg:hidden min-h-[44px] min-w-[44px]"
aria-label="打开菜单"
>
<svg className="h-5 w-5" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M4 6h16M4 12h16M4 18h16" />
</svg>
</button>
{/* 状态指示 */}
<div className="flex items-center gap-2 text-xs text-gray-500">
<span className="inline-block h-2 w-2 rounded-full bg-emerald-500" aria-hidden="true" />
<span className="hidden sm:inline"></span>
</div>
{/* 版本 */}
<div className="text-xs text-gray-500">
<span className="hidden sm:inline">Profeto Admin v1.0</span>
<span className="sm:hidden">v1.0</span>
</div>
</header>
{/* 页面内容 */}
<main className="flex-1 overflow-y-auto p-4 lg:p-6">
<Outlet />
</main>
</div>
</div>
)
}
-129
View File
@@ -1,129 +0,0 @@
# Profeto Admin 后台管理系统
## 概述
Profeto 后台管理界面,为足球 LLM 预测系统提供运维管理能力。
暗色主题设计,支持响应式布局(移动端 / 平板 / 桌面)。
## 文件结构
```
src/admin/
├── AdminApp.tsx # Admin 应用入口(独立路由)
├── AdminLayout.tsx # 管理后台布局(侧边栏 + 顶栏 + 内容区)
├── api.ts # 统一 API 客户端(带超时、错误处理、类型安全)
├── dal.ts # 数据访问层(封装所有 API 端点调用)
├── types.ts # TypeScript 类型定义(与 FastAPI Pydantic 对齐)
├── components.tsx # 通用 UI 组件(Card, Table, Badge, ResponsiveTable...)
├── routes.tsx # 路由定义(/admin/*)
└── pages/
├── Dashboard.tsx # 仪表盘(系统概览)
├── Collection.tsx # 数据采集(触发采集任务)
├── Predictions.tsx # 预测管理(触发预测 + 评估结算)
├── Backtest.tsx # 回测管理(策略验证)
├── Monitoring.tsx # 监控面板(健康检查 + 错误日志)
├── DataSources.tsx # 数据源管理(数据源配置与测试)
├── LLMConfig.tsx # LLM 配置(模型连接与统计)
└── Config.tsx # 系统配置(.env 配置查看与修改指南)
```
## 页面说明
### 1. 仪表盘 (`/admin`)
- 系统健康状态概览
- 联赛 / 比赛 / 预测数量统计
- 最近联赛列表
- 快捷操作导航
### 2. 数据采集 (`/admin/collection`)
- 选择数据源: Bzzoiro / Understat / Injuries
- 选择联赛、日期范围
- 触发采集任务
- 实时任务进度显示(每 5 秒自动刷新)
### 3. 预测管理 (`/admin/predictions`)
- 触发预测(选择比赛 + 模式)
- 模式切换: 五路专家 / 单一模型
- 预测历史列表
### 4. 回测管理 (`/admin/backtest`)
- 回测配置: 联赛、日期范围、场数、模式
- 回测结果: 准确率、已评分数
- 模型评估统计
### 5. 监控面板 (`/admin/monitoring`)
- 系统健康状态(服务 + 检查项)
- 版本与运行时间
### 6. 数据源管理 (`/admin/data-sources`) ✨ 新增
- 数据源状态: API Key 配置状态(脱敏)
- 测试连接: 调用采集 API 验证
- 数据源说明文档
### 7. LLM 配置 (`/admin/llm-config`) ✨ 新增
- 当前配置: provider, model, base_url
- 连接测试: 调用 /predict 验证
- 使用统计: 预测次数、延迟、成功率
- 可用模型列表
### 8. 系统配置 (`/admin/config`) 🔧 增强
- 配置列表: 脱敏显示所有 .env 配置项
- 配置修改指南: SSH 修改 .env + 重启服务
- 快速导航: 数据源 / LLM 配置页面
## 路由设计
| 路径 | 页面 | 描述 |
|------|------|------|
| `/` | 先知主站 | 报纸风格预测展示 |
| `/admin` | 仪表盘 | 系统概览 |
| `/admin/collection` | 数据采集 | 采集任务管理 |
| `/admin/predictions` | 预测管理 | 预测与评估 |
| `/admin/backtest` | 回测管理 | 策略回测 |
| `/admin/monitoring` | 监控面板 | 系统监控 |
| `/admin/data-sources` | 数据源管理 | 数据源配置 |
| `/admin/llm-config` | LLM 配置 | 模型管理 |
| `/admin/config` | 系统配置 | 参数配置 |
## 响应式布局
### 移动端 (< 768px)
- 侧边栏折叠为汉堡菜单,点击展开
- 表格隐藏,显示卡片视图
- 表单单列布局
- 按钮最小 44px 触摸目标
- 统计卡片 1 列
### 平板 (768px - 1024px)
- 侧边栏可折叠
- 部分表格可用
- 统计卡片 2 列
### 桌面 (> 1024px)
- 侧边栏固定显示
- 完整表格视图
- 统计卡片 4 列
## 技术实现
- **路由**: `react-router-dom` v6 嵌套路由
- **样式**: Tailwind CSS 暗色主题 (gray-900/800/700)
- **API**: 统一 fetch 客户端,30s 超时,类型安全
- **类型**: TypeScript strict mode,与后端 Pydantic 模型对齐
- **错误处理**: ApiError 类 + 页面级错误展示
- **响应式**: Tailwind 断点 (sm:, md:, lg:, xl:)
- **触摸友好**: 所有按钮 min-h-[44px]
- **移动端**: 可折叠侧边栏 + 卡片视图替代表格
## 启动方式
```bash
cd /vol2/1000/Docker/Profeto/frontend
npm run dev
# 访问 http://localhost:5173/admin
```
## 访问入口
主站页面顶部「管理后台」链接可跳转至 `/admin`
Admin 后台左侧底栏「返回前台」链接可回到 `/`
-85
View File
@@ -1,85 +0,0 @@
/**
* Admin 后台管理系统 - 统一 API 客户端
*/
const API_BASE = '/api/v1'
const TIMEOUT_MS = 30_000
export class ApiError extends Error {
constructor(
message: string,
public status: number,
public data?: unknown,
) {
super(message)
this.name = 'ApiError'
}
}
async function request<T>(path: string, options: RequestInit = {}): Promise<T> {
// 修复: 正确拼接 API_BASE
const url = path.startsWith('http')
? path
: path.startsWith('/')
? path // 已经是绝对路径(如 /health)
: `${API_BASE}${path}`
const controller = new AbortController()
const timer = setTimeout(() => controller.abort(), TIMEOUT_MS)
try {
const res = await fetch(url, {
...options,
signal: controller.signal,
headers: {
'Content-Type': 'application/json',
...options.headers,
},
})
if (!res.ok) {
let detail: unknown
try {
detail = await res.json()
} catch {
detail = await res.text()
}
throw new ApiError(
detail && typeof detail === 'object' && 'detail' in detail
? String((detail as { detail: unknown }).detail)
: `HTTP ${res.status}: ${res.statusText}`,
res.status,
detail,
)
}
// 修复: 正确判断 204 No Content
if (res.status === 204) {
return undefined as T
}
return res.json() as Promise<T>
} catch (err) {
if (err instanceof ApiError) throw err
if (err instanceof DOMException && err.name === 'AbortError') {
throw new ApiError('请求超时,请稍后重试', 0)
}
throw new ApiError(
err instanceof Error ? err.message : '网络错误,请检查连接',
0,
)
} finally {
clearTimeout(timer)
}
}
export const api = {
get: <T>(path: string) => request<T>(path),
post: <T>(path: string, body?: unknown) =>
request<T>(path, { method: 'POST', body: body ? JSON.stringify(body) : undefined }),
put: <T>(path: string, body?: unknown) =>
request<T>(path, { method: 'PUT', body: body ? JSON.stringify(body) : undefined }),
delete: <T>(path: string) => request<T>(path, { method: 'DELETE' }),
}
export { API_BASE }
-286
View File
@@ -1,286 +0,0 @@
/**
* Admin 后台 - 通用 UI 组件集合
*
* 提供管理界面使用的所有基础组件:
* - Card: 内容卡片
* - StatCard: 统计数字卡片
* - Badge: 状态标签
* - DataTable: 通用数据表格
* - ProgressBar: 进度条
* - EmptyState: 空状态占位
* - SectionHeader: 小节标题
*/
import { ReactNode } from 'react'
// ── 卡片 ────────────────────────────────────────────────────────
export function Card({
children,
className = '',
}: {
children: ReactNode
className?: string
}) {
return (
<div className={`rounded-lg border border-gray-800 bg-gray-900 ${className}`}>
{children}
</div>
)
}
export function CardHeader({
title,
description,
action,
}: {
title: string
description?: string
action?: ReactNode
}) {
return (
<div className="border-b border-gray-800 px-5 py-3">
<div className="flex items-center justify-between">
<h3 className="text-sm font-medium text-gray-200">{title}</h3>
{action}
</div>
{description && (
<p className="mt-1 text-xs text-gray-500">{description}</p>
)}
</div>
)
}
export function CardBody({
children,
className = '',
}: {
children: ReactNode
className?: string
}) {
return <div className={`px-5 py-4 ${className}`}>{children}</div>
}
// ── 统计卡片 ────────────────────────────────────────────────────
export function StatCard({
label,
value,
icon,
trend,
}: {
label: string
value: string | number
icon?: string
trend?: { value: number; positive: boolean }
}) {
return (
<div className="rounded-lg border border-gray-800 bg-gray-900 px-5 py-4">
<div className="flex items-center justify-between">
<span className="text-xs text-gray-500">{label}</span>
{icon && (
<span className="text-lg opacity-40" aria-hidden="true">
{icon}
</span>
)}
</div>
<div className="mt-2 text-2xl font-semibold text-gray-100 tabular-nums">{value}</div>
{trend && (
<div
className={`mt-1 text-xs tabular-nums ${
trend.positive ? 'text-emerald-400' : 'text-red-400'
}`}
>
{trend.positive ? '↑' : '↓'} {Math.abs(trend.value)}%
</div>
)}
</div>
)
}
// ── 状态标签 ────────────────────────────────────────────────────
const BADGE_COLORS: Record<string, string> = {
success: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/30',
running: 'bg-blue-500/15 text-blue-400 border-blue-500/30',
queued: 'bg-yellow-500/15 text-yellow-400 border-yellow-500/30',
failed: 'bg-red-500/15 text-red-400 border-red-500/30',
pending: 'bg-gray-500/15 text-gray-400 border-gray-500/30',
completed: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/30',
error: 'bg-red-500/15 text-red-400 border-red-500/30',
warning: 'bg-yellow-500/15 text-yellow-400 border-yellow-500/30',
info: 'bg-blue-500/15 text-blue-400 border-blue-500/30',
win: 'bg-emerald-500/15 text-emerald-400 border-emerald-500/30',
loss: 'bg-red-500/15 text-red-400 border-red-500/30',
push: 'bg-gray-500/15 text-gray-400 border-gray-500/30',
}
export function Badge({
status,
children,
}: {
status: string
children: ReactNode
}) {
const colorClass = BADGE_COLORS[status] ?? 'bg-gray-500/15 text-gray-400 border-gray-500/30'
return (
<span
className={`inline-flex items-center rounded-full border px-2 py-0.5 text-xs font-medium ${colorClass}`}
>
{children}
</span>
)
}
// ── 数据表格 ────────────────────────────────────────────────────
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function DataTable<T = any>({
columns,
data,
rowKey,
emptyText = '暂无数据',
}: {
columns: { key: string; label: string; render?: (row: T) => ReactNode; width?: string }[]
data: T[]
rowKey: (row: T) => string | number
emptyText?: string
}) {
if (data.length === 0) {
return <EmptyState text={emptyText} />
}
return (
<div className="overflow-x-auto">
<table className="w-full text-left text-sm">
<thead>
<tr className="border-b border-gray-800 text-xs text-gray-500">
{columns.map(col => (
<th key={col.key} className="px-3 py-2.5 font-medium" style={{ width: col.width }}>
{col.label}
</th>
))}
</tr>
</thead>
<tbody>
{data.map(row => (
<tr
key={rowKey(row)}
className="border-b border-gray-800/50 transition-colors hover:bg-gray-800/30"
>
{columns.map(col => (
<td key={col.key} className="px-3 py-2.5 text-gray-300">
{col.render
? col.render(row)
: row != null && typeof row === 'object' && col.key in row
? String((row as Record<string, unknown>)[col.key] ?? '—')
: '—'}
</td>
))}
</tr>
))}
</tbody>
</table>
</div>
)
}
// ── 进度条 ──────────────────────────────────────────────────────
export function ProgressBar({ value }: { value: number }) {
const clamped = Math.max(0, Math.min(100, value))
return (
<div className="h-1.5 w-full rounded-full bg-gray-800" role="progressbar" aria-valuenow={clamped}>
<div
className="h-full rounded-full bg-blue-500 transition-all duration-300"
style={{ width: `${clamped}%` }}
/>
</div>
)
}
// ── 空状态 ──────────────────────────────────────────────────────
export function EmptyState({ text = '暂无数据' }: { text?: string }) {
return (
<div className="flex items-center justify-center py-12 text-sm text-gray-600">
{text}
</div>
)
}
// ── 移动端卡片列表 (替代桌面端表格) ────────────────────────────
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function MobileCardList<T = any>({
data,
renderCard,
emptyText = '暂无数据',
}: {
data: T[]
renderCard: (row: T, index: number) => ReactNode
emptyText?: string
}) {
if (data.length === 0) {
return <EmptyState text={emptyText} />
}
return (
<div className="space-y-3 lg:hidden">
{data.map((row, idx) => (
<div key={idx} className="rounded-lg border border-gray-800 bg-gray-900 p-4">
{renderCard(row, idx)}
</div>
))}
</div>
)
}
// ── 响应式表格容器 ──────────────────────────────────────────────
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export function ResponsiveTable<T = any>({
columns,
data,
rowKey,
cardRender,
emptyText = '暂无数据',
}: {
columns: { key: string; label: string; render?: (row: T) => ReactNode; width?: string }[]
data: T[]
rowKey: (row: T) => string | number
cardRender: (row: T, index: number) => ReactNode
emptyText?: string
}) {
if (data.length === 0) {
return <EmptyState text={emptyText} />
}
return (
<>
{/* 桌面端表格 */}
<div className="hidden lg:block overflow-x-auto">
<DataTable columns={columns} data={data} rowKey={rowKey} emptyText={emptyText} />
</div>
{/* 移动端卡片 */}
<MobileCardList data={data} renderCard={cardRender} emptyText={emptyText} />
</>
)
}
// ── 小节标题 ────────────────────────────────────────────────────
export function SectionHeader({
title,
description,
}: {
title: string
description?: string
}) {
return (
<div className="mb-4">
<h2 className="text-lg font-semibold text-gray-100">{title}</h2>
{description && <p className="mt-1 text-sm text-gray-500">{description}</p>}
</div>
)
}
-238
View File
@@ -1,238 +0,0 @@
/**
* Admin 后台 - 数据访问层
*
* 封装所有 API 端点调用,返回类型安全的数据。
* 所有端点对齐 FastAPI 后端实际实现。
*/
import { api, API_BASE } from './api'
import type {
DashboardStats,
CollectionRequest,
BacktestRequest,
BacktestSummary,
League,
Match,
Prediction,
EvalSummary,
} from './types'
// ── 仪表盘 ──────────────────────────────────────────────────────
/**
* 从多个端点聚合仪表盘数据。
* 后端暂无专用仪表盘端点,这里组合 health + 各列表端点。
*/
export async function fetchDashboard(): Promise<DashboardStats> {
// 并行获取各端点数据
const [leagues, matches, predictions, health] = await Promise.allSettled([
api.get<League[]>(`${API_BASE}/leagues`),
api.get<Match[]>(`${API_BASE}/matches?limit=1`),
api.get<Prediction[]>(`${API_BASE}/predictions?limit=1`),
api.get<{ status: string }>('/health'),
])
return {
leagues: leagues.status === 'fulfilled' ? leagues.value : [],
total_matches: matches.status === 'fulfilled' ? (matches.value as any)?.total ?? 0 : 0,
total_predictions: predictions.status === 'fulfilled' ? (predictions.value as any)?.total ?? 0 : 0,
health: health.status === 'fulfilled' ? (health.value as any).status : 'unknown',
db_tables: [], // 后端暂无表统计端点
last_collection: [], // 后端暂无采集历史端点
recent_errors: [], // 后端暂无错误日志端点
}
}
// ── 数据采集 ────────────────────────────────────────────────────
export async function triggerCollection(req: CollectionRequest): Promise<any> {
const sourceMap: Record<string, { path: string; body: any }> = {
bzzoiro: {
path: `${API_BASE}/ingest/bzzoiro`,
body: {
leagues: req.leagues,
date_from: req.date_from,
date_to: req.date_to,
status: 'finished',
},
},
understat: {
path: `${API_BASE}/ingest/understat`,
body: {
league: req.league,
season: req.season ? parseInt(req.season) : new Date().getFullYear(),
},
},
injuries: {
path: `${API_BASE}/ingest/injuries`,
body: {
date: req.date_from || new Date().toISOString().slice(0, 10),
},
},
}
const cfg = sourceMap[req.source]
if (!cfg) throw new Error(`未知数据源: ${req.source}`)
return api.post(cfg.path, cfg.body)
}
// ── 预测管理 ────────────────────────────────────────────────────
export async function triggerPrediction(req: { match_id: number; mode?: string }): Promise<any> {
return api.post(`${API_BASE}/predict`, {
match_id: req.match_id,
mode: req.mode || 'multi',
})
}
export async function fetchPredictions(limit = 50): Promise<any[]> {
const res = await api.get<any>(`${API_BASE}/predictions?limit=${limit}`)
return Array.isArray(res) ? res : (res as any)?.items ?? []
}
// ── 评估 & 回测 ─────────────────────────────────────────────────
export async function fetchEvalSummary(): Promise<EvalSummary | null> {
try {
return await api.get<EvalSummary>(`${API_BASE}/eval/summary`)
} catch {
return null
}
}
export async function triggerBacktest(req: BacktestRequest): Promise<BacktestSummary> {
return api.post<BacktestSummary>(`${API_BASE}/backtest`, req)
}
// ── 辅助数据 ────────────────────────────────────────────────────
export async function fetchLeagues(): Promise<League[]> {
try {
return await api.get<League[]>(`${API_BASE}/leagues`)
} catch {
return []
}
}
export async function fetchMatches(params: {
league?: string
status?: string
limit?: number
cursor?: string
} = {}): Promise<{ items: Match[]; has_next: boolean; next_cursor: string | null }> {
const sp = new URLSearchParams()
if (params.league) sp.set('league', params.league)
if (params.status) sp.set('status', params.status)
if (params.limit) sp.set('limit', String(params.limit))
if (params.cursor) sp.set('cursor', params.cursor)
try {
return await api.get<any>(`${API_BASE}/matches?${sp}`)
} catch {
return { items: [], has_next: false, next_cursor: null }
}
}
export async function settlePrediction(prediction_id: number, home_goals: number, away_goals: number): Promise<any> {
return api.post(`${API_BASE}/eval/settle`, {
prediction_id,
home_goals,
away_goals,
})
}
// ── 健康检查 ────────────────────────────────────────────────────
export async function fetchHealth(): Promise<any> {
try {
return await api.get<any>('/health')
} catch {
return { status: 'unknown' }
}
}
// ── 数据源管理 ──────────────────────────────────────────────────
/**
* 测试数据源连接 — 调用采集 API 验证连通性
*/
export async function testDataSource(source: 'bzzoiro' | 'understat' | 'injuries'): Promise<any> {
const sourceMap: Record<string, { path: string; body: any }> = {
bzzoiro: { path: `${API_BASE}/ingest/bzzoiro`, body: { leagues: [], date_from: '', date_to: '', status: 'finished' } },
understat: { path: `${API_BASE}/ingest/understat`, body: { league: 'EPL', season: new Date().getFullYear() } },
injuries: { path: `${API_BASE}/ingest/injuries`, body: { date: new Date().toISOString().slice(0, 10) } },
}
const cfg = sourceMap[source]
if (!cfg) throw new Error(`未知数据源: ${source}`)
return api.post(cfg.path, cfg.body)
}
/**
* 获取数据源状态 — 后端暂无专用端点,返回模拟状态
*/
export async function fetchDataSourceStatuses(): Promise<any[]> {
// 后端暂无专用配置端点,返回静态信息
return [
{ name: 'bzzoiro', label: 'Bzzoiro', keyConfigured: true, maskedKey: 'bz***xxx', lastIngestion: null, status: 'configured' },
{ name: 'understat', label: 'Understat', keyConfigured: true, maskedKey: '无需 Key', lastIngestion: null, status: 'configured' },
{ name: 'injuries', label: 'Injuries', keyConfigured: true, maskedKey: 'inj***xxx', lastIngestion: null, status: 'configured' },
]
}
// ── LLM 配置 ────────────────────────────────────────────────────
/**
* 测试 LLM 连接 — 调用预测端点验证
*/
export async function testLLMConnection(matchId?: number): Promise<any> {
return api.post(`${API_BASE}/predict`, {
match_id: matchId || 1,
mode: 'single',
})
}
/**
* 获取 LLM 使用统计 — 从预测列表聚合
*/
export async function fetchLLMUsageStats(): Promise<any> {
try {
const predictions = await fetchPredictions(50)
const total = predictions.length
const successCount = predictions.filter((p: any) => p.pred_1x2).length
return {
total_predictions: total,
avg_latency_ms: 2400, // 后端暂无延迟统计
success_rate: total > 0 ? (successCount / total) * 100 : 0,
recent_predictions: predictions.slice(0, 10).map((p: any) => ({
id: p.id,
match_id: p.match_id,
model: p.model,
created_at: p.created_at,
status: p.pred_1x2 ? 'success' : 'failed',
})),
}
} catch {
return {
total_predictions: 0,
avg_latency_ms: 0,
success_rate: 0,
recent_predictions: [],
}
}
}
// ── 系统配置 ────────────────────────────────────────────────────
/**
* 获取系统配置列表 — 后端暂无配置端点,返回静态信息
*/
export async function fetchSystemConfig(): Promise<any[]> {
return [
{ key: 'LLM_PROVIDER', value_masked: 'openai', description: 'LLM 提供商', is_sensitive: false },
{ key: 'LLM_MODEL', value_masked: 'gpt-4o', description: 'LLM 模型', is_sensitive: false },
{ key: 'LLM_BASE_URL', value_masked: 'https://api.openai.com/v1', description: 'API 基础地址', is_sensitive: false },
{ key: 'LLM_API_KEY', value_masked: 'sk-****...****', description: 'LLM API 密钥', is_sensitive: true },
{ key: 'BZZOIRO_KEY', value_masked: 'bz****...****', description: 'Bzzoiro 数据源密钥', is_sensitive: true },
{ key: 'DATABASE_URL', value_masked: 'postgresql://****@localhost/profeto', description: '数据库连接', is_sensitive: true },
{ key: 'LOG_LEVEL', value_masked: 'INFO', description: '日志级别', is_sensitive: false },
]
}
-199
View File
@@ -1,199 +0,0 @@
/**
* Admin 后台 - 回测管理页面
*
* 响应式布局: 移动端单列,桌面端双列
* 触摸友好: 按钮最小 44px 高度
*/
import { useState } from 'react'
import { triggerBacktest, fetchEvalSummary } from '../dal'
import type { BacktestRequest, EvalSummary } from '../types'
import { Card, CardBody, CardHeader, Badge, SectionHeader } from '../components'
export default function BacktestPage() {
const [leagueId, setLeagueId] = useState('')
const [dateFrom, setDateFrom] = useState('')
const [dateTo, setDateTo] = useState('')
const [limit, setLimit] = useState(20)
const [mode, setMode] = useState<'single' | 'multi'>('single')
const [loading, setLoading] = useState(false)
const [error, setError] = useState<string | null>(null)
const [result, setResult] = useState<any>(null)
const [evalSummary, setEvalSummary] = useState<EvalSummary | null>(null)
async function handleBacktest(e: React.FormEvent) {
e.preventDefault()
setLoading(true)
setError(null)
setResult(null)
try {
const req: BacktestRequest = {
league_id: leagueId ? parseInt(leagueId) : undefined,
date_from: dateFrom || undefined,
date_to: dateTo || undefined,
limit,
mode,
}
const res = await triggerBacktest(req)
setResult(res)
} catch (err: unknown) {
setError(err instanceof Error ? err.message : '回测失败')
} finally {
setLoading(false)
}
}
async function loadEval() {
const summary = await fetchEvalSummary()
setEvalSummary(summary)
}
return (
<div className="space-y-6">
<SectionHeader
title="回测管理"
description="在历史数据上运行预测并评估准确率"
/>
<div className="grid gap-6 lg:grid-cols-2">
{/* 回测配置 */}
<Card>
<CardHeader title="回测配置" />
<CardBody>
<form onSubmit={handleBacktest} className="space-y-4">
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"> ID ()</label>
<input
type="number"
value={leagueId}
onChange={e => setLeagueId(e.target.value)}
placeholder="留空=全部"
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<input
type="date"
value={dateFrom}
onChange={e => setDateFrom(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<input
type="date"
value={dateTo}
onChange={e => setDateTo(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
</div>
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<input
type="number"
value={limit}
onChange={e => setLimit(parseInt(e.target.value) || 20)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<select
value={mode}
onChange={e => setMode(e.target.value as 'single' | 'multi')}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
>
<option value="single"> ()</option>
<option value="multi"> Agent ()</option>
</select>
</div>
{error && (
<div className="rounded-md bg-red-500/10 p-3 text-sm text-red-400 border border-red-500/30">
{error}
</div>
)}
<button
type="submit"
disabled={loading}
className="w-full rounded-md bg-blue-600 px-4 py-2.5 text-white hover:bg-blue-700 disabled:opacity-50 transition-colors min-h-[44px]"
>
{loading ? '回测中...' : '开始回测'}
</button>
</form>
</CardBody>
</Card>
{/* 结果区域 */}
<div className="space-y-6">
{result && (
<Card>
<CardHeader title="回测结果" />
<CardBody>
<div className="grid grid-cols-2 gap-3">
<div className="rounded-lg bg-gray-800 p-4 text-center">
<div className="text-2xl font-bold text-white">{result.scored}/{result.total}</div>
<div className="text-xs text-gray-400 mt-1"></div>
</div>
<div className="rounded-lg bg-gray-800 p-4 text-center">
<div className="text-2xl font-bold text-blue-400">
{result.accuracy_1x2?.toFixed(1) ?? '—'}%
</div>
<div className="text-xs text-gray-400 mt-1">1X2 </div>
</div>
</div>
</CardBody>
</Card>
)}
<Card>
<CardHeader
title="模型评估"
action={
<button
onClick={loadEval}
className="rounded px-2 py-1 text-xs text-blue-400 hover:bg-gray-800 min-h-[44px] min-w-[44px]"
>
</button>
}
/>
<CardBody>
{!evalSummary ? (
<div className="text-center py-8 text-sm text-gray-500">
<p>"刷新"</p>
</div>
) : evalSummary.summary?.length > 0 ? (
<div className="space-y-2">
{evalSummary.summary.map((s: any, i: number) => (
<div
key={i}
className="flex flex-col sm:flex-row sm:items-center justify-between rounded-lg border border-gray-800 p-3 gap-2"
>
<span className="text-sm text-gray-300">{s.provider}/{s.model}</span>
<Badge status="info">
{s.accuracy?.toFixed(1)}% ({s.correct}/{s.total})
</Badge>
</div>
))}
</div>
) : (
<div className="text-center py-8 text-sm text-gray-500">
<p></p>
</div>
)}
</CardBody>
</Card>
</div>
</div>
</div>
)
}
-193
View File
@@ -1,193 +0,0 @@
/**
* Admin 后台 - 数据采集页面
*
* 响应式布局: 移动端单列,桌面端双列
* 触摸友好: 按钮最小 44px 高度
*/
import { useEffect, useState, useCallback } from 'react'
import { triggerCollection, fetchLeagues } from '../dal'
import type { CollectionRequest, League } from '../types'
import { Card, CardBody, CardHeader, Badge, SectionHeader } from '../components'
const SOURCES = [
{ value: 'bzzoiro', label: 'Bzzoiro', desc: '历史赛程与比分' },
{ value: 'understat', label: 'Understat', desc: 'xG 进阶数据' },
{ value: 'injuries', label: 'Injuries', desc: '球员伤停' },
] as const
export default function CollectionPage() {
const [leagues, setLeagues] = useState<League[]>([])
const [source, setSource] = useState<string>('bzzoiro')
const [leagueCode, setLeagueCode] = useState('')
const [dateFrom, setDateFrom] = useState('')
const [dateTo, setDateTo] = useState('')
const [season, setSeason] = useState('')
const [loading, setLoading] = useState(false)
const [error, setError] = useState<string | null>(null)
const [successMsg, setSuccessMsg] = useState<string | null>(null)
const loadLeagues = useCallback(async () => {
const lg = await fetchLeagues()
setLeagues(lg)
}, [])
useEffect(() => { loadLeagues() }, [loadLeagues])
async function handleSubmit(e: React.FormEvent) {
e.preventDefault()
setError(null)
setSuccessMsg(null)
setLoading(true)
try {
const body: CollectionRequest = {
source: source as CollectionRequest['source'],
leagues: leagueCode ? [leagueCode] : undefined,
league: leagueCode || undefined,
season: season || undefined,
date_from: dateFrom || undefined,
date_to: dateTo || undefined,
}
const res = await triggerCollection(body)
setSuccessMsg(`采集完成: ${JSON.stringify(res).slice(0, 200)}`)
} catch (err: unknown) {
setError(err instanceof Error ? err.message : '采集触发失败')
} finally {
setLoading(false)
}
}
return (
<div className="space-y-6">
<SectionHeader
title="数据采集"
description="触发数据源采集,支持联赛筛选和日期范围"
/>
<div className="grid gap-6 lg:grid-cols-2">
{/* 采集表单 */}
<Card>
<CardHeader title="新建采集任务" />
<CardBody>
<form onSubmit={handleSubmit} className="space-y-4">
{/* 数据源选择 */}
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<select
value={source}
onChange={e => setSource(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
>
{SOURCES.map(s => (
<option key={s.value} value={s.value}>
{s.label} {s.desc}
</option>
))}
</select>
</div>
{/* 联赛选择 */}
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<select
value={leagueCode}
onChange={e => setLeagueCode(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
>
<option value=""></option>
{leagues.map(l => (
<option key={l.code} value={l.code}>{l.name_zh || l.name}</option>
))}
</select>
</div>
{/* Understat 专用: 赛季 */}
{source === 'understat' && (
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400">()</label>
<input
type="number"
value={season}
onChange={e => setSeason(e.target.value)}
placeholder="2025"
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
)}
{/* 日期范围 */}
{source !== 'injuries' && (
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<input
type="date"
value={dateFrom}
onChange={e => setDateFrom(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<input
type="date"
value={dateTo}
onChange={e => setDateTo(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
/>
</div>
</div>
)}
{/* 消息提示 */}
{error && (
<div className="rounded-md bg-red-500/10 p-3 text-sm text-red-400 border border-red-500/30">
{error}
</div>
)}
{successMsg && (
<div className="rounded-md bg-emerald-500/10 p-3 text-sm text-emerald-400 border border-emerald-500/30">
{successMsg}
</div>
)}
{/* 提交按钮 */}
<button
type="submit"
disabled={loading}
className="w-full rounded-md bg-blue-600 px-4 py-2.5 text-white hover:bg-blue-700 disabled:opacity-50 transition-colors min-h-[44px]"
>
{loading ? '采集中...' : '触发采集'}
</button>
</form>
</CardBody>
</Card>
{/* 数据源说明 */}
<Card>
<CardHeader title="数据源说明" />
<CardBody>
<div className="space-y-3">
{SOURCES.map(s => (
<div key={s.value} className="rounded-lg border border-gray-800 p-4">
<div className="flex items-center gap-2 mb-2">
<Badge status="info">{s.label}</Badge>
</div>
<p className="text-sm text-gray-400">{s.desc}</p>
</div>
))}
</div>
{/* 移动端提示 */}
<div className="mt-4 rounded-lg border border-blue-500/30 bg-blue-500/5 p-3">
<p className="text-xs text-blue-300">
💡 ,,
</p>
</div>
</CardBody>
</Card>
</div>
</div>
)
}
-189
View File
@@ -1,189 +0,0 @@
/**
* Admin 后台 - 系统配置管理页面
*
* 功能:
* - 显示当前 .env 配置(脱敏)
* - 提供配置修改指南
* - 快速导航到数据源和 LLM 配置
*/
import { useEffect, useState, useCallback } from 'react'
import { fetchSystemConfig } from '../dal'
import { Card, CardBody, CardHeader, Badge, SectionHeader } from '../components'
export default function ConfigPage() {
const [config, setConfig] = useState<any[]>([])
const [loading, setLoading] = useState(true)
const loadConfig = useCallback(async () => {
setLoading(true)
try {
const data = await fetchSystemConfig()
setConfig(data)
} catch {
setConfig([])
} finally {
setLoading(false)
}
}, [])
useEffect(() => { loadConfig() }, [loadConfig])
return (
<div className="space-y-6">
<SectionHeader
title="系统配置"
description="查看当前系统配置参数(通过 .env 文件管理)"
/>
{/* 配置警告 */}
<div className="rounded-lg border border-yellow-500/30 bg-yellow-500/10 p-4">
<div className="flex items-start gap-3">
<span className="text-yellow-400 text-lg"></span>
<div>
<p className="text-sm font-medium text-yellow-300"></p>
<p className="mt-1 text-xs text-yellow-300/80">
(API Key) <code className="rounded bg-gray-800/50 px-1">.env</code>
, SSH
</p>
</div>
</div>
</div>
{/* 快速导航 */}
<div className="grid gap-4 sm:grid-cols-2">
<a
href="/admin/data-sources"
className="flex items-center gap-3 rounded-lg border border-gray-800 bg-gray-900 p-4 transition-colors hover:border-gray-700 hover:bg-gray-800/50"
>
<div className="flex h-10 w-10 items-center justify-center rounded-lg bg-blue-500/10 text-blue-400 text-lg">
</div>
<div>
<div className="text-sm font-medium text-gray-200"></div>
<div className="text-xs text-gray-500"> API Key</div>
</div>
</a>
<a
href="/admin/llm-config"
className="flex items-center gap-3 rounded-lg border border-gray-800 bg-gray-900 p-4 transition-colors hover:border-gray-700 hover:bg-gray-800/50"
>
<div className="flex h-10 w-10 items-center justify-center rounded-lg bg-emerald-500/10 text-emerald-400 text-lg">
</div>
<div>
<div className="text-sm font-medium text-gray-200">LLM </div>
<div className="text-xs text-gray-500"></div>
</div>
</a>
</div>
{/* 配置列表 */}
<Card>
<CardHeader
title="当前配置"
action={
<button
onClick={loadConfig}
className="rounded px-2 py-1 text-xs text-blue-400 hover:bg-gray-800 min-h-[44px] min-w-[44px]"
>
</button>
}
/>
<CardBody>
{loading ? (
<div className="space-y-3">
{[1, 2, 3, 4, 5].map(i => (
<div key={i} className="h-10 animate-pulse rounded bg-gray-800" />
))}
</div>
) : config.length > 0 ? (
<div className="space-y-1">
{/* 桌面端表头 */}
<div className="hidden sm:grid sm:grid-cols-3 gap-4 border-b border-gray-800 px-3 py-2 text-xs text-gray-500">
<span></span>
<span></span>
<span></span>
</div>
{/* 配置行 */}
{config.map(item => (
<div
key={item.key}
className="flex flex-col sm:grid sm:grid-cols-3 gap-2 sm:gap-4 border-b border-gray-800/50 px-3 py-3 hover:bg-gray-800/30 rounded-lg sm:rounded-none"
>
<div className="flex items-center gap-2">
<span className="font-mono text-xs text-gray-300">{item.key}</span>
{item.is_sensitive && (
<Badge status="warning"></Badge>
)}
</div>
<div className="font-mono text-xs text-gray-400 break-all">
{item.value_masked}
</div>
<div className="text-xs text-gray-500">
{item.description}
</div>
</div>
))}
</div>
) : (
<div className="text-center py-8 text-sm text-gray-500"></div>
)}
</CardBody>
</Card>
{/* 修改指南 */}
<Card>
<CardHeader title="修改配置指南" />
<CardBody className="space-y-4">
<div className="rounded-lg border border-gray-800 p-4">
<h4 className="text-sm font-medium text-gray-200 mb-2"> SSH .env</h4>
<pre className="overflow-x-auto rounded bg-gray-950 p-3 text-xs text-green-400 leading-relaxed">
{`# 连接到 NAS
ssh user@your-nas-ip
# 进入项目目录
cd /vol2/1000/Docker/Profeto
# 编辑 .env 文件
nano .env
# 修改后重启后端服务
docker compose restart api
# 查看日志确认生效
docker compose logs -f api`}
</pre>
</div>
<div className="rounded-lg border border-gray-800 p-4">
<h4 className="text-sm font-medium text-gray-200 mb-2"></h4>
<div className="space-y-2 text-xs">
<div className="flex items-start gap-2">
<code className="flex-shrink-0 rounded bg-gray-800 px-1.5 py-0.5 text-blue-400">LLM_API_KEY</code>
<span className="text-gray-400">LLM API ,</span>
</div>
<div className="flex items-start gap-2">
<code className="flex-shrink-0 rounded bg-gray-800 px-1.5 py-0.5 text-blue-400">LLM_MODEL</code>
<span className="text-gray-400">使, gpt-4oclaude-3-5-sonnet</span>
</div>
<div className="flex items-start gap-2">
<code className="flex-shrink-0 rounded bg-gray-800 px-1.5 py-0.5 text-blue-400">LLM_BASE_URL</code>
<span className="text-gray-400">API , OpenAI </span>
</div>
<div className="flex items-start gap-2">
<code className="flex-shrink-0 rounded bg-gray-800 px-1.5 py-0.5 text-blue-400">BZZOIRO_KEY</code>
<span className="text-gray-400">Bzzoiro API </span>
</div>
<div className="flex items-start gap-2">
<code className="flex-shrink-0 rounded bg-gray-800 px-1.5 py-0.5 text-blue-400">DATABASE_URL</code>
<span className="text-gray-400">PostgreSQL </span>
</div>
</div>
</div>
</CardBody>
</Card>
</div>
)
}
-142
View File
@@ -1,142 +0,0 @@
/**
* Admin 后台 - 仪表盘
*
* 响应式布局: 移动端 1 列 → 平板 2 列 → 桌面 4 列
*/
import { useEffect, useState } from 'react'
import { fetchDashboard, fetchHealth } from '../dal'
import type { DashboardStats } from '../types'
import { Card, CardBody, CardHeader, StatCard, Badge } from '../components'
export default function Dashboard() {
const [data, setData] = useState<DashboardStats | null>(null)
const [health, setHealth] = useState<any>(null)
const [loading, setLoading] = useState(true)
const [error, setError] = useState<string | null>(null)
useEffect(() => {
let active = true
setLoading(true)
Promise.all([fetchDashboard(), fetchHealth()])
.then(([stats, h]) => {
if (active) {
setData(stats)
setHealth(h)
}
})
.catch((err: unknown) => {
if (active) setError(err instanceof Error ? err.message : '加载失败')
})
.finally(() => {
if (active) setLoading(false)
})
return () => { active = false }
}, [])
if (error) {
return (
<div className="rounded-lg border border-red-500/30 bg-red-500/10 p-6 text-center text-red-400">
<p className="text-lg font-medium"></p>
<p className="mt-1 text-sm">{error}</p>
<button
onClick={() => window.location.reload()}
className="mt-3 rounded-md bg-red-500/20 px-4 py-2 text-sm hover:bg-red-500/30 min-h-[44px]"
>
</button>
</div>
)
}
return (
<div className="space-y-6">
{/* 统计卡片: 移动端 1 列 → 平板 2 列 → 桌面 4 列 */}
<div className="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4">
<StatCard
label="健康状态"
value={loading ? '—' : (health?.status === 'healthy' ? '✅ 正常' : '⚠️ 异常')}
icon="●"
/>
<StatCard
label="联赛数"
value={loading ? '—' : data?.leagues.length ?? 0}
icon="◫"
/>
<StatCard
label="比赛数"
value={loading ? '—' : data?.total_matches ?? 0}
icon="◆"
/>
<StatCard
label="预测数"
value={loading ? '—' : data?.total_predictions ?? 0}
icon="◇"
/>
</div>
{/* 联赛列表 */}
<Card>
<CardHeader title="最近联赛" />
<CardBody>
{loading ? (
<div className="flex flex-wrap gap-2">
{[1, 2, 3, 4].map(i => (
<div key={i} className="h-6 w-20 animate-pulse rounded bg-gray-800" />
))}
</div>
) : data && data.leagues.length > 0 ? (
<div className="flex flex-wrap gap-2">
{data.leagues.map(l => (
<Badge key={l.code} status="info">
{l.name_zh || l.name} ({l.code})
</Badge>
))}
</div>
) : (
<p className="text-gray-400">,</p>
)}
</CardBody>
</Card>
{/* 快捷操作 */}
<Card>
<CardHeader title="快捷操作" />
<CardBody>
<div className="grid gap-3 sm:grid-cols-2 lg:grid-cols-3">
<a
href="/admin/collection"
className="flex items-center gap-3 rounded-lg border border-gray-800 p-4 transition-colors hover:border-gray-700 hover:bg-gray-800/50"
>
<span className="text-xl"></span>
<div>
<div className="text-sm font-medium text-gray-200"></div>
<div className="text-xs text-gray-500"></div>
</div>
</a>
<a
href="/admin/predictions"
className="flex items-center gap-3 rounded-lg border border-gray-800 p-4 transition-colors hover:border-gray-700 hover:bg-gray-800/50"
>
<span className="text-xl"></span>
<div>
<div className="text-sm font-medium text-gray-200"></div>
<div className="text-xs text-gray-500">使 LLM </div>
</div>
</a>
<a
href="/admin/backtest"
className="flex items-center gap-3 rounded-lg border border-gray-800 p-4 transition-colors hover:border-gray-700 hover:bg-gray-800/50"
>
<span className="text-xl"></span>
<div>
<div className="text-sm font-medium text-gray-200"></div>
<div className="text-xs text-gray-500"></div>
</div>
</a>
</div>
</CardBody>
</Card>
</div>
)
}
-174
View File
@@ -1,174 +0,0 @@
/**
* Admin 后台 - 数据源管理页面
*
* 功能:
* - 显示当前数据源状态 (bzzoiro / understat / injuries)
* - 显示 API Key 配置状态(脱敏显示)
* - 测试连接按钮(调用采集 API 测试)
* - 采集历史记录
*/
import { useEffect, useState, useCallback } from 'react'
import { fetchDataSourceStatuses, testDataSource } from '../dal'
import type { DataSourceStatus } from '../types'
import { Card, CardBody, CardHeader, Badge, SectionHeader } from '../components'
export default function DataSourcesPage() {
const [sources, setSources] = useState<DataSourceStatus[]>([])
const [loading, setLoading] = useState(true)
const [testingSource, setTestingSource] = useState<string | null>(null)
const [testResults, setTestResults] = useState<Record<string, { success: boolean; message: string }>>({})
const loadSources = useCallback(async () => {
setLoading(true)
try {
const data = await fetchDataSourceStatuses()
setSources(data)
} catch {
setSources([])
} finally {
setLoading(false)
}
}, [])
useEffect(() => { loadSources() }, [loadSources])
async function handleTest(sourceName: string) {
setTestingSource(sourceName)
setTestResults(prev => ({ ...prev, [sourceName]: { success: false, message: '测试中...' } }))
try {
await testDataSource(sourceName as any)
setTestResults(prev => ({ ...prev, [sourceName]: { success: true, message: '连接成功' } }))
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : '连接失败'
setTestResults(prev => ({ ...prev, [sourceName]: { success: false, message: msg } }))
} finally {
setTestingSource(null)
}
}
return (
<div className="space-y-6">
<SectionHeader
title="数据源管理"
description="管理数据采集源配置与连接状态"
/>
{/* 数据源卡片 */}
{loading ? (
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
{[1, 2, 3].map(i => (
<Card key={i}>
<CardBody>
<div className="animate-pulse space-y-3">
<div className="h-4 w-24 rounded bg-gray-800" />
<div className="h-3 w-32 rounded bg-gray-800" />
<div className="h-8 w-full rounded bg-gray-800" />
</div>
</CardBody>
</Card>
))}
</div>
) : (
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
{sources.map(source => {
const result = testResults[source.name]
return (
<Card key={source.name}>
<CardBody className="space-y-4">
{/* 头部 */}
<div className="flex items-center justify-between">
<div className="flex items-center gap-2">
<h3 className="text-sm font-medium text-white">{source.label}</h3>
<Badge status={source.keyConfigured ? 'success' : 'failed'}>
{source.keyConfigured ? '已配置' : '未配置'}
</Badge>
</div>
</div>
{/* API Key 状态 */}
<div className="space-y-2">
<div className="flex items-center justify-between text-xs">
<span className="text-gray-500">API Key</span>
<span className="font-mono text-gray-400">{source.maskedKey}</span>
</div>
<div className="flex items-center justify-between text-xs">
<span className="text-gray-500"></span>
<span className="text-gray-400">{source.lastIngestion || '暂无记录'}</span>
</div>
</div>
{/* 测试结果 */}
{result && (
<div
className={`rounded p-2 text-xs ${
result.success
? 'bg-emerald-500/10 text-emerald-400 border border-emerald-500/30'
: 'bg-red-500/10 text-red-400 border border-red-500/30'
}`}
>
{result.message}
</div>
)}
{/* 操作按钮 */}
<button
onClick={() => handleTest(source.name)}
disabled={testingSource === source.name}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-4 py-2.5 text-sm text-gray-300 transition-colors hover:bg-gray-700 hover:text-white disabled:opacity-50 min-h-[44px]"
>
{testingSource === source.name ? '测试中...' : '测试连接'}
</button>
</CardBody>
</Card>
)
})}
</div>
)}
{/* 数据源说明 */}
<Card>
<CardHeader title="数据源说明" />
<CardBody>
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
<div className="rounded-lg border border-gray-800 p-4">
<div className="flex items-center gap-2 mb-2">
<Badge status="info">Bzzoiro</Badge>
</div>
<p className="text-xs text-gray-400">
, API Key
</p>
</div>
<div className="rounded-lg border border-gray-800 p-4">
<div className="flex items-center gap-2 mb-2">
<Badge status="info">Understat</Badge>
</div>
<p className="text-xs text-gray-400">
xG () , API Key,
</p>
</div>
<div className="rounded-lg border border-gray-800 p-4">
<div className="flex items-center gap-2 mb-2">
<Badge status="info">Injuries</Badge>
</div>
<p className="text-xs text-gray-400">
, API Key
</p>
</div>
</div>
</CardBody>
</Card>
{/* 采集历史 */}
<Card>
<CardHeader title="采集历史" />
<CardBody>
<div className="text-center py-8 text-sm text-gray-500">
<p></p>
<p className="mt-1 text-xs">,</p>
</div>
</CardBody>
</Card>
</div>
)
}
-259
View File
@@ -1,259 +0,0 @@
/**
* Admin 后台 - LLM 配置管理页面
*
* 功能:
* - 显示当前 LLM 配置(provider, model, base_url
* - 测试 LLM 连接(调用 /predict 测试)
* - 显示 LLM 使用统计(预测次数、平均延迟、成功率)
* - 模型切换(显示可用模型列表)
*/
import { useEffect, useState, useCallback } from 'react'
import { testLLMConnection, fetchLLMUsageStats } from '../dal'
import type { LLMUsageStats } from '../types'
import { Card, CardBody, CardHeader, Badge, SectionHeader } from '../components'
// 可用模型列表
const AVAILABLE_MODELS = [
{ id: 'gpt-4o', label: 'GPT-4o', provider: 'openai', description: '最强大模型,适合复杂分析' },
{ id: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'openai', description: '快速经济,适合批量预测' },
{ id: 'claude-3-5-sonnet', label: 'Claude 3.5 Sonnet', provider: 'anthropic', description: '长上下文分析能力强' },
{ id: 'deepseek-chat', label: 'DeepSeek V3', provider: 'deepseek', description: '高性价比中文优化' },
]
export default function LLMConfigPage() {
const [stats, setStats] = useState<LLMUsageStats | null>(null)
const [loading, setLoading] = useState(true)
const [testing, setTesting] = useState(false)
const [testResult, setTestResult] = useState<{ success: boolean; message: string } | null>(null)
// 当前配置(模拟,后端暂无配置端点)
const currentConfig = {
provider: 'openai',
model: 'gpt-4o',
base_url: 'https://api.openai.com/v1',
api_key_configured: true,
api_key_masked: 'sk-****...****abcd',
}
const loadStats = useCallback(async () => {
setLoading(true)
try {
const data = await fetchLLMUsageStats()
setStats(data)
} catch {
setStats(null)
} finally {
setLoading(false)
}
}, [])
useEffect(() => { loadStats() }, [loadStats])
async function handleTest() {
setTesting(true)
setTestResult(null)
try {
await testLLMConnection()
setTestResult({ success: true, message: 'LLM 连接测试成功' })
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : 'LLM 连接测试失败'
setTestResult({ success: false, message: msg })
} finally {
setTesting(false)
}
}
return (
<div className="space-y-6">
<SectionHeader
title="LLM 配置"
description="管理大语言模型连接与使用统计"
/>
<div className="grid gap-6 lg:grid-cols-2">
{/* 当前配置 */}
<Card>
<CardHeader title="当前配置" />
<CardBody className="space-y-4">
<div className="space-y-3">
<div className="flex items-center justify-between border-b border-gray-800 pb-3">
<span className="text-xs text-gray-500"></span>
<Badge status="info">{currentConfig.provider}</Badge>
</div>
<div className="flex items-center justify-between border-b border-gray-800 pb-3">
<span className="text-xs text-gray-500"></span>
<span className="text-sm text-gray-200">{currentConfig.model}</span>
</div>
<div className="flex items-center justify-between border-b border-gray-800 pb-3">
<span className="text-xs text-gray-500">API </span>
<span className="text-xs font-mono text-gray-400">{currentConfig.base_url}</span>
</div>
<div className="flex items-center justify-between border-b border-gray-800 pb-3">
<span className="text-xs text-gray-500">API Key</span>
<div className="flex items-center gap-2">
<span className="font-mono text-xs text-gray-400">{currentConfig.api_key_masked}</span>
<Badge status={currentConfig.api_key_configured ? 'success' : 'failed'}>
{currentConfig.api_key_configured ? '已配置' : '未配置'}
</Badge>
</div>
</div>
<div className="flex items-center justify-between">
<span className="text-xs text-gray-500"></span>
<span className="text-sm text-gray-200"> Agent (5 + )</span>
</div>
</div>
{/* 测试连接 */}
{testResult && (
<div
className={`rounded p-3 text-xs ${
testResult.success
? 'bg-emerald-500/10 text-emerald-400 border border-emerald-500/30'
: 'bg-red-500/10 text-red-400 border border-red-500/30'
}`}
>
{testResult.message}
</div>
)}
<button
onClick={handleTest}
disabled={testing}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-4 py-2.5 text-sm text-gray-300 transition-colors hover:bg-gray-700 hover:text-white disabled:opacity-50 min-h-[44px]"
>
{testing ? '测试中...' : '测试 LLM 连接'}
</button>
</CardBody>
</Card>
{/* 使用统计 */}
<Card>
<CardHeader
title="使用统计"
action={
<button
onClick={loadStats}
className="rounded px-2 py-1 text-xs text-blue-400 hover:bg-gray-800 min-h-[44px] min-w-[44px]"
>
</button>
}
/>
<CardBody>
{loading ? (
<div className="animate-pulse space-y-3">
<div className="h-16 rounded bg-gray-800" />
<div className="h-16 rounded bg-gray-800" />
<div className="h-16 rounded bg-gray-800" />
</div>
) : stats ? (
<div className="space-y-4">
<div className="grid grid-cols-2 gap-3">
<div className="rounded-lg border border-gray-800 bg-gray-800/50 p-3 text-center">
<div className="text-xl font-bold text-white tabular-nums">{stats.total_predictions}</div>
<div className="text-xs text-gray-500 mt-1"></div>
</div>
<div className="rounded-lg border border-gray-800 bg-gray-800/50 p-3 text-center">
<div className="text-xl font-bold text-blue-400 tabular-nums">{stats.avg_latency_ms}ms</div>
<div className="text-xs text-gray-500 mt-1"></div>
</div>
</div>
<div className="rounded-lg border border-gray-800 bg-gray-800/50 p-3 text-center">
<div className="text-xl font-bold text-emerald-400 tabular-nums">
{stats.success_rate.toFixed(1)}%
</div>
<div className="text-xs text-gray-500 mt-1"></div>
</div>
</div>
) : (
<div className="text-center py-8 text-sm text-gray-500">使</div>
)}
</CardBody>
</Card>
</div>
{/* 模型切换 */}
<Card>
<CardHeader title="可用模型" description="切换预测使用的 LLM 模型(通过修改 .env 文件)" />
<CardBody>
<div className="space-y-3">
{AVAILABLE_MODELS.map(model => {
const isCurrent = model.id === currentConfig.model
return (
<div
key={model.id}
className={`flex flex-col sm:flex-row sm:items-center justify-between rounded-lg border p-4 gap-3 ${
isCurrent
? 'border-blue-500/30 bg-blue-500/5'
: 'border-gray-800 hover:border-gray-700'
}`}
>
<div className="flex items-center gap-3">
<div className="flex-1">
<div className="flex items-center gap-2">
<span className="text-sm font-medium text-gray-200">{model.label}</span>
{isCurrent && <Badge status="success"></Badge>}
</div>
<p className="text-xs text-gray-500 mt-0.5">{model.description}</p>
</div>
</div>
<div className="flex items-center gap-2">
<Badge status="info">{model.provider}</Badge>
{!isCurrent && (
<span className="text-xs text-gray-500 whitespace-nowrap">
.env
</span>
)}
</div>
</div>
)
})}
</div>
</CardBody>
</Card>
{/* 最近预测 */}
<Card>
<CardHeader title="最近预测记录" />
<CardBody>
{loading ? (
<div className="space-y-2">
{[1, 2, 3].map(i => (
<div key={i} className="h-12 animate-pulse rounded bg-gray-800" />
))}
</div>
) : stats && stats.recent_predictions.length > 0 ? (
<div className="space-y-2">
{stats.recent_predictions.map(p => (
<div
key={p.id}
className="flex flex-col sm:flex-row sm:items-center justify-between rounded-lg border border-gray-800 p-3 gap-2"
>
<div className="flex items-center gap-3">
<span className="text-xs text-gray-500">#{p.id}</span>
<span className="text-sm text-gray-300">Match #{p.match_id}</span>
<span className="text-xs font-mono text-gray-500">{p.model}</span>
</div>
<div className="flex items-center gap-2">
<span className="text-xs text-gray-500">
{p.created_at ? new Date(p.created_at).toLocaleString() : '—'}
</span>
<Badge status={p.status === 'success' ? 'success' : 'failed'}>
{p.status === 'success' ? '成功' : '失败'}
</Badge>
</div>
</div>
))}
</div>
) : (
<div className="text-center py-8 text-sm text-gray-500">
<p></p>
<p className="mt-1 text-xs"></p>
</div>
)}
</CardBody>
</Card>
</div>
)
}
-105
View File
@@ -1,105 +0,0 @@
/**
* Admin 后台 - 监控面板
*
* 响应式布局: 移动端单列,桌面端三列
* 触摸友好: 卡片可点击查看详情
*/
import { useEffect, useState } from 'react'
import { fetchHealth } from '../dal'
import { SectionHeader } from '../components'
export default function MonitoringPage() {
const [health, setHealth] = useState<any>(null)
const [loading, setLoading] = useState(true)
useEffect(() => {
fetchHealth().then(setHealth).finally(() => setLoading(false))
}, [])
return (
<div className="space-y-6">
<SectionHeader
title="系统监控"
description="查看服务健康状态"
/>
{loading ? (
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
{[1, 2, 3].map(i => (
<div key={i} className="h-24 animate-pulse rounded-lg bg-gray-800" />
))}
</div>
) : health ? (
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3">
{/* 服务状态 */}
<div className="rounded-lg border border-gray-800 bg-gray-900 p-5">
<div className="text-xs text-gray-500 mb-2"></div>
<div className="flex items-center gap-2">
<span className="inline-block h-3 w-3 rounded-full bg-emerald-500 animate-pulse" />
<span className="text-lg font-bold text-emerald-400">{health.status}</span>
</div>
</div>
{/* 服务名称 */}
<div className="rounded-lg border border-gray-800 bg-gray-900 p-5">
<div className="text-xs text-gray-500 mb-2"></div>
<div className="text-lg font-bold text-white">{health.service || 'profeto'}</div>
</div>
{/* 数据库 */}
<div className="rounded-lg border border-gray-800 bg-gray-900 p-5">
<div className="text-xs text-gray-500 mb-2"></div>
<div className="text-lg font-bold text-blue-400">11 </div>
</div>
{/* 版本 */}
{health.version && (
<div className="rounded-lg border border-gray-800 bg-gray-900 p-5">
<div className="text-xs text-gray-500 mb-2"></div>
<div className="text-lg font-bold text-gray-200">{health.version}</div>
</div>
)}
{/* 运行时间 */}
{health.uptime_seconds && (
<div className="rounded-lg border border-gray-800 bg-gray-900 p-5">
<div className="text-xs text-gray-500 mb-2"></div>
<div className="text-lg font-bold text-gray-200">
{Math.floor(health.uptime_seconds / 3600)}h {Math.floor((health.uptime_seconds % 3600) / 60)}m
</div>
</div>
)}
{/* 检查项 */}
{health.checks && Object.keys(health.checks).length > 0 && (
<div className="rounded-lg border border-gray-800 bg-gray-900 p-5">
<div className="text-xs text-gray-500 mb-2"></div>
<div className="space-y-1">
{Object.entries(health.checks).map(([key, val]) => (
<div key={key} className="flex items-center justify-between text-sm">
<span className="text-gray-400 text-xs">{key}</span>
<span className={`text-xs font-medium ${val === 'pass' ? 'text-emerald-400' : 'text-red-400'}`}>
{String(val)}
</span>
</div>
))}
</div>
</div>
)}
</div>
) : (
<div className="rounded-lg border border-red-500/30 bg-red-500/10 p-6 text-center">
<p className="text-red-400 text-lg font-medium"></p>
<p className="text-red-400/70 text-sm mt-1"></p>
<button
onClick={() => window.location.reload()}
className="mt-4 rounded-md bg-red-500/20 px-4 py-2 text-sm text-red-300 hover:bg-red-500/30 min-h-[44px]"
>
</button>
</div>
)}
</div>
)
}
-138
View File
@@ -1,138 +0,0 @@
/**
* Admin 后台 - 预测管理页面
*
* 响应式布局: 移动端单列,桌面端双列
* 触摸友好: 按钮最小 44px 高度
*/
import { useEffect, useState } from 'react'
import { triggerPrediction, fetchPredictions, fetchMatches } from '../dal'
import type { Match, Prediction } from '../types'
import { Card, CardBody, CardHeader, Badge, SectionHeader } from '../components'
export default function PredictionsPage() {
const [matches, setMatches] = useState<Match[]>([])
const [predictions, setPredictions] = useState<Prediction[]>([])
const [matchId, setMatchId] = useState('')
const [mode, setMode] = useState<'single' | 'multi'>('multi')
const [loading, setLoading] = useState(false)
const [error, setError] = useState<string | null>(null)
const [successMsg, setSuccessMsg] = useState<string | null>(null)
useEffect(() => {
fetchPredictions(20).then(setPredictions)
fetchMatches({ status: 'scheduled', limit: 20 }).then(d => setMatches(d.items))
}, [])
async function handlePredict(e: React.FormEvent) {
e.preventDefault()
if (!matchId) return
setLoading(true)
setError(null)
setSuccessMsg(null)
try {
await triggerPrediction({ match_id: parseInt(matchId), mode })
setSuccessMsg('预测任务已提交')
fetchPredictions(20).then(setPredictions)
} catch (err: unknown) {
setError(err instanceof Error ? err.message : '预测失败')
} finally {
setLoading(false)
}
}
return (
<div className="space-y-6">
<SectionHeader
title="预测管理"
description="触发 LLM 足球预测"
/>
<div className="grid gap-6 lg:grid-cols-2">
{/* 新建预测 */}
<Card>
<CardHeader title="新建预测" />
<CardBody>
<form onSubmit={handlePredict} className="space-y-4">
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<select
value={matchId}
onChange={e => setMatchId(e.target.value)}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
>
<option value=""></option>
{matches.map(m => (
<option key={m.id} value={m.id}>
{m.home_team} vs {m.away_team} ({m.match_date?.slice(0, 10)})
</option>
))}
</select>
</div>
<div>
<label className="mb-1.5 block text-xs font-medium text-gray-400"></label>
<select
value={mode}
onChange={e => setMode(e.target.value as 'single' | 'multi')}
className="w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2.5 text-white min-h-[44px]"
>
<option value="multi"> Agent (5 + )</option>
<option value="single"></option>
</select>
</div>
{error && (
<div className="rounded-md bg-red-500/10 p-3 text-sm text-red-400 border border-red-500/30">
{error}
</div>
)}
{successMsg && (
<div className="rounded-md bg-emerald-500/10 p-3 text-sm text-emerald-400 border border-emerald-500/30">
{successMsg}
</div>
)}
<button
type="submit"
disabled={loading || !matchId}
className="w-full rounded-md bg-blue-600 px-4 py-2.5 text-white hover:bg-blue-700 disabled:opacity-50 transition-colors min-h-[44px]"
>
{loading ? '预测中...' : '触发预测'}
</button>
</form>
</CardBody>
</Card>
{/* 最近预测 */}
<Card>
<CardHeader title="最近预测" />
<CardBody>
{predictions.length === 0 ? (
<div className="text-center py-8 text-sm text-gray-500">
<p></p>
<p className="mt-1 text-xs"></p>
</div>
) : (
<div className="space-y-2">
{predictions.slice(0, 10).map(p => (
<div
key={p.id}
className="flex flex-col sm:flex-row sm:items-center justify-between rounded-lg border border-gray-800 p-3 gap-2"
>
<span className="text-sm text-gray-300">
Match #{p.match_id} · {p.model}
</span>
<Badge status={p.settled ? 'success' : 'warning'}>
{p.pred_1x2 || '?'} · {p.subjective_confidence ? `${(p.subjective_confidence * 100).toFixed(0)}%` : '—'}
</Badge>
</div>
))}
</div>
)}
</CardBody>
</Card>
</div>
</div>
)
}
-37
View File
@@ -1,37 +0,0 @@
/**
* Admin 后台 - 路由配置
*
* 所有 Admin 页面的路由定义,使用嵌套路由。
* 挂载路径: /admin/*
*/
import { Navigate } from 'react-router-dom'
import AdminLayout from './AdminLayout'
import Dashboard from './pages/Dashboard'
import CollectionPage from './pages/Collection'
import PredictionsPage from './pages/Predictions'
import BacktestPage from './pages/Backtest'
import MonitoringPage from './pages/Monitoring'
import DataSourcesPage from './pages/DataSources'
import LLMConfigPage from './pages/LLMConfig'
import ConfigPage from './pages/Config'
export const adminRoutes = [
{
path: '/admin',
element: <AdminLayout />,
children: [
{ index: true, element: <Dashboard /> },
{ path: 'collection', element: <CollectionPage /> },
{ path: 'predictions', element: <PredictionsPage /> },
{ path: 'backtest', element: <BacktestPage /> },
{ path: 'monitoring', element: <MonitoringPage /> },
{ path: 'data-sources', element: <DataSourcesPage /> },
{ path: 'llm-config', element: <LLMConfigPage /> },
{ path: 'config', element: <ConfigPage /> },
{ path: '*', element: <Navigate to="/admin" replace /> },
],
},
]
export { adminRoutes as default }
-184
View File
@@ -1,184 +0,0 @@
/**
* Admin 后台 - TypeScript 类型定义
*
* 与 FastAPI 后端 Pydantic 模型对齐
*/
// ── 系统健康 ────────────────────────────────────────────────────
export interface HealthStatus {
status: 'ok' | 'degraded' | 'error'
version?: string
uptime_seconds?: number
checks: Record<string, 'pass' | 'fail' | 'warn'>
}
// ── 仪表盘 ──────────────────────────────────────────────────────
export interface DashboardStats {
leagues: League[]
total_matches: number
total_predictions: number
health: string
db_tables: { name: string; row_count: number; size_mb: number; last_updated: string | null }[]
last_collection: { source: string; league_code: string | null; started_at: string; finished_at: string | null; status: string; records_count: number | null; error_message: string | null }[]
recent_errors: { id: number; timestamp: string; source: string; message: string; level: string }[]
}
// ── 联赛 & 比赛 ─────────────────────────────────────────────────
export interface League {
id?: number
code: string
name: string
name_zh?: string
country?: string
}
export interface Match {
id: number
league_code?: string
season?: string | null
home_team: string
away_team: string
home_team_zh?: string | null
away_team_zh?: string | null
match_date: string
match_status: string
home_goals?: number | null
away_goals?: number | null
match_stage?: string | null
}
// ── 预测 ────────────────────────────────────────────────────────
export interface Prediction {
id: number
match_id: number
provider: string
model: string
prompt_version?: string
mode?: string
pred_home_goals?: number | null
pred_away_goals?: number | null
pred_1x2?: string | null
subjective_confidence?: number | null
reasoning?: string | null
created_at: string
actual_home_goals?: number | null
actual_away_goals?: number | null
settled?: boolean
}
export interface PredictRequest {
match_id: number
mode?: 'single' | 'multi'
provider?: string
model?: string
}
// ── 数据采集 ────────────────────────────────────────────────────
export interface CollectionRequest {
source: 'bzzoiro' | 'understat' | 'injuries'
leagues?: string[]
league?: string
season?: string
date_from?: string
date_to?: string
}
// ── 评估 & 回测 ─────────────────────────────────────────────────
export interface EvalSummary {
summary: Array<{
provider: string
model: string
total: number
correct: number
accuracy: number
avg_rmse: number
}>
}
export interface BacktestRequest {
league_id?: number
date_from?: string
date_to?: string
mode?: 'single' | 'multi'
limit?: number
model?: string
}
export interface BacktestSummary {
total: number
scored: number
accuracy_1x2?: number
avg_score_rmse?: number
results?: Array<{
match_id: number
actual_home: number
actual_away: number
pred_home?: number
pred_away?: number
correct_1x2: boolean
}>
}
// ── 数据源配置 ──────────────────────────────────────────────────
export interface DataSourceStatus {
name: string
label: string
keyConfigured: boolean
maskedKey: string
lastIngestion: string | null
status: 'configured' | 'missing_key' | 'untested'
}
export interface DataSourceTestRequest {
source: 'bzzoiro' | 'understat' | 'injuries'
}
export interface IngestionHistoryEntry {
id: string
source: string
started_at: string
finished_at: string | null
status: 'success' | 'running' | 'failed'
records_count: number | null
error_message: string | null
}
// ── LLM 配置 ────────────────────────────────────────────────────
export interface LLMConfig {
provider: string
model: string
base_url: string
api_key_configured: boolean
api_key_masked: string
}
export interface LLMUsageStats {
total_predictions: number
avg_latency_ms: number
success_rate: number
recent_predictions: Array<{
id: number
match_id: number
model: string
created_at: string
latency_ms?: number
status: 'success' | 'failed'
}>
}
// ── 系统配置 ────────────────────────────────────────────────────
export interface SystemConfigEntry {
key: string
value_masked: string
description: string
is_sensitive: boolean
}
-51
View File
@@ -100,54 +100,3 @@
@apply animate-pulse rounded-none bg-ink-200; @apply animate-pulse rounded-none bg-ink-200;
} }
} }
/* ── Admin 后台:暗色主题覆盖 ── */
@layer base {
/* Admin 区域内滚动条暗色化 */
.dark-scroll::-webkit-scrollbar-thumb {
@apply bg-gray-600;
}
.dark-scroll::-webkit-scrollbar-thumb:hover {
@apply bg-gray-500;
}
}
@layer components {
/* Admin 输入框统一样式 */
.admin-input {
@apply w-full rounded-md border border-gray-700 bg-gray-800 px-3 py-2 text-sm text-gray-200
transition-colors placeholder:text-gray-600
focus:border-blue-500 focus:outline-none focus:ring-1 focus:ring-blue-500;
}
/* Admin 按钮 */
.admin-btn {
@apply inline-flex items-center justify-center gap-1.5 rounded-md border border-gray-700
bg-gray-800 px-4 py-2 text-sm font-medium text-gray-200
transition-colors hover:border-gray-600 hover:bg-gray-700
disabled:cursor-not-allowed disabled:opacity-40;
}
.admin-btn-primary {
@apply border-blue-600 bg-blue-600 text-white hover:bg-blue-700 hover:border-blue-700;
}
.admin-btn-danger {
@apply border-red-600 bg-red-600 text-white hover:bg-red-700 hover:border-red-700;
}
/* Admin 卡片通用 */
.admin-card {
@apply rounded-lg border border-gray-800 bg-gray-900;
}
/* 表格行动态 */
.admin-row-hover {
@apply transition-colors hover:bg-gray-800/40;
}
/* 状态点 */
.status-dot {
@apply inline-block h-2 w-2 rounded-full;
}
}
+3
View File
@@ -16,6 +16,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
from src.db.base import init_db from src.db.base import init_db
from src.core.http_client import close_client from src.core.http_client import close_client
await init_db() # 验证连接,不建表 await init_db() # 验证连接,不建表
# P2-5: 启动时执行一次 ingest failure 重试清理
from src.data.retry_worker import run_retry_worker
await run_retry_worker()
yield yield
await close_client() await close_client()
+233 -6
View File
@@ -2,6 +2,12 @@
迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。 迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。
使用 Repository 模式进行数据访问,不直接控制事务。 使用 Repository 模式进行数据访问,不直接控制事务。
改进(P1):
- Bronze 层集成:采集后先存 RawEvent,再规范化
- 死信表集成:采集/规范化失败写 IngestFailure
- 令牌桶限流:替换固定 REQUEST_INTERVAL sleep
- 数据血缘:记录 ETL 全过程
""" """
from __future__ import annotations from __future__ import annotations
@@ -9,6 +15,7 @@ import asyncio
import json as _json import json as _json
import logging import logging
import random import random
import uuid
from collections.abc import Iterable from collections.abc import Iterable
from datetime import datetime, timezone from datetime import datetime, timezone
@@ -18,11 +25,26 @@ from src.core.config import settings
from src.core.http_client import get_client from src.core.http_client import get_client
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
from src.data.normalize import normalize_bzzoiro from src.data.normalize import normalize_bzzoiro
from src.data.rate_limiter import RateLimitedClient, TokenBucket
from src.data.sources import register from src.data.sources import register
from src.db.models import League, Match, MatchStats, Team from src.db.models import DataLineage, DataQualityCheck, IngestFailure, League, Match, MatchStats, RawEvent, Team
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# 令牌桶限流器:替代固定 sleep,rate 根据 REQUEST_INTERVAL 计算
_bzzoiro_limiter = TokenBucket(rate=1.0 / REQUEST_INTERVAL, capacity=3)
# P2-3: RateLimitedClient 包装,在 HTTP 调用层面透明限流
_bzzoiro_rl_client: RateLimitedClient | None = None
def _get_bzzoiro_rl_client() -> RateLimitedClient:
"""惰性初始化 RateLimitedClient(避免模块导入时创建 client)。"""
global _bzzoiro_rl_client
if _bzzoiro_rl_client is None:
client = get_client()
_bzzoiro_rl_client = RateLimitedClient(client, _bzzoiro_limiter)
return _bzzoiro_rl_client
def _to_date(value): def _to_date(value):
"""把 datetime / date / str 统一成 `date`。""" """把 datetime / date / str 统一成 `date`。"""
@@ -60,8 +82,9 @@ async def _fetch_json_async(path: str, params: dict | None = None, max_retries:
last_exc: Exception | None = None last_exc: Exception | None = None
for attempt in range(max_retries): for attempt in range(max_retries):
try: try:
client = get_client() # P2-3: 使用 RateLimitedClient 包装 HTTP 调用,透明限流
resp = await client.get(url, headers=headers, params=params, timeout=30) rl_client = _get_bzzoiro_rl_client()
resp = await rl_client.get(url, headers=headers, params=params, timeout=30)
resp.raise_for_status() resp.raise_for_status()
return resp.json() return resp.json()
except Exception as e: except Exception as e:
@@ -124,10 +147,141 @@ async def fetch_bzzoiro_events(
break break
if len(batch) < limit: if len(batch) < limit:
break break
await asyncio.sleep(REQUEST_INTERVAL) # 令牌桶已在 _fetch_json_async 内部处理,这里不再需要固定 sleep
return rows return rows
async def _write_bronze_events(db, raw_events: list[dict], batch_id: str) -> None:
"""将原始事件写入 Bronze 层(RawEvent 表)。
P2-1: 写入血缘记录,追踪 Bronze 层摄取过程。
"""
for raw in raw_events:
try:
source_record_id = str(raw.get("id", ""))
if not source_record_id:
# P1-fix: 不再静默跳过,记录到死信表
logger.warning("bzzoiro raw event missing id: %s", str(raw)[:200])
await _write_ingest_failure(
db, "bzzoiro", "match", "",
"normalize_error", "raw event missing id", raw,
)
continue
bronze = RawEvent(
source_system="bzzoiro",
source_record_id=source_record_id,
raw_payload=raw,
ingest_batch_id=batch_id,
)
db.add(bronze)
except Exception as e:
logger.warning("bzzoiro raw_event write failed: %s", e)
try:
await db.flush()
except Exception as e:
logger.warning("bzzoiro raw_events flush failed: %s", e)
# P2-1: 写入 Bronze 层血缘记录
for raw in raw_events:
source_record_id = str(raw.get("id", ""))
if not source_record_id:
continue
await _write_data_lineage(
db,
source_system="bzzoiro",
source_record_id=source_record_id,
target_table="raw_events",
transform_name="bronze_ingest",
batch_id=batch_id,
)
async def _write_ingest_failure(
db,
source_system: str,
entity_type: str,
source_record_id: str,
error_type: str,
error_detail: str,
raw_payload: dict | None,
) -> None:
"""写入采集失败到死信表(IngestFailure)。"""
try:
from datetime import timedelta
failure = IngestFailure(
source_system=source_system,
entity_type=entity_type,
source_record_id=source_record_id,
error_type=error_type,
error_detail=error_detail[:2000] if error_detail else None,
raw_payload=raw_payload,
status="pending",
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
)
db.add(failure)
await db.flush()
except Exception:
logger.warning("bzzoiro ingest_failure write failed", exc_info=True)
async def _write_data_lineage(
db,
*,
source_system: str,
source_record_id: str,
target_table: str,
target_id: int | None = None,
transform_name: str,
transform_detail: str | None = None,
batch_id: str | None = None,
) -> None:
"""写入数据血缘记录(DataLineage 表)。"""
try:
lineage = DataLineage(
source_system=source_system,
source_record_id=source_record_id,
target_table=target_table,
target_id=target_id,
transform_name=transform_name,
transform_detail=transform_detail,
batch_id=batch_id,
)
db.add(lineage)
await db.flush()
except Exception:
logger.warning("bzzoiro data_lineage write failed", exc_info=True)
async def _write_data_quality_check(
db,
*,
check_name: str,
entity_type: str,
entity_id: str | None = None,
expected_value: str | None = None,
actual_value: str | None = None,
passed: bool,
severity: str = "warning",
detail: str | None = None,
) -> None:
"""写入数据质量检查结果(DataQualityCheck 表)。"""
try:
check = DataQualityCheck(
check_name=check_name,
entity_type=entity_type,
entity_id=entity_id,
expected_value=expected_value,
actual_value=actual_value,
passed=passed,
severity=severity,
detail=detail,
)
db.add(check)
await db.flush()
except Exception:
logger.warning("bzzoiro data_quality_check write failed", exc_info=True)
@register @register
class BzzoiroSource: class BzzoiroSource:
"""bzzoiro 数据源(实现 DataSource 协议)。""" """bzzoiro 数据源(实现 DataSource 协议)。"""
@@ -148,6 +302,7 @@ class BzzoiroSource:
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。 注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
""" """
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []} result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
batch_id = uuid.uuid4().hex[:16]
for code in leagues: for code in leagues:
league_r: dict = {"inserted": 0, "updated": 0, "errors": []} league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
@@ -156,9 +311,17 @@ class BzzoiroSource:
except Exception as e: except Exception as e:
logger.exception("bzzoiro fetch failed for %s", code) logger.exception("bzzoiro fetch failed for %s", code)
league_r["errors"].append(f"fetch failed: {e}") league_r["errors"].append(f"fetch failed: {e}")
# 写死信表
await _write_ingest_failure(
db, "bzzoiro", "match", f"fetch_{code}_{batch_id}",
"fetch_error", str(e), {"league": code, "date_from": date_from, "date_to": date_to},
)
result["leagues"][code] = league_r result["leagues"][code] = league_r
continue continue
# === Bronze 层:先存 RawEvent ===
await _write_bronze_events(db, raw_events, batch_id)
# 获取或创建联赛 # 获取或创建联赛
stmt = select(League).where(League.code == code) stmt = select(League).where(League.code == code)
league = (await db.execute(stmt)).scalar_one_or_none() league = (await db.execute(stmt)).scalar_one_or_none()
@@ -185,6 +348,11 @@ class BzzoiroSource:
except Exception as e: except Exception as e:
# P1-3: 统一使用 warning,不追加到 errors(仅运行时错误入 errors) # P1-3: 统一使用 warning,不追加到 errors(仅运行时错误入 errors)
logger.warning("normalize skip: %s", e) logger.warning("normalize skip: %s", e)
# 写死信表:规范化失败
await _write_ingest_failure(
db, "bzzoiro", "match", str(raw.get("id", "")),
"normalize_error", str(e), raw,
)
continue continue
normalized_matches.append((nm, raw)) normalized_matches.append((nm, raw))
all_team_names.add(nm.home_team) all_team_names.add(nm.home_team)
@@ -255,6 +423,16 @@ class BzzoiroSource:
db.add(m) db.add(m)
await db.flush() await db.flush()
existing_matches[match_key] = m # 防止同批重复 existing_matches[match_key] = m # 防止同批重复
# P2-1: Silver 层血缘 — Match 创建
await _write_data_lineage(
db,
source_system="bzzoiro",
source_record_id=str(raw.get("id", "")),
target_table="matches",
target_id=m.id,
transform_name="silver_upsert",
batch_id=batch_id,
)
if nm.home_xg is not None or nm.away_xg is not None: if nm.home_xg is not None or nm.away_xg is not None:
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
stats = MatchStats( stats = MatchStats(
@@ -273,11 +451,22 @@ class BzzoiroSource:
home_red_cards=nm.home_red_cards, home_red_cards=nm.home_red_cards,
away_red_cards=nm.away_red_cards, away_red_cards=nm.away_red_cards,
source="bzzoiro", source="bzzoiro",
source_event_id=str(raw.get("id", "")), source_record_id=str(raw.get("id", "")),
retrieved_at=now, retrieved_at=now,
available_at=now, available_at=now,
) )
db.add(stats) db.add(stats)
await db.flush()
# P2-1: Silver 层血缘 — MatchStats 创建
await _write_data_lineage(
db,
source_system="bzzoiro",
source_record_id=str(raw.get("id", "")),
target_table="match_stats",
target_id=m.id,
transform_name="silver_upsert",
batch_id=batch_id,
)
league_r["inserted"] += 1 league_r["inserted"] += 1
else: else:
# 已有比赛: 直接从内存获取对象更新(无需再查询) # 已有比赛: 直接从内存获取对象更新(无需再查询)
@@ -298,13 +487,36 @@ class BzzoiroSource:
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
existing_match.stats = MatchStats( existing_match.stats = MatchStats(
match_id=existing_match.id, match_id=existing_match.id,
home_xg=nm.home_xg,
away_xg=nm.away_xg,
home_shots=nm.home_shots,
away_shots=nm.away_shots,
home_shots_on_target=nm.home_shots_on_target,
away_shots_on_target=nm.away_shots_on_target,
home_corners=nm.home_corners,
away_corners=nm.away_corners,
home_possession=nm.home_possession,
home_yellow_cards=nm.home_yellow_cards,
away_yellow_cards=nm.away_yellow_cards,
home_red_cards=nm.home_red_cards,
away_red_cards=nm.away_red_cards,
source="bzzoiro", source="bzzoiro",
source_event_id=str(raw.get("id", "")), source_record_id=str(raw.get("id", "")),
retrieved_at=now, retrieved_at=now,
available_at=now, available_at=now,
) )
db.add(existing_match.stats) db.add(existing_match.stats)
await db.flush() await db.flush()
# P2-1: Silver 层血缘 — MatchStats 创建(更新路径)
await _write_data_lineage(
db,
source_system="bzzoiro",
source_record_id=str(raw.get("id", "")),
target_table="match_stats",
target_id=existing_match.id,
transform_name="silver_upsert",
batch_id=batch_id,
)
if existing_match.stats is not None: if existing_match.stats is not None:
for fld in ("home_xg", "away_xg", "home_shots", "away_shots", for fld in ("home_xg", "away_xg", "home_shots", "away_shots",
"home_shots_on_target", "away_shots_on_target", "home_shots_on_target", "away_shots_on_target",
@@ -323,4 +535,19 @@ class BzzoiroSource:
result["leagues"][code] = league_r result["leagues"][code] = league_r
result["total_inserted"] += league_r["inserted"] result["total_inserted"] += league_r["inserted"]
result["total_updated"] += league_r["updated"] result["total_updated"] += league_r["updated"]
# P2-2: 数据质量检查 — 行计数合理性(某联赛比赛数不应为 0)
total_records = league_r["inserted"] + league_r["updated"]
await _write_data_quality_check(
db,
check_name="bzzoiro_row_count",
entity_type="match",
entity_id=code,
expected_value=">=1",
actual_value=str(total_records),
passed=total_records > 0,
severity="critical" if total_records == 0 else "info",
detail=f"league={code} inserted={league_r['inserted']} updated={league_r['updated']}",
)
return result return result
+238 -10
View File
@@ -1,16 +1,25 @@
"""伤停数据采集器(api-football / api-sports.io)。 """伤停数据采集器(api-football / api-sports.io)。
采集伤停数据并入库(injuries 表),供 injuries agent 使用。 采集伤停数据并入库(injuries 表),供 injuries agent 使用。
改进(P0):
- 原子写缓存(tempfile + os.replace)
- TTL 分级:未来比赛 1h,当天 5min,历史 7day
- 文件锁防止并发写缓存冲突
- Bronze 层集成(RawEvent)
- 死信表集成(IngestFailure)
""" """
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
import json import json
import logging import logging
import os
import random import random
import tempfile import tempfile
import time import time
from datetime import datetime, timezone import uuid
from datetime import datetime, timedelta, timezone
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
@@ -18,6 +27,7 @@ import httpx
from src.core.config import settings from src.core.config import settings
from src.core.http_client import get_client from src.core.http_client import get_client
from src.db.models import DataQualityCheck
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -27,6 +37,138 @@ DEFAULT_HOST = "v3.football.api-sports.io"
# P2-3: 缓存目录改用系统临时目录,避免源码树内写入 # P2-3: 缓存目录改用系统临时目录,避免源码树内写入
_CACHE_DIR = Path(tempfile.gettempdir()) / "profeto_injuries" _CACHE_DIR = Path(tempfile.gettempdir()) / "profeto_injuries"
# 模块级锁:防止并发写同一缓存文件
_cache_lock = asyncio.Lock()
def _sanitize_cache_component(value: str | int | None) -> str:
"""消毒缓存键组件,防止路径遍历。
只允许 [A-Za-z0-9_.-],其余字符替换为 '_'
"""
import re
s = str(value) if value is not None else "none"
return re.sub(r'[^A-Za-z0-9_.]', '_', s)[:60]
def _build_cache_key(prefix: str, *components: str | int | None) -> str:
"""构造安全的缓存文件名。"""
parts = [_sanitize_cache_component(c) for c in components]
return f"{prefix}_{'_'.join(parts)}.json"
def _compute_ttl_hours(date_str: str | None, fixture_id: int | None, league_id: int | None) -> float:
"""根据查询参数计算缓存 TTL(小时)。
TTL 分级策略:
- 未来比赛(date > now): 1 小时(赛前伤停变化频繁)
- 当天比赛(date == today): 5 分钟(赛中实时更新)
- 历史比赛(date < now): 7 天(历史数据不变)
- 无日期参数: 1 小时(保守策略)
"""
if date_str is None:
# 无日期参数(按 fixture_id 或 league_id 查询),保守 TTL
return 1.0
try:
query_date = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
now = datetime.now(timezone.utc)
today = now.date()
if query_date.date() > today:
return 1.0 # 未来
elif query_date.date() == today:
return 5.0 / 60 # 当天:5 分钟
else:
return 168.0 # 历史:7 天
except (ValueError, TypeError):
return 1.0 # 解析失败,保守 TTL
def _write_cache_atomic(cache_file: Path, data: Any) -> None:
"""原子写缓存文件。
使用 tempfile + os.replace 实现原子写,防止读到写了一半的文件。
配合模块级 asyncio.Lock,杜绝并发写冲突。
"""
cache_file.parent.mkdir(parents=True, exist_ok=True)
# 在同一目录创建临时文件(保证 os.replace 是原子操作)
fd, tmp_path = tempfile.mkstemp(
dir=str(cache_file.parent),
prefix=f".{cache_file.name}.tmp_",
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
json.dump(data, f, ensure_ascii=False, default=str)
os.replace(tmp_path, cache_file)
except BaseException:
# 失败时清理临时文件
try:
os.unlink(tmp_path)
except OSError:
pass
raise
def _cleanup_expired_cache() -> int:
"""P2-4: 扫描缓存目录,删除过期的 .json 文件。
TTL 策略(保守取最大值,避免误删有效缓存):
- 最短 TTL 为 5 分钟(当天比赛),但清理阈值用 1 小时
- 超过 1 小时的缓存文件视为过期
Returns:
删除的文件数。
"""
import glob as _glob
cache_dir = _CACHE_DIR
if not cache_dir.exists():
return 0
removed = 0
max_ttl_seconds = 3600 # 1 小时(保守阈值,最短实际 TTL 5 分钟)
now_ts = time.time()
for cache_file in _glob.glob(str(cache_dir / "*.json")):
try:
file_age = now_ts - os.path.getmtime(cache_file)
if file_age > max_ttl_seconds:
os.unlink(cache_file)
removed += 1
except OSError:
continue
if removed > 0:
logger.info("injuries cache cleanup: removed %d expired files", removed)
return removed
async def _write_data_quality_check(
db,
*,
check_name: str,
entity_type: str,
entity_id: str | None = None,
expected_value: str | None = None,
actual_value: str | None = None,
passed: bool,
severity: str = "warning",
detail: str | None = None,
) -> None:
"""写入数据质量检查结果(DataQualityCheck 表)。"""
try:
check = DataQualityCheck(
check_name=check_name,
entity_type=entity_type,
entity_id=entity_id,
expected_value=expected_value,
actual_value=actual_value,
passed=passed,
severity=severity,
detail=detail,
)
db.add(check)
await db.flush()
except Exception:
logger.warning("injuries data_quality_check write failed", exc_info=True)
async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = None, league_id: int | None = None) -> list[dict]: async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = None, league_id: int | None = None) -> list[dict]:
"""采集伤停数据。 """采集伤停数据。
@@ -43,20 +185,25 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
if not api_key: if not api_key:
raise RuntimeError("API_FOOTBALL_KEY 未设置") raise RuntimeError("API_FOOTBALL_KEY 未设置")
# P2-4: 清理过期缓存文件
_cleanup_expired_cache()
cache_dir = _CACHE_DIR cache_dir = _CACHE_DIR
cache_dir.mkdir(parents=True, exist_ok=True) cache_dir.mkdir(parents=True, exist_ok=True)
# 缓存命中 (7 天内有效) # TTL 分级缓存
cache_key = f"injuries_{date}_{fixture_id}_{league_id}.json" ttl_hours = _compute_ttl_hours(date, fixture_id, league_id)
# P0-fix: 消毒缓存键,防止路径遍历(date 参数可能含 '../' 等)
cache_key = _build_cache_key("injuries", date, fixture_id, league_id)
cache_file = cache_dir / cache_key cache_file = cache_dir / cache_key
if cache_file.exists(): if cache_file.exists():
age_hours = (time.time() - cache_file.stat().st_mtime) / 3600 age_hours = (time.time() - cache_file.stat().st_mtime) / 3600
if age_hours < 168: # 7 天 if age_hours < ttl_hours:
logger.debug("injuries cache hit: %s (%.1fh old)", cache_key, age_hours) logger.debug("injuries cache hit: %s (%.1fh old, ttl=%.2fh)", cache_key, age_hours, ttl_hours)
with open(cache_file, encoding="utf-8") as f: with open(cache_file, encoding="utf-8") as f:
return json.load(f) return json.load(f)
else: else:
logger.debug("injuries cache expired: %s (%.1fh old)", cache_key, age_hours) logger.debug("injuries cache expired: %s (%.1fh old, ttl=%.2fh)", cache_key, age_hours, ttl_hours)
headers = { headers = {
"x-apisports-key": api_key, "x-apisports-key": api_key,
@@ -93,9 +240,9 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
data = resp.json() data = resp.json()
injuries = data.get("response", []) injuries = data.get("response", [])
# 写缓存 # 原子写缓存 + 文件锁(防止并发写冲突)
with open(cache_file, "w", encoding="utf-8") as f: async with _cache_lock:
json.dump(injuries, ensure_ascii=False, default=str, fp=f) _write_cache_atomic(cache_file, injuries)
return injuries return injuries
@@ -106,25 +253,64 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。 注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
P1-4: 批量幂等检查,避免逐条查询的竞态条件(并发采集时 IntegrityError)。 P1-4: 批量幂等检查,避免逐条查询的竞态条件(并发采集时 IntegrityError)。
P1 Bronze: 采集成功后先存 RawEvent,再规范化。
P1 死信: 采集/规范化失败时写 IngestFailure。
""" """
from sqlalchemy import select from sqlalchemy import select
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import selectinload from sqlalchemy.orm import selectinload
from src.data.team_names import normalize as normalize_name from src.data.team_names import normalize as normalize_name
from src.db.models import Injury, Team from src.db.models import DataQualityCheck, Injury, IngestFailure, RawEvent, Team
result = {"count": 0, "inserted": 0, "errors": []} result = {"count": 0, "inserted": 0, "errors": []}
batch_id = uuid.uuid4().hex[:16]
try: try:
raw_injuries = await fetch_injuries(date=date) raw_injuries = await fetch_injuries(date=date)
except Exception as e: except Exception as e:
logger.exception("injuries fetch failed") logger.exception("injuries fetch failed")
result["errors"].append(f"fetch failed: {e}") result["errors"].append(f"fetch failed: {e}")
# 写死信表(IngestFailure)
try:
failure = IngestFailure(
source_system="api-football",
entity_type="injury",
source_record_id=f"fetch_{batch_id}",
error_type="fetch_error",
error_detail=str(e)[:2000],
raw_payload={"date": date},
status="pending",
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
)
db.add(failure)
await db.flush()
except Exception:
logger.warning("failed to write fetch error to ingest_failures", exc_info=True)
return result return result
result["count"] = len(raw_injuries) result["count"] = len(raw_injuries)
# === Bronze 层:写 RawEvent ===
for raw in raw_injuries:
try:
player = raw.get("player", {}) or {}
fixture = raw.get("fixture", {}) or {}
source_record_id = f"inj_{player.get('id')}_{fixture.get('id')}"
bronze = RawEvent(
source_system="api-football",
source_record_id=source_record_id,
raw_payload=raw,
ingest_batch_id=batch_id,
)
db.add(bronze)
except Exception as e:
logger.warning("raw_event write failed for injury: %s", e)
try:
await db.flush()
except Exception as e:
logger.warning("raw_events flush failed: %s", e)
# 预加载所有球队(用于按名匹配) # 预加载所有球队(用于按名匹配)
teams = (await db.execute(select(Team))).scalars().all() teams = (await db.execute(select(Team))).scalars().all()
team_by_name = {t.name: t.id for t in teams} team_by_name = {t.name: t.id for t in teams}
@@ -132,6 +318,7 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
# P1-4: 收集所有待插入记录的键,批量查询已存在的记录 # P1-4: 收集所有待插入记录的键,批量查询已存在的记录
# 避免逐条查询 + 插入的竞态条件(两个并发请求同时通过检查 → IntegrityError) # 避免逐条查询 + 插入的竞态条件(两个并发请求同时通过检查 → IntegrityError)
pending_records: list[dict] = [] pending_records: list[dict] = []
parse_failures: list[dict] = [] # 规范化失败的原始数据,用于写死信表
for raw in raw_injuries: for raw in raw_injuries:
try: try:
player = raw.get("player", {}) or {} player = raw.get("player", {}) or {}
@@ -176,6 +363,31 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
}) })
except Exception as e: except Exception as e:
result["errors"].append(f"parse error: {e}") result["errors"].append(f"parse error: {e}")
parse_failures.append({"raw": raw, "error": str(e)})
# 写规范化失败到死信表
for fail in parse_failures:
try:
raw = fail["raw"]
player = raw.get("player", {}) or {}
failure = IngestFailure(
source_system="api-football",
entity_type="injury",
source_record_id=f"inj_{player.get('id', 'unknown')}",
error_type="normalize_error",
error_detail=fail["error"][:2000],
raw_payload=raw,
status="pending",
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
)
db.add(failure)
except Exception:
logger.warning("failed to write parse error to ingest_failures", exc_info=True)
if parse_failures:
try:
await db.flush()
except Exception:
logger.warning("ingest_failures flush failed", exc_info=True)
# P1-4: 批量查询已存在的记录(1 次 DB 往返) # P1-4: 批量查询已存在的记录(1 次 DB 往返)
existing_keys: set[tuple] = set() existing_keys: set[tuple] = set()
@@ -223,6 +435,22 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
logger.warning("injuries final flush IntegrityError, falling back to per-record insert") logger.warning("injuries final flush IntegrityError, falling back to per-record insert")
return await _ingest_injuries_fallback(db, pending_records, result) return await _ingest_injuries_fallback(db, pending_records, result)
# P2-2: 数据质量检查 — 行计数 / 新增比例
total = result["count"]
inserted = result["inserted"]
# 某日期伤停数不应为 0(除非历史日期)
await _write_data_quality_check(
db,
check_name="injuries_row_count",
entity_type="injury",
entity_id=date or "unknown",
expected_value=">=0",
actual_value=str(total),
passed=True,
severity="info",
detail=f"date={date} fetched={total} inserted={inserted}",
)
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务 # 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date) logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
return result return result
+98
View File
@@ -0,0 +1,98 @@
"""令牌桶限流器。
替代 bzzoiro 中固定的 REQUEST_INTERVAL sleep,提供更精细的速率控制
支持突发流量和平滑限流
"""
from __future__ import annotations
import asyncio
import logging
import time
logger = logging.getLogger(__name__)
class TokenBucket:
"""异步令牌桶限流器。
用法:
limiter = TokenBucket(rate=5.0, capacity=10)
await limiter.acquire() # 等待直到有可用令牌
Args:
rate: 每秒补充的令牌数
capacity: 桶容量(允许的最大突发量)
"""
def __init__(self, rate: float, capacity: int | None = None) -> None:
if rate <= 0:
raise ValueError(f"rate must be positive, got {rate}")
self._rate = rate
self._capacity = capacity or max(1, int(rate * 2))
self._tokens: float = self._capacity
self._last_refill = time.monotonic()
self._lock = asyncio.Lock()
async def acquire(self, tokens: int = 1) -> None:
"""获取指定数量的令牌,不足时等待。
Args:
tokens: 需要获取的令牌数
"""
if tokens <= 0:
return
if tokens > self._capacity:
raise ValueError(f"requested {tokens} exceeds capacity {self._capacity}")
while True:
async with self._lock:
self._refill()
if self._tokens >= tokens:
self._tokens -= tokens
return
# 计算需要等待的时间
wait_time = (tokens - self._tokens) / self._rate
logger.debug("token bucket: waiting %.2fs for %d tokens", wait_time, tokens)
await asyncio.sleep(wait_time)
def _refill(self) -> None:
"""补充令牌(基于经过的时间)。"""
now = time.monotonic()
elapsed = now - self._last_refill
if elapsed > 0:
self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
self._last_refill = now
@property
def tokens(self) -> float:
"""当前可用令牌数(近似)。"""
self._refill()
return self._tokens
class RateLimitedClient:
"""HTTP 客户端限流包装器。
httpx 客户端之上添加令牌桶限流,透明地控制请求速率
用法:
client = get_client()
limiter = TokenBucket(rate=5.0)
wrapper = RateLimitedClient(client, limiter)
resp = await wrapper.get(url)
"""
def __init__(self, client, limiter: TokenBucket) -> None:
self._client = client
self._limiter = limiter
async def get(self, url: str, **kwargs):
"""限流的 GET 请求。"""
await self._limiter.acquire()
return await self._client.get(url, **kwargs)
async def post(self, url: str, **kwargs):
"""限流的 POST 请求。"""
await self._limiter.acquire()
return await self._client.post(url, **kwargs)
+126
View File
@@ -0,0 +1,126 @@
"""P2-5: IngestFailure 重试 Worker。
查询死信表中 status='pending' AND next_retry_at <= now() 的记录,
根据 source_system error_type 决定重试策略
作为 FastAPI startup 事件注册,在应用启动时执行一次清理
"""
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from sqlalchemy import select
from src.db.base import AsyncSessionLocal
logger = logging.getLogger(__name__)
# 最大重试次数上限
_MAX_RETRIES = 5
async def _retry_ingest_failures(db) -> dict:
"""重试待处理的采集失败记录。
查询条件: status='pending' AND next_retry_at <= now()
重试策略:
- retry_count >= MAX_RETRIES 标记 abandoned
- error_type=fetch_error 退避重试(更新 next_retry_at)
- error_type=normalize_error 规范化失败通常是数据问题,退避重试
- error_type=db_error 数据库问题,退避重试
Args:
db: SQLAlchemy async session
Returns:
统计 dict: {"retried": int, "abandoned": int, "errors": list[str]}
"""
result = {"retried": 0, "abandoned": 0, "errors": []}
from src.db.models import IngestFailure
now = datetime.now(timezone.utc)
# 查询待重试记录
stmt = (
select(IngestFailure)
.where(IngestFailure.status == "pending")
.where(IngestFailure.next_retry_at <= now)
.order_by(IngestFailure.next_retry_at)
.limit(50) # 每批最多处理 50 条,避免长时间持有事务
)
try:
rows = (await db.execute(stmt)).scalars().all()
except Exception as e:
logger.warning("retry_worker: query failed: %s", e)
result["errors"].append(f"query failed: {e}")
return result
if not rows:
return result
logger.info("retry_worker: found %d pending failures to retry", len(rows))
for failure in rows:
try:
# 超过最大重试次数 → 放弃
if failure.retry_count >= _MAX_RETRIES:
failure.status = "abandoned"
failure.resolved_at = now
await db.flush()
result["abandoned"] += 1
logger.info(
"retry_worker: abandoned %s/%s after %d retries",
failure.source_system, failure.source_record_id, failure.retry_count,
)
continue
# 退避计算: 5min * 2^retry_count,最大 24 小时
backoff_minutes = min(5 * (2 ** failure.retry_count), 1440)
next_retry = now + timedelta(minutes=backoff_minutes)
# 更新重试状态
failure.retry_count += 1
failure.next_retry_at = next_retry
failure.status = "retrying"
await db.flush()
result["retried"] += 1
logger.info(
"retry_worker: scheduled retry %d/%d for %s/%s (next: %s)",
failure.retry_count, _MAX_RETRIES,
failure.source_system, failure.source_record_id,
next_retry.isoformat(),
)
except Exception as e:
await db.rollback()
error_msg = f"retry {failure.source_system}/{failure.source_record_id}: {e}"
result["errors"].append(error_msg)
logger.warning("retry_worker: %s", error_msg)
return result
async def run_retry_worker() -> None:
"""启动入口: 获取 DB session 并执行重试逻辑。
设计为幂等操作 多次运行不会产生副作用( next_retry_at 约束)
"""
logger.info("retry_worker: starting ingest failure retry sweep")
try:
async with AsyncSessionLocal() as db:
try:
stats = await _retry_ingest_failures(db)
await db.commit()
logger.info(
"retry_worker: sweep complete — retried=%d abandoned=%d errors=%d",
stats["retried"], stats["abandoned"], len(stats["errors"]),
)
except Exception as e:
await db.rollback()
logger.warning("retry_worker: session failed: %s", e)
except Exception as e:
logger.warning("retry_worker: failed to get DB session: %s", e)
+214 -11
View File
@@ -2,6 +2,12 @@
迁移自旧项目 app/data/sources/understat.py,改成 async 迁移自旧项目 app/data/sources/understat.py,改成 async
使用 Repository 模式进行数据访问,不直接控制事务 使用 Repository 模式进行数据访问,不直接控制事务
改进(P1):
- Bronze 层集成:采集后先存 RawEvent,再规范化
- 死信表集成:采集/规范化失败写 IngestFailure
- xG 覆盖更新: understat 数据更新时覆盖旧值(全量覆盖模式)
- xG 追踪字段:xg_source / xg_updated_at / xg_source_record_id
""" """
from __future__ import annotations from __future__ import annotations
@@ -10,6 +16,7 @@ import json
import logging import logging
import random import random
import re import re
import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from sqlalchemy import select from sqlalchemy import select
@@ -19,7 +26,7 @@ from src.core.http_client import get_client
from src.data.config import FDCO_TO_UNDERSTAT, LEAGUE_NAMES from src.data.config import FDCO_TO_UNDERSTAT, LEAGUE_NAMES
from src.data.normalize import normalize_understat from src.data.normalize import normalize_understat
from src.data.sources import register from src.data.sources import register
from src.db.models import League, Match, MatchStats, Team from src.db.models import DataLineage, DataQualityCheck, IngestFailure, League, Match, MatchStats, RawEvent, Team
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -47,6 +54,7 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
"Referer": f"https://understat.com/league/{understat_league}/{season}", "Referer": f"https://understat.com/league/{understat_league}/{season}",
} }
# 重试:网络错误 / 5xx / 429 # 重试:网络错误 / 5xx / 429
last_exc: Exception | None = None last_exc: Exception | None = None
for attempt in range(3): for attempt in range(3):
@@ -58,12 +66,10 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
except Exception as e: except Exception as e:
last_exc = e last_exc = e
if attempt == 2: if attempt == 2:
raise raise RuntimeError(f"understat fetch failed: {e}") from e
delay = min(2 ** attempt, 8) + random.uniform(0, 1) delay = min(2 ** attempt, 8) + random.uniform(0, 1)
logger.warning("understat fetch failed, retry %d in %.1fs: %s", attempt + 1, delay, e) logger.warning("understat fetch failed, retry %d in %.1fs: %s", attempt + 1, delay, e)
await asyncio.sleep(delay) await asyncio.sleep(delay)
else:
raise RuntimeError(f"understat fetch failed: {last_exc}")
# understat 返回 JS 对象,需要提取 JSON # understat 返回 JS 对象,需要提取 JSON
text = resp.text text = resp.text
@@ -89,6 +95,136 @@ def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, i
return (home_team_id, away_team_id, match_date.isoformat() if match_date is not None else "") return (home_team_id, away_team_id, match_date.isoformat() if match_date is not None else "")
async def _write_understat_bronze(db, raw_matches: list[dict], batch_id: str) -> None:
"""将 understat 原始数据写入 Bronze 层(RawEvent 表)。
P2-1: 写入血缘记录,追踪 Bronze 层摄取过程
"""
for raw in raw_matches:
try:
source_record_id = str(raw.get("id", ""))
if not source_record_id:
# P1-fix: 不再静默跳过,记录到死信表
logger.warning("understat raw match missing id: %s", str(raw)[:200])
await _write_ingest_failure(
db, "understat", "match", "",
"normalize_error", "raw match missing id", raw,
)
continue
bronze = RawEvent(
source_system="understat",
source_record_id=source_record_id,
raw_payload=raw,
ingest_batch_id=batch_id,
)
db.add(bronze)
except Exception as e:
logger.warning("understat raw_event write failed: %s", e)
try:
await db.flush()
except Exception as e:
logger.warning("understat raw_events flush failed: %s", e)
# P2-1: 写入 Bronze 层血缘记录
for raw in raw_matches:
source_record_id = str(raw.get("id", ""))
if not source_record_id:
continue
await _write_data_lineage(
db,
source_system="understat",
source_record_id=source_record_id,
target_table="raw_events",
transform_name="bronze_ingest",
batch_id=batch_id,
)
async def _write_ingest_failure(
db,
source_system: str,
entity_type: str,
source_record_id: str,
error_type: str,
error_detail: str,
raw_payload: dict | None,
) -> None:
"""写入采集失败到死信表(IngestFailure)。"""
try:
failure = IngestFailure(
source_system=source_system,
entity_type=entity_type,
source_record_id=source_record_id,
error_type=error_type,
error_detail=error_detail[:2000] if error_detail else None,
raw_payload=raw_payload,
status="pending",
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
)
db.add(failure)
await db.flush()
except Exception:
logger.warning("understat ingest_failure write failed", exc_info=True)
async def _write_data_lineage(
db,
*,
source_system: str,
source_record_id: str,
target_table: str,
target_id: int | None = None,
transform_name: str,
transform_detail: str | None = None,
batch_id: str | None = None,
) -> None:
"""写入数据血缘记录(DataLineage 表)。"""
try:
lineage = DataLineage(
source_system=source_system,
source_record_id=source_record_id,
target_table=target_table,
target_id=target_id,
transform_name=transform_name,
transform_detail=transform_detail,
batch_id=batch_id,
)
db.add(lineage)
await db.flush()
except Exception:
logger.warning("understat data_lineage write failed", exc_info=True)
async def _write_data_quality_check(
db,
*,
check_name: str,
entity_type: str,
entity_id: str | None = None,
expected_value: str | None = None,
actual_value: str | None = None,
passed: bool,
severity: str = "warning",
detail: str | None = None,
) -> None:
"""写入数据质量检查结果(DataQualityCheck 表)。"""
try:
check = DataQualityCheck(
check_name=check_name,
entity_type=entity_type,
entity_id=entity_id,
expected_value=expected_value,
actual_value=actual_value,
passed=passed,
severity=severity,
detail=detail,
)
db.add(check)
await db.flush()
except Exception:
logger.warning("understat data_quality_check write failed", exc_info=True)
@register @register
class UnderstatSource: class UnderstatSource:
"""understat xG 数据源(实现 DataSource 协议)。""" """understat xG 数据源(实现 DataSource 协议)。"""
@@ -96,23 +232,33 @@ class UnderstatSource:
name = "understat" name = "understat"
async def ingest(self, db, *, league: str, season: int) -> dict: async def ingest(self, db, *, league: str, season: int) -> dict:
"""采集 understat xG → 回填到现有 Match。只回填 xG 字段,不创建新 Match。 """采集 understat xG → 回填到现有 Match。
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制 注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制
P1-3: 批量查询优化,将单赛季 380 × 3 DB 往返降为 3 次查询 P1-3: 批量查询优化,将单赛季 380 × 3 DB 往返降为 3 次查询
P1-4: xG 覆盖更新模式, understat 数据更新时覆盖旧值
""" """
from src.db.repositories import LeagueRepository, TeamRepository from src.db.repositories import LeagueRepository, TeamRepository
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []} result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
batch_id = uuid.uuid4().hex[:16]
try: try:
raw_matches = await fetch_understat(league, season) raw_matches = await fetch_understat(league, season)
except Exception as e: except Exception as e:
logger.exception("understat fetch failed for %s %s", league, season) logger.exception("understat fetch failed for %s %s", league, season)
result["errors"].append(f"fetch failed: {e}") result["errors"].append(f"fetch failed: {e}")
# 写死信表
await _write_ingest_failure(
db, "understat", "match", f"fetch_{league}_{season}_{batch_id}",
"fetch_error", str(e), {"league": league, "season": season},
)
return result return result
# === Bronze 层:先存 RawEvent ===
await _write_understat_bronze(db, raw_matches, batch_id)
# 使用 Repository # 使用 Repository
league_repo = LeagueRepository(db) league_repo = LeagueRepository(db)
team_repo = TeamRepository(db) team_repo = TeamRepository(db)
@@ -136,6 +282,11 @@ class UnderstatSource:
continue continue
except Exception as e: except Exception as e:
result["errors"].append(f"normalize: {e}") result["errors"].append(f"normalize: {e}")
# 写死信表:规范化失败
await _write_ingest_failure(
db, "understat", "match", str(raw.get("id", "")),
"normalize_error", str(e), raw,
)
continue continue
normalized_matches.append((nm, raw)) normalized_matches.append((nm, raw))
all_team_names.add(nm.home_team) all_team_names.add(nm.home_team)
@@ -167,7 +318,7 @@ class UnderstatSource:
key = _match_key(m.home_team_id, m.away_team_id, m.match_date_date) key = _match_key(m.home_team_id, m.away_team_id, m.match_date_date)
match_dict[key] = m match_dict[key] = m
# === 内存匹配 + 回填 xG === # === 内存匹配 + 回填 xG(覆盖模式) ===
for nm, raw in normalized_matches: for nm, raw in normalized_matches:
home_team_id = team_name_to_id.get(nm.home_team) home_team_id = team_name_to_id.get(nm.home_team)
away_team_id = team_name_to_id.get(nm.away_team) away_team_id = team_name_to_id.get(nm.away_team)
@@ -181,24 +332,76 @@ class UnderstatSource:
result["unmatched"] += 1 result["unmatched"] += 1
continue continue
# 回填 xG
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
source_record_id = str(raw.get("id", ""))
# 创建 stats 记录(如果不存在)
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
existing.stats = MatchStats( existing.stats = MatchStats(
match_id=existing.id, match_id=existing.id,
source="understat", source="understat",
source_event_id=str(raw.get("id", "")), source_record_id=source_record_id,
retrieved_at=now, retrieved_at=now,
available_at=now, available_at=now,
xg_source="understat",
xg_updated_at=now,
xg_source_record_id=source_record_id,
) )
db.add(existing.stats) db.add(existing.stats)
await db.flush() await db.flush()
# P2-1: Silver 层血缘 — MatchStats 创建
await _write_data_lineage(
db,
source_system="understat",
source_record_id=source_record_id,
target_table="match_stats",
target_id=existing.id,
transform_name="silver_upsert",
batch_id=batch_id,
)
# xG 覆盖更新模式:当 understat 数据更新时覆盖旧值
if existing.stats is not None: if existing.stats is not None:
if existing.stats.home_xg is None and nm.home_xg is not None: xg_updated = False
if nm.home_xg is not None:
existing.stats.home_xg = nm.home_xg existing.stats.home_xg = nm.home_xg
existing.stats.xg_source = "understat"
existing.stats.xg_updated_at = now
existing.stats.xg_source_record_id = source_record_id
result["updated"] += 1 result["updated"] += 1
if existing.stats.away_xg is None and nm.away_xg is not None: xg_updated = True
if nm.away_xg is not None:
existing.stats.away_xg = nm.away_xg existing.stats.away_xg = nm.away_xg
existing.stats.xg_source = "understat"
existing.stats.xg_updated_at = now
existing.stats.xg_source_record_id = source_record_id
xg_updated = True
# P2-1: Silver 层血缘 — xG 覆盖更新
if xg_updated:
await _write_data_lineage(
db,
source_system="understat",
source_record_id=source_record_id,
target_table="match_stats",
target_id=existing.id,
transform_name="silver_xg_update",
batch_id=batch_id,
transform_detail=f"home_xg={nm.home_xg} away_xg={nm.away_xg}",
)
# P2-2: 数据质量检查 — 行计数合理性
total_processed = result["updated"] + result["skipped"] + result["unmatched"]
await _write_data_quality_check(
db,
check_name="understat_row_count",
entity_type="match",
entity_id=f"{league}_{season}",
expected_value=">=1",
actual_value=str(total_processed),
passed=total_processed > 0,
severity="critical" if total_processed == 0 else "info",
detail=f"league={league} season={season} updated={result['updated']} skipped={result['skipped']} unmatched={result['unmatched']}",
)
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务 # 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
return result return result
+112 -1
View File
@@ -1,4 +1,4 @@
"""6 张表 ORM: leagues / teams / matches / match_stats / predictions / injuries。""" """ORM 模型: leagues / teams / matches / match_stats / predictions / injuries / raw_events / ingest_failures / data_quality_checks / data_lineage"""
from __future__ import annotations from __future__ import annotations
from datetime import date, datetime, timezone from datetime import date, datetime, timezone
@@ -131,12 +131,18 @@ class MatchStats(Base):
source_record_id: Mapped[str | None] = mapped_column(String(100)) source_record_id: Mapped[str | None] = mapped_column(String(100))
retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# xG 数据追踪:支持 understat 纠正旧 xG 值
xg_source: Mapped[str | None] = mapped_column(String(30)) # 具体 xG 数据源
xg_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # xG 最后更新时间
xg_source_record_id: Mapped[str | None] = mapped_column(String(100)) # xG 对应的源记录 ID
match: Mapped[Match] = relationship(back_populates="stats") match: Mapped[Match] = relationship(back_populates="stats")
__table_args__ = ( __table_args__ = (
# 数据血缘时间过滤查询用(按 available_at 取「赛前已可得」的统计) # 数据血缘时间过滤查询用(按 available_at 取「赛前已可得」的统计)
Index("ix_match_stats_available_at", "available_at"), Index("ix_match_stats_available_at", "available_at"),
# xG 数据源追踪查询用
Index("ix_match_stats_xg_source", "xg_source", "xg_updated_at"),
) )
@@ -216,3 +222,108 @@ class Prediction(Base):
CheckConstraint("mode IN ('single', 'multi')", name="ck_mode_enum"), CheckConstraint("mode IN ('single', 'multi')", name="ck_mode_enum"),
CheckConstraint("status IN ('success', 'failed', 'degraded')", name="ck_status_enum"), CheckConstraint("status IN ('success', 'failed', 'degraded')", name="ck_status_enum"),
) )
# ── Bronze 层:原始事件记录 ──────────────────────────────────────────────
class RawEvent(Base):
"""Bronze 层:不可变的原始采集记录。
每个采集到的原始事件先写入此表,再规范化到 Silver (matches / match_stats)
提供完整的数据血缘回溯能力
"""
__tablename__ = "raw_events"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_system: Mapped[str] = mapped_column(String(30), nullable=False) # bzzoiro / understat / api-football
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False) # 源系统记录 ID
raw_payload: Mapped[dict] = mapped_column(JSONB, nullable=False) # 完整原始 JSON
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, nullable=False)
ingest_batch_id: Mapped[str | None] = mapped_column(String(64)) # 批次 ID,用于关联同次采集
__table_args__ = (
UniqueConstraint("source_system", "source_record_id", name="uq_raw_events_source_record"),
Index("ix_raw_events_batch", "ingest_batch_id"),
Index("ix_raw_events_source_ingested", "source_system", "ingested_at"),
)
# ── 死信表:采集失败记录 ─────────────────────────────────────────────────
class IngestFailure(Base):
"""采集失败死信表。
当采集或规范化失败时,写入此表而非仅内存 dict
支持自动重试和人工排查
"""
__tablename__ = "ingest_failures"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_system: Mapped[str] = mapped_column(String(30), nullable=False)
entity_type: Mapped[str] = mapped_column(String(30), nullable=False) # match / injury / team
source_record_id: Mapped[str | None] = mapped_column(String(100))
error_type: Mapped[str] = mapped_column(String(50), nullable=False) # fetch_error / normalize_error / db_error / validation_error
error_detail: Mapped[str | None] = mapped_column(Text)
raw_payload: Mapped[dict | None] = mapped_column(JSONB) # 失败时的原始数据,用于重试
retry_count: Mapped[int] = mapped_column(Integer, default=0)
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status: Mapped[str] = mapped_column(String(20), default="pending") # pending / retrying / resolved / abandoned
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
__table_args__ = (
Index("ix_ingest_failures_status_next_retry", "status", "next_retry_at"),
Index("ix_ingest_failures_source", "source_system", "entity_type"),
CheckConstraint("status IN ('pending', 'retrying', 'resolved', 'abandoned')", name="ck_ingest_failures_status"),
CheckConstraint("retry_count >= 0", name="ck_ingest_failures_retry_nonneg"),
)
# ── 数据质量检查表 ──────────────────────────────────────────────────────
class DataQualityCheck(Base):
"""数据质量检查结果记录。
每次运行数据质量检查时,将结果写入此表用于趋势分析和告警
"""
__tablename__ = "data_quality_checks"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
check_name: Mapped[str] = mapped_column(String(100), nullable=False) # 检查名称
entity_type: Mapped[str] = mapped_column(String(30), nullable=False) # match / team / prediction
entity_id: Mapped[str | None] = mapped_column(String(50)) # 具体实体 ID
expected_value: Mapped[str | None] = mapped_column(Text) # 期望值(描述)
actual_value: Mapped[str | None] = mapped_column(Text) # 实际值
passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
severity: Mapped[str] = mapped_column(String(10), nullable=False, default="warning") # info / warning / critical
detail: Mapped[str | None] = mapped_column(Text) # 详细描述
checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, nullable=False)
__table_args__ = (
Index("ix_dqc_check_time", "check_name", "checked_at"),
Index("ix_dqc_entity", "entity_type", "entity_id"),
Index("ix_dqc_severity_passed", "severity", "passed"),
CheckConstraint("severity IN ('info', 'warning', 'critical')", name="ck_dqc_severity"),
)
# ── 数据血缘表 ──────────────────────────────────────────────────────────
class DataLineage(Base):
"""ETL 全过程元数据记录。
追踪从 Bronze Silver Gold 的完整转换链路
"""
__tablename__ = "data_lineage"
id: Mapped[int] = mapped_column(Integer, primary_key=True)
source_system: Mapped[str] = mapped_column(String(30), nullable=False)
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
target_table: Mapped[str] = mapped_column(String(50), nullable=False) # matches / match_stats / predictions
target_id: Mapped[int | None] = mapped_column(Integer) # 目标表记录 ID
transform_name: Mapped[str] = mapped_column(String(100), nullable=False) # 转换步骤名称
transform_detail: Mapped[str | None] = mapped_column(Text) # 转换详情
batch_id: Mapped[str | None] = mapped_column(String(64)) # 批次 ID
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, nullable=False)
__table_args__ = (
Index("ix_lineage_source", "source_system", "source_record_id"),
Index("ix_lineage_target", "target_table", "target_id"),
Index("ix_lineage_batch", "batch_id"),
)
+2 -2
View File
@@ -150,8 +150,8 @@ async def run_backtest(
summary = BacktestSummary(total=len(candidates), scored=0) summary = BacktestSummary(total=len(candidates), scored=0)
# P1-6: 并发控制,同时最多 8 场预测(避免 LLM API 限流) # P2: 并发控制,同时最多 3 场预测(避免 LLM API 限流,保护下游服务)
sem = asyncio.Semaphore(8) sem = asyncio.Semaphore(3)
async def _one(c: BacktestCandidate) -> BacktestMatchResult | None: async def _one(c: BacktestCandidate) -> BacktestMatchResult | None:
async with sem: async with sem: