refactor: 以 bzzoiro 为唯一数据源的全面重构
数据源统一为 bzzoiro,移除 Understat 与 injuries:
- 删除 src/data/understat.py / injuries.py 及相关测试
- 删除 injuries 模型与表;扩展 match_stats(xG 之外增加 big_chances/fouls)
- 新增 standings 表(联赛积分榜:位置/积分/xG/走势/分区)
- matches 表增加 source_event_id 血缘列,支撑统计回填
采集管线(bzzoiro 三条管线):
- events:赛程/比分(/events/),记录 source_event_id
- standings:积分榜快照(/leagues/{id}/standings/)
- stats:已完赛比赛详细统计回填(/events/{id}/stats/)
预测增强:
- standings_slice 替代 injuries_slice;积分榜专家替代阵容完整性专家
- AGENT_META runtime_config 同步更新
管理后台:
- ingest 路由重写为单一 bzzoiro 入口 + task 参数(events/standings/stats/all)
- 新增 /admin/data-completeness 数据完整性分析 API
- 数据源状态页简化为 bzzoiro 单源
前端:
- 采集页重构为任务驱动(比赛/积分榜/统计回填/全量)
- 新增「数据完整性」可视化页(覆盖率矩阵/字段完整率/健康摘要)
- 新增主站积分榜页(/standings)与比赛详情完整统计面板
- agent 名称同步更新(injuries→standings)
迁移 0015_bzzoiro_single_source 已在容器内验证通过,后端测试全部通过。
Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
co-authored by
new-provider/LongCat-2.0 <
new-provider/LongCat-2.0 <
parent
ec8f36abb2
commit
f05dc1ae15
+60
-87
@@ -1,4 +1,11 @@
|
||||
"""采集路由。"""
|
||||
"""采集路由(bzzoiro 单一数据源)。
|
||||
|
||||
任务类型:
|
||||
events — 比赛日程/比分(/events/)
|
||||
standings — 联赛积分榜(/leagues/{id}/standings/)
|
||||
stats — 已完赛比赛详细统计回填(/events/{id}/stats/)
|
||||
all — 依次执行以上三项
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
@@ -7,10 +14,10 @@ import logging
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from src.api.deps import require_admin
|
||||
from src.api.schemas import IngestBzzoiroRequest, IngestResponse, IngestUnderstatRequest, IngestInjuriesRequest, IngestSimpleResponse
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, FDCO_TO_UNDERSTAT
|
||||
from src.api.schemas import IngestBzzoiroRequest
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS
|
||||
from src.data.bzzoiro import ingest_bzzoiro_event_stats, ingest_bzzoiro_standings
|
||||
from src.data.sources import get_source
|
||||
from src.data.injuries import ingest_injuries
|
||||
from src.db.unit_of_work import get_uow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -20,6 +27,8 @@ router = APIRouter(prefix="/api/v1", tags=["ingest"])
|
||||
# 后台采集任务注册表:持强引用防止被 GC
|
||||
_background_tasks: set[asyncio.Task] = set()
|
||||
|
||||
VALID_TASKS = {"events", "standings", "stats", "all"}
|
||||
|
||||
|
||||
def _spawn(coro) -> None:
|
||||
"""启动后台采集任务;异常已在任务内记录到系统日志。"""
|
||||
@@ -30,96 +39,60 @@ def _spawn(coro) -> None:
|
||||
|
||||
@router.post("/ingest/bzzoiro", dependencies=[Depends(require_admin)])
|
||||
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
||||
"""触发 bzzoiro 采集。"""
|
||||
# 未指定联赛 = 采集全部已知联赛;未指定状态 = 已完赛 + 未开赛都采集
|
||||
"""触发 bzzoiro 采集(events / standings / stats / all)。"""
|
||||
if req.task not in VALID_TASKS:
|
||||
raise HTTPException(status_code=422, detail=f"未知任务类型: {req.task}(可选: {', '.join(sorted(VALID_TASKS))})")
|
||||
leagues = req.leagues or list(BZZOIRO_LEAGUE_IDS.keys())
|
||||
statuses = [req.status] if req.status else ["finished", "scheduled"]
|
||||
_spawn(_run_bzzoiro(leagues, req.date_from, req.date_to, statuses))
|
||||
task_label = {"events": "比赛数据", "standings": "积分榜", "stats": "统计回填", "all": "全量(比赛+积分榜+统计)"}[req.task]
|
||||
_spawn(_run_bzzoiro(req.task, leagues, req))
|
||||
return {
|
||||
"ok": True,
|
||||
"message": f"采集任务已启动(后台执行,状态: {', '.join(statuses)}),请在「系统日志」查看进度与结果",
|
||||
"message": f"采集任务已启动(后台执行,任务: {task_label}),请在「系统日志」查看进度与结果",
|
||||
}
|
||||
|
||||
|
||||
async def _run_bzzoiro(leagues: list[str], date_from: str | None, date_to: str | None, statuses: list[str]) -> None:
|
||||
async def _run_bzzoiro(task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None:
|
||||
"""后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。"""
|
||||
try:
|
||||
source = get_source("bzzoiro")
|
||||
merged: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||
async with get_uow() as session:
|
||||
for st in statuses:
|
||||
r = await source.ingest(
|
||||
session,
|
||||
leagues=leagues,
|
||||
date_from=date_from,
|
||||
date_to=date_to,
|
||||
status=st,
|
||||
if task in ("events", "all"):
|
||||
statuses = [req.status] if req.status else ["finished", "scheduled"]
|
||||
source = get_source("bzzoiro")
|
||||
merged: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||
async with get_uow() as session:
|
||||
for st in statuses:
|
||||
r = await source.ingest(
|
||||
session, leagues=leagues,
|
||||
date_from=req.date_from, date_to=req.date_to, status=st,
|
||||
)
|
||||
merged["total_inserted"] += r.get("total_inserted", 0)
|
||||
merged["total_updated"] += r.get("total_updated", 0)
|
||||
merged["errors"].extend(r.get("errors", []))
|
||||
for code, stat in r.get("leagues", {}).items():
|
||||
acc = merged["leagues"].setdefault(code, {"inserted": 0, "updated": 0, "errors": []})
|
||||
acc["inserted"] += stat.get("inserted", 0)
|
||||
acc["updated"] += stat.get("updated", 0)
|
||||
acc["errors"].extend(stat.get("errors", []))
|
||||
logger.info(
|
||||
"bzzoiro 比赛采集完成: 新增 %d, 更新 %d, 联赛 %d 个, 状态 %s",
|
||||
merged["total_inserted"], merged["total_updated"], len(merged["leagues"]), statuses,
|
||||
)
|
||||
if merged["errors"]:
|
||||
logger.warning("bzzoiro 比赛采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3])
|
||||
|
||||
if task in ("standings", "all"):
|
||||
async with get_uow() as session:
|
||||
r = await ingest_bzzoiro_standings(session, leagues=leagues, season=req.season)
|
||||
if r["errors"]:
|
||||
logger.warning("bzzoiro 积分榜采集部分失败: %s", r["errors"][:3])
|
||||
else:
|
||||
logger.info("bzzoiro 积分榜采集完成: upsert %d 条", r["total_upserted"])
|
||||
|
||||
if task in ("stats", "all"):
|
||||
async with get_uow() as session:
|
||||
r = await ingest_bzzoiro_event_stats(
|
||||
session, leagues=leagues, limit=req.limit, only_missing=True
|
||||
)
|
||||
merged["total_inserted"] += r.get("total_inserted", 0)
|
||||
merged["total_updated"] += r.get("total_updated", 0)
|
||||
merged["errors"].extend(r.get("errors", []))
|
||||
for code, stat in r.get("leagues", {}).items():
|
||||
acc = merged["leagues"].setdefault(code, {"inserted": 0, "updated": 0, "errors": []})
|
||||
acc["inserted"] += stat.get("inserted", 0)
|
||||
acc["updated"] += stat.get("updated", 0)
|
||||
acc["errors"].extend(stat.get("errors", []))
|
||||
league_errors = {c: stat["errors"] for c, stat in merged["leagues"].items() if stat.get("errors")}
|
||||
logger.info(
|
||||
"bzzoiro 采集完成: 新增 %d, 更新 %d, 联赛 %d 个, 状态 %s",
|
||||
merged["total_inserted"], merged["total_updated"], len(merged["leagues"]), statuses,
|
||||
)
|
||||
if league_errors:
|
||||
sample = {c: errs[:1] for c, errs in list(league_errors.items())[:3]}
|
||||
logger.warning("bzzoiro 部分联赛存在错误: %s", sample)
|
||||
if merged["errors"]:
|
||||
logger.warning("bzzoiro 采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3])
|
||||
logger.debug("bzzoiro 采集明细: leagues=%s", list(merged["leagues"].keys()))
|
||||
if r["errors"]:
|
||||
logger.warning("bzzoiro 统计回填错误 %d 条: %s", len(r["errors"]), r["errors"][:3])
|
||||
except Exception:
|
||||
logger.exception("bzzoiro 采集任务失败")
|
||||
|
||||
|
||||
@router.post("/ingest/understat", dependencies=[Depends(require_admin)])
|
||||
async def ingest_understat_route(req: IngestUnderstatRequest):
|
||||
"""触发 understat xG 回填。"""
|
||||
leagues_to_run = [req.league] if req.league else list(FDCO_TO_UNDERSTAT.keys())
|
||||
_spawn(_run_understat(leagues_to_run, req.season))
|
||||
return {"ok": True, "message": "xG 回填任务已启动(后台执行),请在「系统日志」查看结果"}
|
||||
|
||||
|
||||
async def _run_understat(leagues_to_run: list[str], season: int) -> None:
|
||||
try:
|
||||
source = get_source("understat")
|
||||
merged: dict = {"count": 0, "updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
||||
async with get_uow() as session:
|
||||
for league in leagues_to_run:
|
||||
r = await source.ingest(session, league=league, season=season)
|
||||
for k in ("count", "updated", "skipped", "unmatched"):
|
||||
merged[k] += r.get(k, 0)
|
||||
merged["errors"].extend(r.get("errors", []))
|
||||
logger.info(
|
||||
"understat 回填完成: 联赛 %d 个, 更新 %d, 未匹配 %d, 错误 %d",
|
||||
len(leagues_to_run), merged["updated"], merged["unmatched"], len(merged["errors"]),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("understat 回填任务失败")
|
||||
|
||||
|
||||
@router.post("/ingest/injuries", dependencies=[Depends(require_admin)])
|
||||
async def ingest_injuries_route(req: IngestInjuriesRequest):
|
||||
"""触发伤停采集。"""
|
||||
_spawn(_run_injuries(req.date))
|
||||
return {"ok": True, "message": "伤停采集任务已启动(后台执行),请在「系统日志」查看结果"}
|
||||
|
||||
|
||||
async def _run_injuries(date: str | None) -> None:
|
||||
try:
|
||||
async with get_uow() as session:
|
||||
result = await ingest_injuries(session, date=date)
|
||||
logger.info(
|
||||
"injuries 采集完成: 新增 %d, 更新 %d, 错误 %d",
|
||||
result.get("count", 0), result.get("updated", 0), len(result.get("errors", [])),
|
||||
)
|
||||
if result.get("errors"):
|
||||
logger.warning("injuries 采集错误: %s", result["errors"][:3])
|
||||
except Exception:
|
||||
logger.exception("injuries 采集任务失败")
|
||||
logger.exception("bzzoiro 采集任务失败(task=%s)", task)
|
||||
|
||||
Reference in New Issue
Block a user