refactor: 以 bzzoiro 为唯一数据源的全面重构

数据源统一为 bzzoiro,移除 Understat 与 injuries:
- 删除 src/data/understat.py / injuries.py 及相关测试
- 删除 injuries 模型与表;扩展 match_stats(xG 之外增加 big_chances/fouls)
- 新增 standings 表(联赛积分榜:位置/积分/xG/走势/分区)
- matches 表增加 source_event_id 血缘列,支撑统计回填

采集管线(bzzoiro 三条管线):
- events:赛程/比分(/events/),记录 source_event_id
- standings:积分榜快照(/leagues/{id}/standings/)
- stats:已完赛比赛详细统计回填(/events/{id}/stats/)

预测增强:
- standings_slice 替代 injuries_slice;积分榜专家替代阵容完整性专家
- AGENT_META runtime_config 同步更新

管理后台:
- ingest 路由重写为单一 bzzoiro 入口 + task 参数(events/standings/stats/all)
- 新增 /admin/data-completeness 数据完整性分析 API
- 数据源状态页简化为 bzzoiro 单源

前端:
- 采集页重构为任务驱动(比赛/积分榜/统计回填/全量)
- 新增「数据完整性」可视化页(覆盖率矩阵/字段完整率/健康摘要)
- 新增主站积分榜页(/standings)与比赛详情完整统计面板
- agent 名称同步更新(injuries→standings)

迁移 0015_bzzoiro_single_source 已在容器内验证通过,后端测试全部通过。

Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
2026-09-20 19:13:07 +08:00
co-authored by new-provider/LongCat-2.0 <
parent ec8f36abb2
commit f05dc1ae15
41 changed files with 1603 additions and 1885 deletions
+160 -93
View File
@@ -28,33 +28,21 @@ from src.core.runtime_config import (
set_runtime_value,
)
from src.db.base import AsyncSession, get_db_read
from src.db.models import Injury, Match, MatchStats
from src.db.models import League, Match, MatchStats, Standing
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
# ── 数据源元数据 ────────────────────────────────────────────────
# ── 数据源元数据(bzzoiro 单一数据源) ────────────────────────────
_SOURCES: list[dict] = [
{
"name": "bzzoiro",
"label": "Bzzoiro",
"description": "历史赛程比分数据,覆盖全球主要联赛",
"description": "唯一数据源:赛程比分 + 积分榜 + 比赛详细统计(xG/射门/控球等)",
"setting_keys": ["BZZOIRO_KEY", "BZZOIRO_BASE"],
},
{
"name": "understat",
"label": "Understat",
"description": "xG(预期进球)进阶数据,无需 API Key,网页抓取",
"setting_keys": [],
},
{
"name": "injuries",
"label": "Injuries (API-Football)",
"description": "球员伤停信息,用于预测时考虑阵容完整性",
"setting_keys": ["API_FOOTBALL_KEY"],
},
]
@@ -63,9 +51,7 @@ class SettingUpdateIn(BaseModel):
async def _last_ingestion(db: AsyncSession, source: str) -> datetime | None:
"""源最近一次采集时间(取自数据血缘字段,无记录返回 None)。"""
if source == "injuries":
return (await db.execute(select(func.max(Injury.retrieved_at)))).scalar()
"""源最近一次采集时间(取自数据血缘字段,无记录返回 None)。"""
return (
await db.execute(
select(func.max(MatchStats.retrieved_at)).where(MatchStats.source == source)
@@ -303,20 +289,7 @@ async def test_datasource(name: str):
params={"date_from": today, "date_to": today},
)
if name == "understat":
return await _probe(
"https://understat.com/league/EPL/2025",
headers={"User-Agent": "Mozilla/5.0", "Accept": "text/html"},
)
# injuries (api-football)
api_key = await get_runtime_value("API_FOOTBALL_KEY")
if not api_key:
return {"ok": False, "status": None, "latency_ms": 0, "detail": "API_FOOTBALL_KEY 未配置"}
return await _probe(
"https://v3.football.api-sports.io/status",
headers={"x-apisports-key": api_key},
)
raise HTTPException(404, f"未知数据源: {name}")
# ── 数据源健康/最近采集状态(只读,不触发采集) ──────────────────────
@@ -324,16 +297,12 @@ async def test_datasource(name: str):
@router.get("/ingest/status")
async def ingest_status(db: AsyncSession = Depends(get_db_read)):
"""数据源采集健康概览(只读,不触发任何采集)。
返回尽量可得的信息;基于现有表近似的数据会标明 approximation。
失败追踪目前依赖系统日志缓冲,无专用采集失败表。
"""
# ── bzzoiro: 落库目标是 matches 表,无专用采集时间戳 ──
# 近似:以 matches 表最大 match_date(已覆盖的最远比赛日) 与 created_at 作为参考
"""数据源采集健康概览(bzzoiro 单源;只读,不触发任何采集)。"""
bzzoiro_key = await get_runtime_value("BZZOIRO_KEY")
bzzoiro_base = await get_runtime_value("BZZOIRO_BASE")
row = (
# 比赛覆盖
match_row = (
await db.execute(
select(
func.count().label("cnt"),
@@ -342,68 +311,39 @@ async def ingest_status(db: AsyncSession = Depends(get_db_read)):
).where(Match.match_status == "finished")
)
).one()
# 统计覆盖(精确 retrieved_at)
stats_row = (
await db.execute(
select(
func.count().label("cnt"),
func.max(MatchStats.retrieved_at).label("latest_retrieved"),
).where(MatchStats.source == "bzzoiro")
)
).one()
# 积分榜覆盖
standings_row = (
await db.execute(select(func.count()).select_from(Standing))
).scalar()
bzzoiro = {
"name": "bzzoiro",
"label": "Bzzoiro",
"key_configured": bool(bzzoiro_key),
"base_url": (bzzoiro_base.rstrip("/") if bzzoiro_base else None) or settings.BZZOIRO_BASE,
"reachable": None, # 不主动探测
"last_success_at": row.latest_row_at.isoformat() if row.latest_row_at else None,
"latest_match_date": row.latest_match_date.isoformat() if row.latest_match_date else None,
"recent_count": row.cnt or 0,
"note": "approx:基于 matches.finished 表,last_success_at 为行写入时间而非精确采集完成时间",
"last_success_at": (stats_row.latest_retrieved or match_row.latest_row_at),
"last_success_at_iso": (
stats_row.latest_retrieved or match_row.latest_row_at
).isoformat() if (stats_row.latest_retrieved or match_row.latest_row_at) else None,
"latest_match_date": match_row.latest_match_date.isoformat() if match_row.latest_match_date else None,
"recent_count": match_row.cnt or 0,
"stats_count": stats_row.cnt or 0,
"standings_count": standings_row or 0,
"note": "last_success_at 取 match_stats.retrieved_at(统计回填)与 matches.created_at(比赛行)的较大者",
"last_failure": _last_failure_log("bzzoiro"),
}
# ── understat: 落库到 match_stats(source=understat),有精确 retrieved_at ──
row = (
await db.execute(
select(
func.count().label("cnt"),
func.max(MatchStats.retrieved_at).label("latest_retrieved"),
).where(MatchStats.source == "understat")
)
).one()
understat = {
"name": "understat",
"label": "Understat",
"key_configured": True, # 无需 Key
"reachable": None,
"last_success_at": row.latest_retrieved.isoformat() if row.latest_retrieved else None,
"recent_count": row.cnt or 0,
"note": "基于 match_stats.source=understat 的 retrieved_at",
"last_failure": _last_failure_log("understat"),
}
# ── injuries: 落库到 injuries 表,有精确 retrieved_at;区分 Key/无数据/有数据 ──
api_key = await get_runtime_value("API_FOOTBALL_KEY")
row = (
await db.execute(
select(
func.count().label("cnt"),
func.max(Injury.retrieved_at).label("latest_retrieved"),
)
)
).one()
if not api_key:
injuries_status, injuries_note = "key_not_configured", "API_FOOTBALL_KEY 未配置"
elif not row.cnt:
injuries_status, injuries_note = "no_data", "本地无伤停数据,请先采集"
else:
injuries_status, injuries_note = "has_data", f"{row.cnt} 条伤停记录"
injuries = {
"name": "injuries",
"label": "Injuries (API-Football)",
"key_configured": bool(api_key),
"reachable": None,
"status": injuries_status,
"last_success_at": row.latest_retrieved.isoformat() if row.latest_retrieved else None,
"recent_count": row.cnt or 0,
"note": injuries_note,
"last_failure": _last_failure_log("injuries"),
}
return {"sources": [bzzoiro, understat, injuries]}
return {"sources": [bzzoiro]}
def _last_failure_log(source: str) -> dict | None:
@@ -437,3 +377,130 @@ async def admin_stats(db: AsyncSession = Depends(get_db_read)):
)
).one()
return {"predictions": {"total": r.total, "last_24h": r.last_24h, "last_7d": r.last_7d}}
# ── 数据完整性分析(可视化数据源) ────────────────────────────────
@router.get("/data-completeness")
async def data_completeness(db: AsyncSession = Depends(get_db_read)):
"""按联赛统计数据完整性:比赛覆盖、字段覆盖、积分榜覆盖。
前端「数据完整性」页据此渲染,回答三个问题:
1. 数据是否齐全(各联赛比赛/统计/积分榜量级)
2. 字段是否齐全(每张统计表各字段非空率)
3. 覆盖是否新鲜(最近一场/最近一次采集)
"""
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_NAMES, LEAGUE_COUNTRIES
out_leagues: list[dict] = []
for code, bzz_id in BZZOIRO_LEAGUE_IDS.items():
# 比赛覆盖
m = (
await db.execute(
select(
func.count().label("total"),
func.count().filter(Match.match_status == "finished").label("finished"),
func.count().filter(Match.match_status == "scheduled").label("scheduled"),
func.count().filter(Match.source_event_id.is_not(None)).label("with_source_id"),
func.max(Match.match_date).label("latest_match"),
func.min(Match.match_date).label("earliest_match"),
)
.select_from(Match)
.join(League, League.id == Match.league_id)
.where(League.code == code)
)
).one()
# 统计字段覆盖(联表 matches)
s = (
await db.execute(
select(
func.count().label("rows"),
func.count(MatchStats.home_xg).label("xg"),
func.count(MatchStats.home_shots).label("shots"),
func.count(MatchStats.home_possession).label("possession"),
func.count(MatchStats.home_corners).label("corners"),
func.count(MatchStats.home_fouls).label("fouls"),
func.count(MatchStats.home_big_chances).label("big_chances"),
func.count(MatchStats.home_yellow_cards).label("cards"),
)
.select_from(MatchStats)
.join(Match, Match.id == MatchStats.match_id)
.join(League, League.id == Match.league_id)
.where(League.code == code)
)
).one()
# 积分榜覆盖
st = (
await db.execute(
select(
func.count().label("rows"),
func.max(Standing.retrieved_at).label("latest_retrieved"),
)
.select_from(Standing)
.join(League, League.id == Standing.league_id)
.where(League.code == code)
)
).one()
stats_rows = s.rows or 0
pct = lambda n: round(n / stats_rows * 100, 1) if stats_rows else 0.0 # noqa: E731
out_leagues.append(
{
"code": code,
"name": LEAGUE_NAMES.get(code, code),
"country": LEAGUE_COUNTRIES.get(code),
"matches": {
"total": m.total or 0,
"finished": m.finished or 0,
"scheduled": m.scheduled or 0,
"with_source_id": m.with_source_id or 0,
"earliest_match": m.earliest_match.isoformat() if m.earliest_match else None,
"latest_match": m.latest_match.isoformat() if m.latest_match else None,
},
"stats": {
"rows": stats_rows,
"fields": {
"xg": {"count": s.xg or 0, "pct": pct(s.xg or 0)},
"shots": {"count": s.shots or 0, "pct": pct(s.shots or 0)},
"possession": {"count": s.possession or 0, "pct": pct(s.possession or 0)},
"corners": {"count": s.corners or 0, "pct": pct(s.corners or 0)},
"fouls": {"count": s.fouls or 0, "pct": pct(s.fouls or 0)},
"big_chances": {"count": s.big_chances or 0, "pct": pct(s.big_chances or 0)},
"cards": {"count": s.cards or 0, "pct": pct(s.cards or 0)},
},
},
"standings": {
"rows": st.rows or 0,
"latest_retrieved": st.latest_retrieved.isoformat() if st.latest_retrieved else None,
},
}
)
# 整体健康信号
total_finished = sum(l["matches"]["finished"] for l in out_leagues)
total_stats = sum(l["stats"]["rows"] for l in out_leagues)
stats_coverage = round(total_stats / total_finished * 100, 1) if total_finished else 0.0
issues: list[str] = []
for l in out_leagues:
if l["matches"]["finished"] == 0:
issues.append(f"{l['name']}: 无已完赛比赛,请先运行「比赛数据」采集")
elif l["stats"]["rows"] == 0:
issues.append(f"{l['name']}: 已完赛 {l['matches']['finished']} 场但无统计回填,请运行「统计回填」采集")
elif stats_coverage < 80:
issues.append(f"{l['name']}: 统计覆盖率仅 {stats_coverage}%,建议增量回填")
if l["standings"]["rows"] == 0:
issues.append(f"{l['name']}: 无积分榜数据,请运行「积分榜」采集")
if not issues:
issues.append("各联赛数据完整度良好")
return {
"generated_at": datetime.now(timezone.utc).isoformat(),
"leagues": out_leagues,
"totals": {
"finished_matches": total_finished,
"stats_rows": total_stats,
"stats_coverage_pct": stats_coverage,
},
"issues": issues,
}