feat: standings Bronze 血缘补齐 + 队名归一 MVP

standings 成功 upsert 后写入 RawEvent(source_record_id=standings:{league}:{season}) + DataLineage(target_table=standings),与 events/stats 管线对称。队名归一化收敛到 TeamRepository.get_or_create 唯一咽喉点, 创建新 Team 时 info 日志打出原始名与归一后名; Admin 新增 GET /api/v1/admin/team-name-duplicates 只读接口, 启发式列出近似重名候选(大小写变体/子串/前缀碰撞),不做自动合并。
2026-09-21 23:28:46 +08:00
parent 317a5e338a
commit 66f0844798
5 changed files with 685 additions and 5 deletions
@@ -0,0 +1,343 @@
+"""后台管理:管理区统计、数据完整性分析、数据质量检查。
+
+所有接口需管理员鉴权(require_admin)。路由前缀 /api/v1/admin。
+"""
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import func, select
+
+from src.api.deps import require_admin
+from src.db.base import AsyncSession, get_db_read
+from src.db.models import DataQualityCheck, IngestFailure, League, Match, MatchStats, Prediction, Standing
+
+logger = logging.getLogger(__name__)
+
+router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
+
+
+@router.get("/stats")
+async def admin_stats(db: AsyncSession = Depends(get_db_read)):
+    """管理区统计(只读):预测次数 + 比赛覆盖。轻量聚合,无 LLM 调用。"""
+    day_ago = datetime.now(timezone.utc) - timedelta(days=1)
+    week_ago = datetime.now(timezone.utc) - timedelta(days=7)
+    r = (
+        await db.execute(
+            select(
+                func.count().label("total"),
+                func.count().filter(Prediction.created_at >= day_ago).label("last_24h"),
+                func.count().filter(Prediction.created_at >= week_ago).label("last_7d"),
+            )
+        )
+    ).one()
+    # F3 修复: 补充真实比赛计数(非 limit=100 近似)
+    match_cnt = (await db.execute(select(func.count()).select_from(Match))).scalar() or 0
+    finished_cnt = (await db.execute(select(func.count()).where(Match.match_status == "finished"))).scalar() or 0
+    stats_cnt = (await db.execute(select(func.count()).select_from(MatchStats))).scalar() or 0
+    standings_cnt = (await db.execute(select(func.count()).select_from(Standing))).scalar() or 0
+    return {
+        "predictions": {"total": r.total, "last_24h": r.last_24h, "last_7d": r.last_7d},
+        "matches": {"total": match_cnt, "finished": finished_cnt},
+        "stats": {"total": stats_cnt},
+        "standings": {"total": standings_cnt},
+    }
+
+
+# ── 数据完整性分析(可视化数据源) ────────────────────────────────
+
+
+@router.get("/data-completeness")
+async def data_completeness(db: AsyncSession = Depends(get_db_read)):
+    """按联赛统计数据完整性:比赛覆盖、字段覆盖、积分榜覆盖。
+
+    前端「数据完整性」页据此渲染,回答三个问题:
+      1. 数据是否齐全(各联赛比赛/统计/积分榜量级)
+      2. 字段是否齐全(每张统计表各字段非空率)
+      3. 覆盖是否新鲜(最近一场/最近一次采集)
+    """
+    from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_NAMES, LEAGUE_COUNTRIES
+
+    out_leagues: list[dict] = []
+    for code, bzz_id in BZZOIRO_LEAGUE_IDS.items():
+        # 比赛覆盖
+        m = (
+            await db.execute(
+                select(
+                    func.count().label("total"),
+                    func.count().filter(Match.match_status == "finished").label("finished"),
+                    func.count().filter(Match.match_status == "scheduled").label("scheduled"),
+                    func.count().filter(Match.source_event_id.is_not(None)).label("with_source_id"),
+                    func.max(Match.match_date).label("latest_match"),
+                    func.min(Match.match_date).label("earliest_match"),
+                )
+                .select_from(Match)
+                .join(League, League.id == Match.league_id)
+                .where(League.code == code)
+            )
+        ).one()
+        # 统计字段覆盖(联表 matches)
+        s = (
+            await db.execute(
+                select(
+                    func.count().label("rows"),
+                    func.count(MatchStats.home_xg).label("xg"),
+                    func.count(MatchStats.home_shots).label("shots"),
+                    func.count(MatchStats.home_possession).label("possession"),
+                    func.count(MatchStats.home_corners).label("corners"),
+                    func.count(MatchStats.home_fouls).label("fouls"),
+                    func.count(MatchStats.home_big_chances).label("big_chances"),
+                    func.count(MatchStats.home_yellow_cards).label("cards"),
+                )
+                .select_from(MatchStats)
+                .join(Match, Match.id == MatchStats.match_id)
+                .join(League, League.id == Match.league_id)
+                .where(League.code == code)
+            )
+        ).one()
+        # 积分榜覆盖
+        st = (
+            await db.execute(
+                select(
+                    func.count().label("rows"),
+                    func.max(Standing.retrieved_at).label("latest_retrieved"),
+                )
+                .select_from(Standing)
+                .join(League, League.id == Standing.league_id)
+                .where(League.code == code)
+            )
+        ).one()
+
+        stats_rows = s.rows or 0
+        pct = lambda n: round(n / stats_rows * 100, 1) if stats_rows else 0.0  # noqa: E731
+        out_leagues.append(
+            {
+                "code": code,
+                "name": LEAGUE_NAMES.get(code, code),
+                "country": LEAGUE_COUNTRIES.get(code),
+                "matches": {
+                    "total": m.total or 0,
+                    "finished": m.finished or 0,
+                    "scheduled": m.scheduled or 0,
+                    "with_source_id": m.with_source_id or 0,
+                    "earliest_match": m.earliest_match.isoformat() if m.earliest_match else None,
+                    "latest_match": m.latest_match.isoformat() if m.latest_match else None,
+                },
+                "stats": {
+                    "rows": stats_rows,
+                    "fields": {
+                        "xg": {"count": s.xg or 0, "pct": pct(s.xg or 0)},
+                        "shots": {"count": s.shots or 0, "pct": pct(s.shots or 0)},
+                        "possession": {"count": s.possession or 0, "pct": pct(s.possession or 0)},
+                        "corners": {"count": s.corners or 0, "pct": pct(s.corners or 0)},
+                        "fouls": {"count": s.fouls or 0, "pct": pct(s.fouls or 0)},
+                        "big_chances": {"count": s.big_chances or 0, "pct": pct(s.big_chances or 0)},
+                        "cards": {"count": s.cards or 0, "pct": pct(s.cards or 0)},
+                    },
+                },
+                "standings": {
+                    "rows": st.rows or 0,
+                    "latest_retrieved": st.latest_retrieved.isoformat() if st.latest_retrieved else None,
+                },
+            }
+        )
+
+    # 整体健康信号
+    total_finished = sum(l["matches"]["finished"] for l in out_leagues)
+    total_stats = sum(l["stats"]["rows"] for l in out_leagues)
+    stats_coverage = round(total_stats / total_finished * 100, 1) if total_finished else 0.0
+    issues: list[str] = []
+    for l in out_leagues:
+        if l["matches"]["finished"] == 0:
+            issues.append(f"{l['name']}: 无已完赛比赛,请先运行「比赛数据」采集")
+        elif l["stats"]["rows"] == 0:
+            issues.append(f"{l['name']}: 已完赛 {l['matches']['finished']} 场但无统计回填,请运行「统计回填」采集")
+        elif stats_coverage < 80:
+            issues.append(f"{l['name']}: 统计覆盖率仅 {stats_coverage}%,建议增量回填")
+        if l["standings"]["rows"] == 0:
+            issues.append(f"{l['name']}: 无积分榜数据,请运行「积分榜」采集")
+    if not issues:
+        issues.append("各联赛数据完整度良好")
+
+    return {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "leagues": out_leagues,
+        "totals": {
+            "finished_matches": total_finished,
+            "stats_rows": total_stats,
+            "stats_coverage_pct": stats_coverage,
+        },
+        "issues": issues,
+    }
+
+
+# ── 数据质量检查 API ────────────────────────────────────────────
+
+
+@router.get("/data-quality")
+async def data_quality_checks(db: AsyncSession = Depends(get_db_read)):
+    """数据质量检查结果(只读)。"""
+    # 最近的失败记录
+    failures = (
+        await db.execute(
+            select(IngestFailure)
+            .where(IngestFailure.status.in_(["pending", "retrying"]))
+            .order_by(IngestFailure.created_at.desc())
+            .limit(20)
+        )
+    ).scalars().all()
+
+    # 最近的质量检查
+    checks = (
+        await db.execute(
+            select(DataQualityCheck)
+            .order_by(DataQualityCheck.checked_at.desc())
+            .limit(20)
+        )
+    ).scalars().all()
+
+    return {
+        "failures": [
+            {
+                "id": f.id,
+                "source": f.source_system,
+                "entity_type": f.entity_type,
+                "source_record_id": f.source_record_id,
+                "error_type": f.error_type,
+                "error_detail": f.error_detail,
+                "retry_count": f.retry_count,
+                "status": f.status,
+                "created_at": f.created_at.isoformat() if f.created_at else None,
+            }
+            for f in failures
+        ],
+        "checks": [
+            {
+                "id": c.id,
+                "check_name": c.check_name,
+                "entity_type": c.entity_type,
+                "passed": c.passed,
+                "severity": c.severity,
+                "detail": c.detail,
+                "checked_at": c.checked_at.isoformat() if c.checked_at else None,
+            }
+            for c in checks
+        ],
+    }
+
+
+@router.post("/data-quality/run")
+async def run_data_quality_check(db: AsyncSession = Depends(get_db_read)):
+    """手动触发一次数据质量检查。"""
+    checks = []
+
+    # 检查1: 已完赛但无统计的比赛
+    finished_no_stats = (
+        await db.execute(
+            select(func.count())
+            .select_from(Match)
+            .outerjoin(MatchStats, Match.id == MatchStats.match_id)
+            .where(Match.match_status == "finished")
+            .where(MatchStats.id.is_(None))
+        )
+    ).scalar() or 0
+
+    checks.append(DataQualityCheck(
+        check_name="finished_without_stats",
+        entity_type="match",
+        actual_value=float(finished_no_stats),
+        passed=finished_no_stats == 0,
+        severity="warning" if finished_no_stats > 0 else "info",
+        detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
+    ))
+
+    # 检查2: 积分榜缺失的联赛
+    leagues_without_standings = (
+        await db.execute(
+            select(func.count())
+            .select_from(League)
+            .outerjoin(Standing, League.id == Standing.league_id)
+            .where(Standing.id.is_(None))
+        )
+    ).scalar() or 0
+
+    checks.append(DataQualityCheck(
+        check_name="league_without_standings",
+        entity_type="league",
+        actual_value=float(leagues_without_standings),
+        passed=leagues_without_standings == 0,
+        severity="warning" if leagues_without_standings > 0 else "info",
+        detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
+    ))
+
+    for c in checks:
+        db.add(c)
+    await db.commit()
+
+    return {"ok": True, "checks": [{"name": c.check_name, "passed": c.passed} for c in checks]}
+
+
+# ── 近似重名候选(只读,启发式,不做自动合并) ──────────────────────
+
+
+@router.get("/team-name-duplicates")
+async def team_name_duplicates(db: AsyncSession = Depends(get_db_read)):
+    """只读列出近似重名候选(大小写变体/子串包含/前缀碰撞)。
+
+    启发式规则(命中任一即列为候选):
+      - 大小写变体: lower(name) 相同但 name 不同
+      - 子串包含: A 是 B 的子串且 len(A) ≥ 5
+      - 前缀碰撞: 前 8 字符相同(忽略大小写)
+
+    仅作排查参考,合并需走人工 SQL(见 docs/05-data.md)。
+    """
+    teams = (await db.execute(select(Team.id, Team.name))).all()
+    by_lower: dict[str, list[dict]] = {}
+    for t in teams:
+        key = (t.name or "").lower()
+        by_lower.setdefault(key, []).append({"id": t.id, "name": t.name})
+
+    groups: list[dict] = []
+
+    # 规则1: 大小写变体(lower 相同但原名不同)
+    for key, members in by_lower.items():
+        if len(members) > 1:
+            groups.append({
+                "rule": "case_variant",
+                "key": key,
+                "members": members,
+            })
+
+    # 规则2 & 3: 子串包含 / 前缀碰撞(仅在 lower 名不同的组间比较)
+    distinct = [m for members in by_lower.values() for m in members]
+    seen_pairs: set[tuple[int, int]] = set()
+    for i, a in enumerate(distinct):
+        na = (a["name"] or "").lower()
+        for b in distinct[i + 1:]:
+            nb = (b["name"] or "").lower()
+            if na == nb:
+                continue  # 已被规则1覆盖
+            pair = (min(a["id"], b["id"]), max(a["id"], b["id"]))
+            if pair in seen_pairs:
+                continue
+            hit = None
+            if len(na) >= 5 and na in nb:
+                hit = "substring"
+            elif len(nb) >= 5 and nb in na:
+                hit = "substring"
+            elif len(na) >= 8 and len(nb) >= 8 and na[:8] == nb[:8]:
+                hit = "prefix"
+            if hit:
+                seen_pairs.add(pair)
+                groups.append({
+                    "rule": hit,
+                    "members": [a, b],
+                })
+
+    return {
+        "count": len(groups),
+        "hint": "命中任一启发式仅表示'可疑',合并前请人工确认是否同一球队",
+        "groups": groups,
+    }
@@ -5,6 +5,10 @@ Repository 只负责查询,不负责事务提交。
 """
 from __future__ import annotations

+import logging
+
+logger = logging.getLogger(__name__)
+
 from datetime import datetime
 from sqlalchemy import select
 from sqlalchemy.orm import selectinload
@@ -108,10 +112,22 @@ class TeamRepository:
        return (await self._session.execute(stmt)).scalar_one_or_none()

    async def get_or_create(self, name: str, *, name_zh: str | None = None) -> Team:
-        """按名获取球队,不存在则创建(name_zh 供 bzzoiro 管线写中文名)。"""
-        team = await self.get_by_name(name)
+        """按名获取球队,不存在则创建(name_zh 供 bzzoiro 管线写中文名)。
+
+        归一化咽喉:所有入库 Team.name 必须经过 team_names.normalize,
+        此处统一收敛,避免各调用点散落归一化逻辑导致重复 Team。
+        创建新 Team 时 info 打出原始名与归一后的规范名,便于排查重名。
+        """
+        from src.data.team_names import normalize as normalize_name
+
+        normalized = normalize_name(name) or name.strip()
+        team = await self.get_by_name(normalized)
        if team is None:
-            team = Team(name=name, name_zh=name_zh)
+            logger.info(
+                "创建新 Team: %s -> %s",
+                name, normalized,
+            )
+            team = Team(name=normalized, name_zh=name_zh)
            self._session.add(team)
            await self._session.flush()
        return team