feat: 数据采集管线基础设施接线 + 数据管线管理页

后端:
- bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档)
- 采集失败写入 IngestFailure(死信队列,支持重试)
- 写入 DataLineage(ETL 血缘追踪)
- 新增 DataQualityScheduler(每小时自动质量检查)
- 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发)
- 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试)
- 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛)
- lifespan 启动 quality_scheduler

前端:
- 新增「数据管线」管理页(/admin/data-pipeline)
- 采集失败记录列表(状态/重试次数/错误类型)
- 数据质量检查结果(通过/未通过/严重度)
- 手动触发质量检查按钮
- 失败记录重试按钮
- 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充)
- 导航统一为 React Router Link
- AdminStats 类型扩展(matches/stats/standings 真实计数)

Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
shangfangjian
2026-09-21 10:04:26 +08:00
co-authored by new-provider/LongCat-2.0 <
parent b6e367a640
commit ae89d0f04f
9 changed files with 567 additions and 4 deletions
+76 -2
View File
@@ -27,7 +27,7 @@ from src.data.key_ring import get_key_ring
from src.data.normalize import normalize_bzzoiro
from src.data.team_names_zh import zh_name
from src.data.sources import register
from src.db.models import League, Match, MatchStats, Standing, Team
from src.db.models import League, Match, MatchStats, Standing, Team, RawEvent, IngestFailure, DataLineage
logger = logging.getLogger(__name__)
@@ -337,7 +337,81 @@ class BzzoiroSource:
result["leagues"][code] = league_r
result["total_inserted"] += league_r["inserted"]
result["total_updated"] += league_r["updated"]
return result
# 管线基础设施:写入 RawEvent(原始事件存档)
batch_id = f"bzzoiro-events-{code}-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}"
for nm, raw in normalized_matches:
try:
_write_raw_event(db, "bzzoiro", str(raw.get("id", "")), raw, batch_id)
except Exception:
pass # 基础设施写入失败不影响主流程
# 写入 DataLineage(血缘追踪)
for nm, raw in normalized_matches:
try:
_write_lineage(db, "bzzoiro", str(raw.get("id", ""), "matches", None, "normalize_bzzoiro", {"league_code": code}, batch_id)
except Exception:
pass
except Exception as e:
# 管线基础设施:写入 IngestFailure(失败死信)
try:
_write_ingest_failure(db, "bzzoiro", "events", None, "fetch_failed", str(e)[:500])
except Exception:
pass
logger.exception("bzzoiro events ingest failed for %s", code)
league_r["errors"].append(str(e))
result["leagues"][code] = league_r
result["errors"].append(f"{code}: {e}")
continue
return result
# ============================================================
# 管线基础设施:RawEvent / IngestFailure / DataLineage
# ============================================================
def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None:
"""写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。"""
from sqlalchemy import select as _select
stmt = _select(RawEvent).where(
RawEvent.source_system == source_system,
RawEvent.source_record_id == source_record_id,
)
existing = (await db.execute(stmt)).scalar_one_or_none()
if existing is None:
db.add(RawEvent(
source_system=source_system,
source_record_id=source_record_id,
raw_payload=raw_payload,
ingest_batch_id=batch_id,
))
def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None:
"""写入采集失败死信。"""
db.add(IngestFailure(
source_system=source_system,
entity_type=entity_type,
source_record_id=source_record_id,
error_type=error_type,
error_detail=error_detail,
raw_payload=raw_payload,
))
def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
"""写入 ETL 血缘追踪。"""
db.add(DataLineage(
source_system=source_system,
source_record_id=source_record_id,
target_table=target_table,
target_id=target_id,
transform_name=transform_name,
transform_detail=transform_detail,
batch_id=batch_id,
))
# ============================================================