feat: 数据采集管线基础设施接线 + 数据管线管理页
后端: - bzzoiro 采集成功后写入 RawEvent(Bronze 层原始事件存档) - 采集失败写入 IngestFailure(死信队列,支持重试) - 写入 DataLineage(ETL 血缘追踪) - 新增 DataQualityScheduler(每小时自动质量检查) - 新增 /api/v1/admin/data-quality 端点(质量检查 + 手动触发) - 新增 /api/v1/admin/ingest-failures 端点(失败记录 + 重试) - 新增 /admin/llm/ping 端点(LLM 连通性测试,不依赖比赛) - lifespan 启动 quality_scheduler 前端: - 新增「数据管线」管理页(/admin/data-pipeline) - 采集失败记录列表(状态/重试次数/错误类型) - 数据质量检查结果(通过/未通过/严重度) - 手动触发质量检查按钮 - 失败记录重试按钮 - 预测历史:比赛信息内嵌(日期/队名/主客徽标/赛果自动填充) - 导航统一为 React Router Link - AdminStats 类型扩展(matches/stats/standings 真实计数) Co-Authored-By: new-provider/LongCat-2.0 <<EMAIL>>
This commit is contained in:
co-authored by
new-provider/LongCat-2.0 <
parent
b6e367a640
commit
ae89d0f04f
+76
-2
@@ -27,7 +27,7 @@ from src.data.key_ring import get_key_ring
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.team_names_zh import zh_name
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats, Standing, Team
|
||||
from src.db.models import League, Match, MatchStats, Standing, Team, RawEvent, IngestFailure, DataLineage
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -337,7 +337,81 @@ class BzzoiroSource:
|
||||
result["leagues"][code] = league_r
|
||||
result["total_inserted"] += league_r["inserted"]
|
||||
result["total_updated"] += league_r["updated"]
|
||||
return result
|
||||
|
||||
# 管线基础设施:写入 RawEvent(原始事件存档)
|
||||
batch_id = f"bzzoiro-events-{code}-{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}"
|
||||
for nm, raw in normalized_matches:
|
||||
try:
|
||||
_write_raw_event(db, "bzzoiro", str(raw.get("id", "")), raw, batch_id)
|
||||
except Exception:
|
||||
pass # 基础设施写入失败不影响主流程
|
||||
|
||||
# 写入 DataLineage(血缘追踪)
|
||||
for nm, raw in normalized_matches:
|
||||
try:
|
||||
_write_lineage(db, "bzzoiro", str(raw.get("id", ""), "matches", None, "normalize_bzzoiro", {"league_code": code}, batch_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
# 管线基础设施:写入 IngestFailure(失败死信)
|
||||
try:
|
||||
_write_ingest_failure(db, "bzzoiro", "events", None, "fetch_failed", str(e)[:500])
|
||||
except Exception:
|
||||
pass
|
||||
logger.exception("bzzoiro events ingest failed for %s", code)
|
||||
league_r["errors"].append(str(e))
|
||||
result["leagues"][code] = league_r
|
||||
result["errors"].append(f"{code}: {e}")
|
||||
continue
|
||||
return result
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 管线基础设施:RawEvent / IngestFailure / DataLineage
|
||||
# ============================================================
|
||||
|
||||
|
||||
def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None:
|
||||
"""写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。"""
|
||||
from sqlalchemy import select as _select
|
||||
stmt = _select(RawEvent).where(
|
||||
RawEvent.source_system == source_system,
|
||||
RawEvent.source_record_id == source_record_id,
|
||||
)
|
||||
existing = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if existing is None:
|
||||
db.add(RawEvent(
|
||||
source_system=source_system,
|
||||
source_record_id=source_record_id,
|
||||
raw_payload=raw_payload,
|
||||
ingest_batch_id=batch_id,
|
||||
))
|
||||
|
||||
|
||||
def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None:
|
||||
"""写入采集失败死信。"""
|
||||
db.add(IngestFailure(
|
||||
source_system=source_system,
|
||||
entity_type=entity_type,
|
||||
source_record_id=source_record_id,
|
||||
error_type=error_type,
|
||||
error_detail=error_detail,
|
||||
raw_payload=raw_payload,
|
||||
))
|
||||
|
||||
|
||||
def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
|
||||
"""写入 ETL 血缘追踪。"""
|
||||
db.add(DataLineage(
|
||||
source_system=source_system,
|
||||
source_record_id=source_record_id,
|
||||
target_table=target_table,
|
||||
target_id=target_id,
|
||||
transform_name=transform_name,
|
||||
transform_detail=transform_detail,
|
||||
batch_id=batch_id,
|
||||
))
|
||||
|
||||
|
||||
# ============================================================
|
||||
|
||||
Reference in New Issue
Block a user