fix(pipeline): 死信表真正接线 + 前端 HTTP 收敛与状态修正
- bzzoiro events/standings/stats 抓取失败写入 IngestFailure 死信(尽力而为, 写入失败不影响主流程);顺带修复 standings 失败路径 league_r 缺 errors 键 的 KeyError —— 该路径此前从未跑通,一旦失败会顶掉原始异常 - admin/api.ts 收敛为 lib/http.ts 薄门面,消除第二套 HTTP 实现; UNAUTHORIZED_EVENT 定义移至共享层,断开 lib→admin 反向依赖 - STATUS_META 死键 live 改为 in_play(对齐 normalize.py 口径), 补 paused/postponed/cancelled/suspended;移除无人消费的 DashboardStats.total_matches(items.length 近似,上限 100) - 新增 tests/test_ingest_deadletter.py(6 例,变异验证判别力)
This commit is contained in:
+47
-1
@@ -200,6 +200,13 @@ class BzzoiroSource:
|
||||
# 单联赛抓取失败隔离:记录错误后继续其余联赛,不拖垮整批
|
||||
logger.exception("bzzoiro fetch failed for %s", code)
|
||||
league_r["errors"].append(f"fetch failed: {e}")
|
||||
await _safe_write_ingest_failure(
|
||||
db,
|
||||
entity_type="events",
|
||||
source_record_id=None,
|
||||
error=e,
|
||||
raw_payload={"league": code, "status": status, "date_from": date_from, "date_to": date_to},
|
||||
)
|
||||
result["leagues"][code] = league_r
|
||||
continue
|
||||
|
||||
@@ -367,6 +374,31 @@ async def _write_ingest_failure(db, source_system: str, entity_type: str, source
|
||||
))
|
||||
|
||||
|
||||
async def _safe_write_ingest_failure(
|
||||
db,
|
||||
*,
|
||||
entity_type: str,
|
||||
source_record_id: str | None,
|
||||
error: Exception,
|
||||
raw_payload: dict | None = None,
|
||||
) -> None:
|
||||
"""抓取失败时尽力写入死信表(失败不影响主流程)。
|
||||
|
||||
死信是「可观测性」基础设施,与 RawEvent/Lineage 同级:写入失败只记
|
||||
warning,绝不能让原始抓取错误之外的新异常打断采集循环。
|
||||
"""
|
||||
try:
|
||||
await _write_ingest_failure(
|
||||
db, "bzzoiro", entity_type, source_record_id,
|
||||
"fetch_error", str(error), raw_payload,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"写入 ingest_failures 死信失败(entity=%s, record=%s): %s",
|
||||
entity_type, source_record_id, error, exc_info=True,
|
||||
)
|
||||
|
||||
|
||||
async def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
|
||||
"""写入 ETL 血缘追踪。"""
|
||||
db.add(DataLineage(
|
||||
@@ -418,12 +450,19 @@ async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str |
|
||||
|
||||
result: dict = {"leagues": {}, "total_upserted": 0, "errors": []}
|
||||
for code in leagues:
|
||||
league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0}
|
||||
league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0, "errors": []}
|
||||
try:
|
||||
payload = await fetch_bzzoiro_standings(code, season=season)
|
||||
except Exception as e:
|
||||
logger.exception("bzzoiro standings fetch failed for %s", code)
|
||||
league_r["errors"].append(str(e))
|
||||
await _safe_write_ingest_failure(
|
||||
db,
|
||||
entity_type="standings",
|
||||
source_record_id=None,
|
||||
error=e,
|
||||
raw_payload={"league": code, "season": season},
|
||||
)
|
||||
result["leagues"][code] = league_r
|
||||
result["errors"].append(f"{code}: {e}")
|
||||
continue
|
||||
@@ -634,6 +673,13 @@ async def ingest_bzzoiro_event_stats(
|
||||
except Exception as e:
|
||||
logger.warning("stats fetch failed match=%s event=%s: %s", m.id, m.source_event_id, e)
|
||||
result["errors"].append(f"match {m.id}: {e}")
|
||||
await _safe_write_ingest_failure(
|
||||
db,
|
||||
entity_type="match_stats",
|
||||
source_record_id=str(m.source_event_id),
|
||||
error=e,
|
||||
raw_payload={"match_id": m.id},
|
||||
)
|
||||
await asyncio.sleep(REQUEST_INTERVAL)
|
||||
continue
|
||||
|
||||
|
||||
Reference in New Issue
Block a user