debt(D1): events 成功路径补写 Bronze 层(RawEvent + DataLineage)

- 新增 _events_record_id: 上游 id 缺失时用 (league:home:away:date) 合成稳定幂等键
- 新增 _write_events_bronze: best-effort 写 RawEvent(幂等) + Lineage(matches/events_ingest)
- ingest 插入与变更更新后触发;同批 seen 集合防重复;基础设施失败只 warning
- TDD: 6 测试(插入/合成键/幂等跳过/变更更新/无变化不写/失败不拖垮),双变异验证通过
This commit is contained in:
2026-09-21 19:23:37 +08:00
parent f6c0145c32
commit 3a9f3f5a0e
2 changed files with 338 additions and 0 deletions
+79
View File
@@ -266,7 +266,15 @@ class BzzoiroSource:
}
# else: existing_matches 保持空 dict(全量新比赛)
# D1: Bronze 层批次信息(每联赛每批次一个 batch_id;seen 防同批重复写入)
now = datetime.now(timezone.utc)
bronze_batch_id = f"bzzoiro-events-{code}-{now:%Y%m%d%H%M%S}"
bronze_written: set[str] = set()
for nm, raw in normalized_matches:
# D1: RawEvent 幂等键(上游 id 或合成键),插入/变更更新共用
record_id = _events_record_id(code, nm, raw)
# 球队: 内存查找 + 按需创建
home_team_id = team_name_to_id.get(nm.home_team)
if home_team_id is None:
@@ -310,6 +318,18 @@ class BzzoiroSource:
# 统计字段不在 /events/ 载荷中(单独由 stats 管线回填),
# 此处不再创建 MatchStats。
league_r["inserted"] += 1
# D1: 成功插入 → 补写 Bronze 层(原始载荷 + 血缘)
if record_id not in bronze_written:
bronze_written.add(record_id)
await _write_events_bronze(
db,
source_record_id=record_id,
raw_payload=raw,
target_match_id=m.id,
league_code=code,
match_status=nm.match_status,
batch_id=bronze_batch_id,
)
else:
# 已有比赛: 直接从内存获取对象更新(无需再查询)
changed = False
@@ -332,6 +352,18 @@ class BzzoiroSource:
changed = True
if changed:
league_r["updated"] += 1
# D1: 变更更新 → 补写血缘(RawEvent 幂等键不变,重复采集自动跳过)
if record_id not in bronze_written:
bronze_written.add(record_id)
await _write_events_bronze(
db,
source_record_id=record_id,
raw_payload=raw,
target_match_id=existing_match.id,
league_code=code,
match_status=nm.match_status,
batch_id=bronze_batch_id,
)
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
result["leagues"][code] = league_r
@@ -412,6 +444,53 @@ async def _write_lineage(db, source_system: str, source_record_id: str, target_t
))
def _events_record_id(league_code: str, nm, raw: dict) -> str:
"""events 载荷的 RawEvent 幂等键。
优先用上游 event id;缺失时用 (league:home:away:date) 合成稳定键 ——
取 normalize 后的队名与天级日期(与 _match_key 同口径),不依赖 DB 自增 id,
保证同一来源比赛重复采集时命中同一条 RawEvent,不产生重复原始载荷。
"""
eid = _to_int_or_none(raw.get("id"))
if eid is not None:
return str(eid)
d = _to_date(nm.date)
date_part = d.isoformat() if d is not None else "na"
return f"{league_code}:{nm.home_team}:{nm.away_team}:{date_part}"
async def _write_events_bronze(
db,
*,
source_record_id: str,
raw_payload: dict,
target_match_id: int | None,
league_code: str,
match_status: str | None,
batch_id: str,
) -> None:
"""events 成功插入/更新单场比赛后的 Bronze 层补写:RawEvent(幂等) + DataLineage。
D1(工程债):此前只有 stats 回填写 RawEvent/Lineage,events 管线作为比赛
主数据的唯一入口反而不留溯源记录。幂等性由 _write_raw_event 的
source_record_id 查重保证;best-effort:基础设施写入失败只记 warning,
绝不拖垮采集主流程(与 _safe_write_ingest_failure 同级约束)。
"""
try:
await _write_raw_event(db, "bzzoiro", source_record_id, raw_payload, batch_id)
await _write_lineage(
db, "bzzoiro", source_record_id,
"matches", target_match_id, "events_ingest",
{"league": league_code, "match_status": match_status},
batch_id,
)
except Exception:
logger.warning(
"events Bronze 写入失败(record=%s, match=%s),不影响采集主流程",
source_record_id, target_match_id, exc_info=True,
)
# ============================================================
# 积分榜管线:/leagues/{id}/standings/ → standings 表
# ============================================================