_upsert_prediction 改为 _insert_or_find_by_fingerprint: - 同 input_hash → 返回已有行(绝不 UPDATE pred_/reasoning/agent_outputs) - 不同 input_hash → INSERT 新行 input_hash 升级为规范 JSON SHA-256,捕获:match_id, cutoff, prompt_version, prompt_hash, system_prompt_hash, provider, model, mode, run_type, temperature, context_hash, agent_ids。移除旧 (match, provider, model, mode, run_type) 唯一约束, 改为 partial unique index(WHERE input_hash IS NOT NULL,兼容旧 NULL 数据)。 三条路径(single/multi/baseline)统一传足指纹字段。 迁移 0024 + 测试 test_p0_prediction_fingerprint(10/10);全量 295 通过。
471 lines
24 KiB
Python
471 lines
24 KiB
Python
"""ORM 模型: leagues / teams / matches / match_stats / standings / predictions。
|
|
|
|
数据源统一为 bzzoiro(单一数据源),伤停(injuries)与 Understat 已移除。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from datetime import date, datetime, timezone
|
|
|
|
from sqlalchemy import (
|
|
BigInteger,
|
|
Boolean,
|
|
CheckConstraint,
|
|
Date,
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
String,
|
|
Text,
|
|
UniqueConstraint,
|
|
column,
|
|
func,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
|
|
from src.db.base import Base
|
|
|
|
|
|
def _utcnow() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class League(Base):
|
|
__tablename__ = "leagues"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
code: Mapped[str] = mapped_column(String(20), unique=True, nullable=False)
|
|
name: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
country: Mapped[str | None] = mapped_column(String(50))
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
|
|
matches: Mapped[list["Match"]] = relationship(back_populates="league")
|
|
|
|
|
|
class Team(Base):
|
|
__tablename__ = "teams"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
name: Mapped[str] = mapped_column(String(120), unique=True, nullable=False)
|
|
name_zh: Mapped[str | None] = mapped_column(String(60))
|
|
team_type: Mapped[str] = mapped_column(String(20), default="club")
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
|
|
home_matches: Mapped[list["Match"]] = relationship(foreign_keys="Match.home_team_id", back_populates="home_team")
|
|
away_matches: Mapped[list["Match"]] = relationship(foreign_keys="Match.away_team_id", back_populates="away_team")
|
|
|
|
|
|
class TeamAlias(Base):
|
|
"""球队别名:同一球队的不同写法(大小写/译名/缩写)映射到归一后的 teams.id。
|
|
|
|
入库流程(get_or_create):normalize(name) → 查 teams.name → 查 team_aliases
|
|
→ 都没有再 insert 新 Team。别名不自动合并历史重复队,需显式添加。
|
|
alias_normalized 为 normalize(别名)后的稳定幂等键,用作 PK 避免重复插入。
|
|
"""
|
|
__tablename__ = "team_aliases"
|
|
|
|
# normalize(别名)后的值,稳定幂等,用作主键
|
|
alias_normalized: Mapped[str] = mapped_column(String(120), primary_key=True)
|
|
team_id: Mapped[int] = mapped_column(ForeignKey("teams.id", ondelete="CASCADE"), nullable=False)
|
|
original_alias: Mapped[str] = mapped_column(String(120), nullable=False) # 原始写法(保留供参考)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
|
|
|
|
class Match(Base):
|
|
__tablename__ = "matches"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
league_id: Mapped[int] = mapped_column(ForeignKey("leagues.id"), nullable=False)
|
|
season: Mapped[str | None] = mapped_column(String(12))
|
|
home_team_id: Mapped[int] = mapped_column(ForeignKey("teams.id"), nullable=False)
|
|
away_team_id: Mapped[int] = mapped_column(ForeignKey("teams.id"), nullable=False)
|
|
match_date: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
match_date_date: Mapped[date] = mapped_column(
|
|
"match_date_date",
|
|
Date,
|
|
nullable=False,
|
|
index=True,
|
|
)
|
|
match_status: Mapped[str] = mapped_column(String(20), default="scheduled")
|
|
# P0-01:比分可信度标记。known=有可靠比分;missing=完赛但缺分(保留 NULL 不伪造 0:0);
|
|
# unknown=待定(无比分且未确认完赛)。禁止把缺分写成 0:0。
|
|
score_status: Mapped[str] = mapped_column(String(20), server_default="unknown", nullable=False)
|
|
home_goals: Mapped[int | None] = mapped_column(Integer)
|
|
away_goals: Mapped[int | None] = mapped_column(Integer)
|
|
home_ht_goals: Mapped[int | None] = mapped_column(Integer)
|
|
away_ht_goals: Mapped[int | None] = mapped_column(Integer)
|
|
match_stage: Mapped[str | None] = mapped_column(String(100))
|
|
# 数据血缘:bzzoiro 上游事件 ID,用于 /events/{id}/stats/ 统计回填
|
|
source_event_id: Mapped[int | None] = mapped_column(BigInteger, index=True)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
|
|
|
league: Mapped[League] = relationship(back_populates="matches", lazy="selectin")
|
|
# lazy="selectin": 这些关系在业务里几乎总是一起读取(切片/回测/展示)。
|
|
# 默认的 lazy="select" 在 async SQLAlchemy 下,于 session 之外或未显式
|
|
# eager-load 时访问会抛 MissingGreenlet —— 已因此导致 form/stats/h2h
|
|
# 三个专家切片静默失败。统一改为预加载,从根上消除这类问题。
|
|
home_team: Mapped[Team] = relationship(
|
|
foreign_keys=[home_team_id], back_populates="home_matches", lazy="selectin"
|
|
)
|
|
away_team: Mapped[Team] = relationship(
|
|
foreign_keys=[away_team_id], back_populates="away_matches", lazy="selectin"
|
|
)
|
|
stats: Mapped["MatchStats | None"] = relationship(
|
|
back_populates="match", cascade="all, delete-orphan", lazy="select"
|
|
)
|
|
predictions: Mapped[list["Prediction"]] = relationship(back_populates="match", cascade="all, delete-orphan")
|
|
|
|
__table_args__ = (
|
|
Index("ix_matches_league_date", "league_id", match_date.desc()),
|
|
Index("ix_matches_home_date", "home_team_id", match_date.desc()),
|
|
Index("ix_matches_away_date", "away_team_id", match_date.desc()),
|
|
Index("ix_matches_status_date", "match_status", match_date.desc()),
|
|
Index(
|
|
"ix_matches_unique",
|
|
"league_id",
|
|
"home_team_id",
|
|
"away_team_id",
|
|
"match_date_date",
|
|
unique=True,
|
|
),
|
|
# P0-01:比分可信度约束(替代原 ck_matches_finished_has_score):
|
|
# - score_status=known → 必须有比分(非 NULL)
|
|
# - score_status=missing → 必须 NULL(完赛缺分,禁止伪造 0:0)
|
|
# - score_status=unknown → 必须 NULL
|
|
CheckConstraint(
|
|
"score_status IN ('known', 'missing', 'unknown')",
|
|
name="ck_matches_score_status_enum",
|
|
),
|
|
CheckConstraint(
|
|
"match_status <> 'finished'"
|
|
" OR (score_status = 'known' AND home_goals IS NOT NULL AND away_goals IS NOT NULL)"
|
|
" OR (score_status IN ('missing', 'unknown') AND home_goals IS NULL AND away_goals IS NULL)",
|
|
name="ck_matches_score_integrity",
|
|
),
|
|
CheckConstraint(
|
|
"match_status IN ('finished', 'scheduled', 'in_play', 'paused', 'postponed', 'cancelled', 'suspended')",
|
|
name="ck_matches_status_enum",
|
|
),
|
|
# 半场进球 ≤ 全场进球
|
|
CheckConstraint(
|
|
"home_ht_goals IS NULL OR home_goals IS NULL OR home_ht_goals <= home_goals",
|
|
name="ck_matches_home_ht_le_full",
|
|
),
|
|
CheckConstraint(
|
|
"away_ht_goals IS NULL OR away_goals IS NULL OR away_ht_goals <= away_goals",
|
|
name="ck_matches_away_ht_le_full",
|
|
),
|
|
)
|
|
|
|
|
|
class MatchStats(Base):
|
|
__tablename__ = "match_stats"
|
|
|
|
match_id: Mapped[int] = mapped_column(ForeignKey("matches.id", ondelete="CASCADE"), primary_key=True)
|
|
home_xg: Mapped[float | None] = mapped_column(Float)
|
|
away_xg: Mapped[float | None] = mapped_column(Float)
|
|
home_shots: Mapped[int | None] = mapped_column(Integer)
|
|
away_shots: Mapped[int | None] = mapped_column(Integer)
|
|
home_shots_on_target: Mapped[int | None] = mapped_column(Integer)
|
|
away_shots_on_target: Mapped[int | None] = mapped_column(Integer)
|
|
home_corners: Mapped[int | None] = mapped_column(Integer)
|
|
away_corners: Mapped[int | None] = mapped_column(Integer)
|
|
home_possession: Mapped[float | None] = mapped_column(Float)
|
|
home_yellow_cards: Mapped[int | None] = mapped_column(Integer)
|
|
away_yellow_cards: Mapped[int | None] = mapped_column(Integer)
|
|
home_red_cards: Mapped[int | None] = mapped_column(Integer)
|
|
away_red_cards: Mapped[int | None] = mapped_column(Integer)
|
|
# bzzoiro /events/{id}/stats/ 扩展字段
|
|
home_big_chances: Mapped[int | None] = mapped_column(Integer)
|
|
away_big_chances: Mapped[int | None] = mapped_column(Integer)
|
|
home_fouls: Mapped[int | None] = mapped_column(Integer)
|
|
away_fouls: Mapped[int | None] = mapped_column(Integer)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
|
# 数据血缘:追踪统计数据的来源和可用时间
|
|
source: Mapped[str | None] = mapped_column(String(30)) # bzzoiro / understat
|
|
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
|
retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# xG 数据血缘:单独追踪 xG 字段的来源与更新时间(xG 可能独立于其他统计被更新)
|
|
xg_source: Mapped[str | None] = mapped_column(String(30))
|
|
xg_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
xg_source_record_id: Mapped[str | None] = mapped_column(String(100))
|
|
|
|
match: Mapped[Match] = relationship(back_populates="stats")
|
|
|
|
__table_args__ = (
|
|
# 数据血缘时间过滤查询用(按 available_at 取「赛前已可得」的统计)
|
|
Index("ix_match_stats_available_at", "available_at"),
|
|
)
|
|
|
|
|
|
class Standing(Base):
|
|
"""联赛积分榜快照(bzzoiro /leagues/{id}/standings/)。
|
|
|
|
P0-02: 改为追加快照(append-only)。每次采集 INSERT 新行,available_at=now;
|
|
查询取 available_at<=cutoff 的每队最新快照(DISTINCT ON team_id ORDER available_at DESC)。
|
|
回测时可还原任意历史时刻的榜单,不再只是"最新快照、忽略 cutoff"。
|
|
同一 (league_id, season, team_id, available_at) 唯一,ON CONFLICT DO NOTHING。
|
|
zone 来自 bzzoiro 分区(如 champions_league / europa_league / relegation)。
|
|
"""
|
|
__tablename__ = "standings"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
league_id: Mapped[int] = mapped_column(ForeignKey("leagues.id"), nullable=False)
|
|
season: Mapped[str] = mapped_column(String(12), nullable=False)
|
|
team_id: Mapped[int] = mapped_column(ForeignKey("teams.id"), nullable=False)
|
|
position: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
played: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
won: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
drawn: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
lost: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
goals_for: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
goals_against: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
goal_diff: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
points: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
xg_for: Mapped[float | None] = mapped_column(Float)
|
|
xg_against: Mapped[float | None] = mapped_column(Float)
|
|
form: Mapped[str | None] = mapped_column(String(20)) # 近期赛果串,如 "WWDLW"
|
|
zone: Mapped[str | None] = mapped_column(String(50)) # champions_league / relegation 等
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
|
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
# P0-02: 快照可用时间(采集时间),唯一键组成部分 + cutoff 过滤依据
|
|
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
|
|
league: Mapped[League] = relationship()
|
|
team: Mapped[Team] = relationship(lazy="selectin")
|
|
|
|
__table_args__ = (
|
|
# P0-02: (league, season, team, available_at) 唯一,支持追加快照 + ON CONFLICT DO NOTHING
|
|
UniqueConstraint("league_id", "season", "team_id", "available_at", name="uq_standings_league_season_team_available"),
|
|
Index("ix_standings_league_season_pos", "league_id", "season", "position"),
|
|
Index("ix_standings_league_season_team_available", "league_id", "season", "team_id", "available_at"),
|
|
)
|
|
|
|
|
|
class Prediction(Base):
|
|
__tablename__ = "predictions"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
match_id: Mapped[int] = mapped_column(ForeignKey("matches.id", ondelete="CASCADE"), nullable=False)
|
|
provider: Mapped[str] = mapped_column(String(30), nullable=False)
|
|
model: Mapped[str] = mapped_column(String(80), nullable=False)
|
|
prompt_version: Mapped[str] = mapped_column(String(20), nullable=False, default="v1")
|
|
prompt_tokens: Mapped[int | None] = mapped_column(Integer)
|
|
completion_tokens: Mapped[int | None] = mapped_column(Integer)
|
|
latency_ms: Mapped[int | None] = mapped_column(Integer)
|
|
pred_home_goals: Mapped[float | None] = mapped_column(Float)
|
|
pred_away_goals: Mapped[float | None] = mapped_column(Float)
|
|
# 备选比分(次可能比分,可空)
|
|
alt_pred_home_goals: Mapped[int | None] = mapped_column(Integer)
|
|
alt_pred_away_goals: Mapped[int | None] = mapped_column(Integer)
|
|
pred_1x2: Mapped[str | None] = mapped_column(String(3))
|
|
subjective_confidence: Mapped[float | None] = mapped_column(Float) # LLM 主观置信度,非概率
|
|
reasoning: Mapped[str | None] = mapped_column(Text)
|
|
raw_response: Mapped[dict | None] = mapped_column(JSONB)
|
|
# multi-agent 模式: 各专家报告
|
|
mode: Mapped[str] = mapped_column(String(20), nullable=False, default="single")
|
|
# D5: multi-agent 模式存各专家报告列表(list[dict]);历史数据/兼容路径可能存 dict。
|
|
# 仅修正类型标注与真实 JSON 形状一致,列类型(JSONB)与数据不变。
|
|
agent_outputs: Mapped[list[dict] | dict | None] = mapped_column(JSONB)
|
|
# Fix: agent_weights 独立持久化到列(原本只在 raw_response 中)
|
|
agent_weights: Mapped[dict | None] = mapped_column(JSONB)
|
|
# 预测状态: success / failed / degraded
|
|
status: Mapped[str] = mapped_column(String(20), nullable=False, default="success")
|
|
# Fix: run_type 区分实盘(live)与回测(backtest),避免回测覆盖实盘预测
|
|
run_type: Mapped[str] = mapped_column(String(10), nullable=False, default="live")
|
|
# 时间语义:区分比赛时间、预测创建时间、数据截止时间
|
|
match_kickoff_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
prediction_created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
prediction_cutoff_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
input_hash: Mapped[str | None] = mapped_column(String(64))
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
actual_home_goals: Mapped[int | None] = mapped_column(Integer)
|
|
actual_away_goals: Mapped[int | None] = mapped_column(Integer)
|
|
settled: Mapped[bool] = mapped_column(Boolean, default=False)
|
|
|
|
match: Mapped[Match] = relationship(back_populates="predictions")
|
|
|
|
__table_args__ = (
|
|
# P0-03: 幂等指纹——input_hash 非空时唯一(同指纹→返回已有行,不 UPDATE/INSERT);
|
|
# 兼容旧数据 NULL input_hash(不强制回填)。
|
|
Index(
|
|
"ix_predictions_input_hash_unique", "input_hash", unique=True,
|
|
postgresql_where=column("input_hash").isnot(None),
|
|
),
|
|
Index("ix_predictions_match", "match_id"),
|
|
# 数据截止时间过滤查询用(按 prediction_cutoff_at 取「赛前已生成」的预测)
|
|
Index("ix_predictions_cutoff_at", "prediction_cutoff_at"),
|
|
# 数据库级约束:最后一道防线
|
|
CheckConstraint("pred_home_goals >= 0", name="ck_pred_home_goals_nonneg"),
|
|
CheckConstraint("pred_away_goals >= 0", name="ck_pred_away_goals_nonneg"),
|
|
CheckConstraint("subjective_confidence >= 0 AND subjective_confidence <= 1", name="ck_confidence_range"),
|
|
CheckConstraint("pred_1x2 IN ('1', 'X', '2')", name="ck_pred_1x2_enum"),
|
|
CheckConstraint("mode IN ('single', 'multi', 'baseline')", name="ck_mode_enum"),
|
|
CheckConstraint("status IN ('success', 'failed', 'degraded')", name="ck_status_enum"),
|
|
CheckConstraint("run_type IN ('live', 'backtest')", name="ck_run_type_enum"),
|
|
)
|
|
|
|
|
|
class AppSetting(Base):
|
|
"""后台管理的运行时设置(如数据源 API Key),读取时优先于 .env 默认值。"""
|
|
__tablename__ = "app_settings"
|
|
|
|
key: Mapped[str] = mapped_column(String(100), primary_key=True)
|
|
value: Mapped[str] = mapped_column(Text, nullable=False)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
|
|
|
|
|
class Schedule(Base):
|
|
"""定时采集任务配置。"""
|
|
__tablename__ = "schedules"
|
|
|
|
id: Mapped[str] = mapped_column(String(50), primary_key=True)
|
|
task: Mapped[str] = mapped_column(String(20), nullable=False) # events / standings / stats / all
|
|
cron: Mapped[str] = mapped_column(String(100), nullable=False) # cron 表达式
|
|
leagues: Mapped[str | None] = mapped_column(Text) # 逗号分隔的联赛代码,空=全部
|
|
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
last_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
last_status: Mapped[str | None] = mapped_column(String(20)) # success / failed
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
|
|
|
|
|
# ── 数据管线基础设施(对应迁移 0008) ──────────────────────────────────
|
|
# Bronze 层、死信、质量监控、血缘追踪 4 张表。
|
|
|
|
|
|
class RawEvent(Base):
|
|
"""Bronze 层:采集到的原始事件存档,便于重放与审计。
|
|
|
|
⚠️ 预留未启用:当前 bzzoiro 管线不写入此表。
|
|
未来接线计划:events 采集成功后写入 raw_payload,支持重放与审计。
|
|
"""
|
|
__tablename__ = "raw_events"
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
raw_payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
|
|
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
ingest_batch_id: Mapped[str | None] = mapped_column(String(36))
|
|
|
|
__table_args__ = (
|
|
UniqueConstraint("source_system", "source_record_id", name="uq_raw_event"),
|
|
Index("ix_raw_event_batch", "ingest_batch_id"),
|
|
)
|
|
|
|
|
|
class IngestJob(Base):
|
|
"""采集任务状态:跟踪每次触后台采集任务的执行进度与结果。
|
|
|
|
POST /api/v1/ingest/bzzoiro 触发时写入(pending→running→success/failed),
|
|
前端 Collection 页据此轮询到终态,替代此前"30 秒后盲标完成"的模拟。
|
|
分批 get_uow / BzzoiroSource / IngestFailure / Bronze/Lineage 均不受影响
|
|
(本表仅作状态追踪,不介入采集事务)。
|
|
"""
|
|
__tablename__ = "ingest_jobs"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True) # uuid4
|
|
task: Mapped[str] = mapped_column(String(20), nullable=False)
|
|
params: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
|
|
status: Mapped[str] = mapped_column(String(20), nullable=False, server_default="pending")
|
|
result: Mapped[dict | None] = mapped_column(JSONB)
|
|
error: Mapped[str | None] = mapped_column(Text)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
__table_args__ = (
|
|
Index("ix_ingest_job_status_created", "status", "created_at"),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'running', 'success', 'failed')",
|
|
name="ck_ingest_job_status",
|
|
),
|
|
)
|
|
|
|
|
|
class IngestFailure(Base):
|
|
"""采集失败死信:记录失败原因、重试次数与下次重试时间。
|
|
|
|
bzzoiro 三条管线(events / standings / stats)抓取失败时经由
|
|
bzzoiro._safe_write_ingest_failure 写入本表(尽力而为,写入失败
|
|
不影响采集主流程)。admin 可通过 /admin/schedules/ingest-failures
|
|
查看与重试。
|
|
"""
|
|
__tablename__ = "ingest_failures"
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
|
error_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
error_detail: Mapped[str | None] = mapped_column(Text)
|
|
raw_payload: Mapped[dict | None] = mapped_column(JSONB)
|
|
retry_count: Mapped[int] = mapped_column(Integer, server_default="0")
|
|
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
status: Mapped[str] = mapped_column(String(20), server_default="pending")
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
__table_args__ = (
|
|
Index("ix_ingest_failure_status", "status", "next_retry_at"),
|
|
CheckConstraint(
|
|
"status IN ('pending', 'retrying', 'resolved', 'abandoned')",
|
|
name="ck_ingest_failure_status",
|
|
),
|
|
)
|
|
|
|
|
|
class DataQualityCheck(Base):
|
|
"""数据质量监控:记录每次质量检查的结果。
|
|
|
|
⚠️ 预留未启用:当前 bzzoiro 管线不写入此表。
|
|
未来接线计划:定时检查比赛/统计/积分榜完整性,写入检查结果。
|
|
"""
|
|
__tablename__ = "data_quality_checks"
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
check_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
entity_id: Mapped[int | None] = mapped_column(Integer)
|
|
expected_value: Mapped[float | None] = mapped_column(Float)
|
|
actual_value: Mapped[float] = mapped_column(Float, nullable=False)
|
|
passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
|
severity: Mapped[str] = mapped_column(String(10), server_default="warning")
|
|
detail: Mapped[dict | None] = mapped_column(JSONB)
|
|
checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
|
|
__table_args__ = (
|
|
Index("ix_dqc_checked_at", "checked_at"),
|
|
Index("ix_dqc_entity", "entity_type", "entity_id"),
|
|
)
|
|
|
|
|
|
class DataLineage(Base):
|
|
"""ETL 血缘追踪:记录从源到目标的转换过程。
|
|
|
|
⚠️ 预留未启用:当前 bzzoiro 管线不写入此表。
|
|
未来接线计划:每次采集写入 source_record_id → target_table/id 映射。
|
|
"""
|
|
__tablename__ = "data_lineage"
|
|
|
|
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
|
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
|
target_table: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
target_id: Mapped[int | None] = mapped_column(Integer)
|
|
transform_name: Mapped[str] = mapped_column(String(50), nullable=False)
|
|
transform_detail: Mapped[dict | None] = mapped_column(JSONB)
|
|
batch_id: Mapped[str | None] = mapped_column(String(36))
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
|
|
__table_args__ = (
|
|
Index("ix_lineage_source", "source_system", "source_record_id"),
|
|
Index("ix_lineage_target", "target_table", "target_id"),
|
|
Index("ix_lineage_batch", "batch_id"),
|
|
)
|