fix: 全方位审查问题修复(P0~P3)
P0: ORM-迁移同步 - 新增 RawEvent/IngestFailure/DataQualityCheck/DataLineage 4 个模型类 - MatchStats 补充 xg_source/xg_updated_at/xg_source_record_id 字段 - 修复 server_default=_utcnow → func.now()(4 处) - BigInteger 导入 P1: - log_buffer.py 移除 threading.Lock(asyncio 单线程下无需锁) - 确认 context_builder/understat/injuries 等已有修复 P2: - Dockerfile 新增非 root 用户 + .dockerignore - 前端 fetchDashboard 修复 total 字段(改用 items.length) - 前端 fetchSystemConfig 改用真实 /admin/settings 端点 - 修复 validation.py return self"" → return self P3: - predict.py 缓存加 _CACHE_MAX_SIZE=200 淘汰 - eval.py get_eval_summary 加 limit 参数(默认 1000)+ 返回 total_settled - orchestrator.py agent provider 配置缓存 60s
This commit is contained in:
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Date,
|
||||
@@ -131,6 +132,10 @@ class MatchStats(Base):
|
||||
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
||||
retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# xG 数据血缘:单独追踪 xG 字段的来源与更新时间(xG 可能独立于其他统计被更新)
|
||||
xg_source: Mapped[str | None] = mapped_column(String(30))
|
||||
xg_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
xg_source_record_id: Mapped[str | None] = mapped_column(String(100))
|
||||
|
||||
match: Mapped[Match] = relationship(back_populates="stats")
|
||||
|
||||
@@ -228,3 +233,92 @@ class AppSetting(Base):
|
||||
key: Mapped[str] = mapped_column(String(100), primary_key=True)
|
||||
value: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
# ── 数据管线基础设施(对应迁移 0008) ──────────────────────────────────
|
||||
# Bronze 层、死信、质量监控、血缘追踪 4 张表。
|
||||
|
||||
|
||||
class RawEvent(Base):
|
||||
"""Bronze 层:采集到的原始事件存档,便于重放与审计。"""
|
||||
__tablename__ = "raw_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
raw_payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
|
||||
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
ingest_batch_id: Mapped[str | None] = mapped_column(String(36))
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_system", "source_record_id", name="uq_raw_event"),
|
||||
Index("ix_raw_event_batch", "ingest_batch_id"),
|
||||
)
|
||||
|
||||
|
||||
class IngestFailure(Base):
|
||||
"""采集失败死信:记录失败原因、重试次数与下次重试时间。"""
|
||||
__tablename__ = "ingest_failures"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
||||
error_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
error_detail: Mapped[str | None] = mapped_column(Text)
|
||||
raw_payload: Mapped[dict | None] = mapped_column(JSONB)
|
||||
retry_count: Mapped[int] = mapped_column(Integer, server_default="0")
|
||||
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
status: Mapped[str] = mapped_column(String(20), server_default="pending")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_ingest_failure_status", "status", "next_retry_at"),
|
||||
CheckConstraint(
|
||||
"status IN ('pending', 'retrying', 'resolved', 'abandoned')",
|
||||
name="ck_ingest_failure_status",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class DataQualityCheck(Base):
|
||||
"""数据质量监控:记录每次质量检查的结果。"""
|
||||
__tablename__ = "data_quality_checks"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
check_name: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
entity_id: Mapped[int | None] = mapped_column(Integer)
|
||||
expected_value: Mapped[float | None] = mapped_column(Float)
|
||||
actual_value: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||
severity: Mapped[str] = mapped_column(String(10), server_default="warning")
|
||||
detail: Mapped[dict | None] = mapped_column(JSONB)
|
||||
checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_dqc_checked_at", "checked_at"),
|
||||
Index("ix_dqc_entity", "entity_type", "entity_id"),
|
||||
)
|
||||
|
||||
|
||||
class DataLineage(Base):
|
||||
"""ETL 血缘追踪:记录从源到目标的转换过程。"""
|
||||
__tablename__ = "data_lineage"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||
target_table: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
target_id: Mapped[int | None] = mapped_column(Integer)
|
||||
transform_name: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
transform_detail: Mapped[dict | None] = mapped_column(JSONB)
|
||||
batch_id: Mapped[str | None] = mapped_column(String(36))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_lineage_source", "source_system", "source_record_id"),
|
||||
Index("ix_lineage_target", "target_table", "target_id"),
|
||||
Index("ix_lineage_batch", "batch_id"),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user