fix: 全方位审查问题修复(P0~P3)

P0: ORM-迁移同步
- 新增 RawEvent/IngestFailure/DataQualityCheck/DataLineage 4 个模型类
- MatchStats 补充 xg_source/xg_updated_at/xg_source_record_id 字段
- 修复 server_default=_utcnow → func.now()(4 处)
- BigInteger 导入

P1:
- log_buffer.py 移除 threading.Lock(asyncio 单线程下无需锁)
- 确认 context_builder/understat/injuries 等已有修复

P2:
- Dockerfile 新增非 root 用户 + .dockerignore
- 前端 fetchDashboard 修复 total 字段(改用 items.length)
- 前端 fetchSystemConfig 改用真实 /admin/settings 端点
- 修复 validation.py return self"" → return self

P3:
- predict.py 缓存加 _CACHE_MAX_SIZE=200 淘汰
- eval.py get_eval_summary 加 limit 参数(默认 1000)+ 返回 total_settled
- orchestrator.py agent provider 配置缓存 60s
This commit is contained in:
Profeto Agent
2026-09-19 04:38:53 +00:00
parent 786f10aa11
commit 11efe91ce9
10 changed files with 209 additions and 32 deletions
+94
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
from datetime import date, datetime, timezone
from sqlalchemy import (
BigInteger,
Boolean,
CheckConstraint,
Date,
@@ -131,6 +132,10 @@ class MatchStats(Base):
source_record_id: Mapped[str | None] = mapped_column(String(100))
retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
# xG 数据血缘:单独追踪 xG 字段的来源与更新时间(xG 可能独立于其他统计被更新)
xg_source: Mapped[str | None] = mapped_column(String(30))
xg_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
xg_source_record_id: Mapped[str | None] = mapped_column(String(100))
match: Mapped[Match] = relationship(back_populates="stats")
@@ -228,3 +233,92 @@ class AppSetting(Base):
key: Mapped[str] = mapped_column(String(100), primary_key=True)
value: Mapped[str] = mapped_column(Text, nullable=False)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
# ── 数据管线基础设施(对应迁移 0008) ──────────────────────────────────
# Bronze 层、死信、质量监控、血缘追踪 4 张表。
class RawEvent(Base):
"""Bronze 层:采集到的原始事件存档,便于重放与审计。"""
__tablename__ = "raw_events"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
raw_payload: Mapped[dict] = mapped_column(JSONB, nullable=False)
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
ingest_batch_id: Mapped[str | None] = mapped_column(String(36))
__table_args__ = (
UniqueConstraint("source_system", "source_record_id", name="uq_raw_event"),
Index("ix_raw_event_batch", "ingest_batch_id"),
)
class IngestFailure(Base):
"""采集失败死信:记录失败原因、重试次数与下次重试时间。"""
__tablename__ = "ingest_failures"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
source_record_id: Mapped[str | None] = mapped_column(String(100))
error_type: Mapped[str] = mapped_column(String(50), nullable=False)
error_detail: Mapped[str | None] = mapped_column(Text)
raw_payload: Mapped[dict | None] = mapped_column(JSONB)
retry_count: Mapped[int] = mapped_column(Integer, server_default="0")
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
status: Mapped[str] = mapped_column(String(20), server_default="pending")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
__table_args__ = (
Index("ix_ingest_failure_status", "status", "next_retry_at"),
CheckConstraint(
"status IN ('pending', 'retrying', 'resolved', 'abandoned')",
name="ck_ingest_failure_status",
),
)
class DataQualityCheck(Base):
"""数据质量监控:记录每次质量检查的结果。"""
__tablename__ = "data_quality_checks"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
check_name: Mapped[str] = mapped_column(String(100), nullable=False)
entity_type: Mapped[str] = mapped_column(String(50), nullable=False)
entity_id: Mapped[int | None] = mapped_column(Integer)
expected_value: Mapped[float | None] = mapped_column(Float)
actual_value: Mapped[float] = mapped_column(Float, nullable=False)
passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
severity: Mapped[str] = mapped_column(String(10), server_default="warning")
detail: Mapped[dict | None] = mapped_column(JSONB)
checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
__table_args__ = (
Index("ix_dqc_checked_at", "checked_at"),
Index("ix_dqc_entity", "entity_type", "entity_id"),
)
class DataLineage(Base):
"""ETL 血缘追踪:记录从源到目标的转换过程。"""
__tablename__ = "data_lineage"
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
source_system: Mapped[str] = mapped_column(String(50), nullable=False)
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
target_table: Mapped[str] = mapped_column(String(50), nullable=False)
target_id: Mapped[int | None] = mapped_column(Integer)
transform_name: Mapped[str] = mapped_column(String(50), nullable=False)
transform_detail: Mapped[dict | None] = mapped_column(JSONB)
batch_id: Mapped[str | None] = mapped_column(String(36))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
__table_args__ = (
Index("ix_lineage_source", "source_system", "source_record_id"),
Index("ix_lineage_target", "target_table", "target_id"),
Index("ix_lineage_batch", "batch_id"),
)