Compare commits
18
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a0ce2e2a6a | ||
|
|
5de8ffb09d | ||
|
|
2fd8a80ad9 | ||
|
|
5676198392 | ||
|
|
ff0045ad93 | ||
|
|
983b620659 | ||
|
|
fa69795d69 | ||
|
|
c5f92c9a54 | ||
|
|
7df46544b8 | ||
|
|
d847f4f3f4 | ||
|
|
29e718962f | ||
|
|
e3cacc35e4 | ||
|
|
77d01450b1 | ||
|
|
acea7e699d | ||
|
|
f965650c10 | ||
|
|
c89bfe2af7 | ||
|
|
71bf723a10 | ||
|
|
235fb0de97 |
@@ -3,6 +3,10 @@ APP_ENV=development
|
|||||||
LOG_LEVEL=INFO
|
LOG_LEVEL=INFO
|
||||||
|
|
||||||
# ---- 数据库 ----
|
# ---- 数据库 ----
|
||||||
|
POSTGRES_USER=football
|
||||||
|
POSTGRES_PASSWORD=football
|
||||||
|
POSTGRES_DB=football
|
||||||
|
POSTGRES_PORT=5432
|
||||||
DATABASE_URL=postgresql+asyncpg://football:football@localhost:5432/football
|
DATABASE_URL=postgresql+asyncpg://football:football@localhost:5432/football
|
||||||
|
|
||||||
# ---- LLM (OpenAI-compatible,必填一个) ----
|
# ---- LLM (OpenAI-compatible,必填一个) ----
|
||||||
@@ -10,6 +14,11 @@ LLM_PROVIDER=openai
|
|||||||
LLM_API_KEY=sk-xxxx
|
LLM_API_KEY=sk-xxxx
|
||||||
LLM_BASE_URL=https://api.openai.com/v1
|
LLM_BASE_URL=https://api.openai.com/v1
|
||||||
LLM_MODEL=gpt-4o
|
LLM_MODEL=gpt-4o
|
||||||
|
# 多 Agent 分档模型(留空则回落 LLM_MODEL)
|
||||||
|
LLM_SPECIALIST_MODEL=
|
||||||
|
LLM_AGGREGATOR_MODEL=
|
||||||
|
# 单次 LLM 调用超时(秒)
|
||||||
|
LLM_TIMEOUT=60
|
||||||
|
|
||||||
# ---- 数据源 ----
|
# ---- 数据源 ----
|
||||||
BZZOIRO_KEY=
|
BZZOIRO_KEY=
|
||||||
@@ -17,3 +26,8 @@ API_FOOTBALL_KEY=
|
|||||||
|
|
||||||
# ---- CORS ----
|
# ---- CORS ----
|
||||||
CORS_ORIGINS=http://localhost:5173,http://localhost:3000
|
CORS_ORIGINS=http://localhost:5173,http://localhost:3000
|
||||||
|
|
||||||
|
# ---- 管理接口鉴权 ----
|
||||||
|
# 采集/回测/回填接口的访问密钥(请求头 X-API-Key)。
|
||||||
|
# 留空 = 不启用鉴权(本地开发默认);生产环境必须设置强随机值。
|
||||||
|
ADMIN_API_KEY=
|
||||||
|
|||||||
@@ -10,3 +10,7 @@ frontend/dist/
|
|||||||
CLAUDE.md
|
CLAUDE.md
|
||||||
docs/AGENTS.md
|
docs/AGENTS.md
|
||||||
docs/agents/
|
docs/agents/
|
||||||
|
|
||||||
|
# 本地审查/预览脚手架(不入库)
|
||||||
|
.tools/
|
||||||
|
.preview/
|
||||||
|
|||||||
@@ -26,6 +26,21 @@ def upgrade() -> None:
|
|||||||
op.add_column('predictions', sa.Column('input_hash', sa.String(length=64), nullable=True))
|
op.add_column('predictions', sa.Column('input_hash', sa.String(length=64), nullable=True))
|
||||||
|
|
||||||
# 3. 新增 CHECK 约束
|
# 3. 新增 CHECK 约束
|
||||||
|
# 注意: 对已有数据行加 CHECK 约束时,若存在越界数据 ALTER TABLE 会中途失败,
|
||||||
|
# 导致迁移卡在半完成状态。这里先做一次性清洗(把越界值收敛到合法域),
|
||||||
|
# 再建约束,保证在非空库上也能成功。
|
||||||
|
op.execute("UPDATE predictions SET pred_home_goals = 0 WHERE pred_home_goals IS NOT NULL AND pred_home_goals < 0")
|
||||||
|
op.execute("UPDATE predictions SET pred_away_goals = 0 WHERE pred_away_goals IS NOT NULL AND pred_away_goals < 0")
|
||||||
|
op.execute(
|
||||||
|
"UPDATE predictions SET subjective_confidence = "
|
||||||
|
"CASE WHEN subjective_confidence < 0 THEN 0 "
|
||||||
|
" WHEN subjective_confidence > 1 THEN 1 ELSE subjective_confidence END "
|
||||||
|
"WHERE subjective_confidence IS NOT NULL "
|
||||||
|
" AND (subjective_confidence < 0 OR subjective_confidence > 1)"
|
||||||
|
)
|
||||||
|
op.execute("UPDATE predictions SET pred_1x2 = NULL WHERE pred_1x2 IS NOT NULL AND pred_1x2 NOT IN ('1', 'X', '2')")
|
||||||
|
op.execute("UPDATE predictions SET mode = 'single' WHERE mode IS NOT NULL AND mode NOT IN ('single', 'multi')")
|
||||||
|
|
||||||
op.create_check_constraint('ck_pred_home_goals_nonneg', 'predictions', 'pred_home_goals >= 0')
|
op.create_check_constraint('ck_pred_home_goals_nonneg', 'predictions', 'pred_home_goals >= 0')
|
||||||
op.create_check_constraint('ck_pred_away_goals_nonneg', 'predictions', 'pred_away_goals >= 0')
|
op.create_check_constraint('ck_pred_away_goals_nonneg', 'predictions', 'pred_away_goals >= 0')
|
||||||
op.create_check_constraint('ck_confidence_range', 'predictions', 'subjective_confidence >= 0 AND subjective_confidence <= 1')
|
op.create_check_constraint('ck_confidence_range', 'predictions', 'subjective_confidence >= 0 AND subjective_confidence <= 1')
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""清理 schema 与 ORM 模型的漂移
|
||||||
|
|
||||||
|
Revision ID: 0006_schema_model_drift_cleanup
|
||||||
|
Revises: 0005_prediction_status_and_stats_provenance
|
||||||
|
Create Date: 2026-09-15
|
||||||
|
|
||||||
|
背景(见代码审查报告 P1-1):
|
||||||
|
0005 迁移在 predictions 表留下了 `cutoff_at` 列(删除语句被注释掉),
|
||||||
|
但 ORM 模型 `Prediction` 中并无该字段 —— 这是一处会长期存在的 schema 漂移,
|
||||||
|
而 alembic autogenerate 会持续建议 drop 它,造成噪声。
|
||||||
|
|
||||||
|
同时 0005 创建的两个索引 `ix_match_stats_available_at` 与
|
||||||
|
`ix_predictions_cutoff_at` 未在 ORM 声明,autogenerate 会建议删除它们 ——
|
||||||
|
一旦被误删,数据血缘相关的时间过滤查询会退化为全表扫描。
|
||||||
|
|
||||||
|
本迁移做两件事:
|
||||||
|
1. drop 掉幽灵列 predictions.cutoff_at(数据已由 prediction_cutoff_at 承载,
|
||||||
|
迁移前先把非空值回填过去,避免丢数据)
|
||||||
|
2. 显式重建这两个索引(幂等:先 drop if exists 再 create),使 DB 状态与
|
||||||
|
修正后的 ORM 声明一致
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '0006_schema_model_drift_cleanup'
|
||||||
|
down_revision: Union[str, None] = '0005_prediction_status_and_stats_provenance'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
|
||||||
|
# --- 1. 幽灵列 cutoff_at: 先把数据回填到 prediction_cutoff_at 再删除 ---
|
||||||
|
pred_cols = {c["name"] for c in inspector.get_columns("predictions")}
|
||||||
|
if "cutoff_at" in pred_cols:
|
||||||
|
if "prediction_cutoff_at" in pred_cols:
|
||||||
|
# 仅回填尚未有值的行,避免覆盖更权威的数据
|
||||||
|
op.execute(
|
||||||
|
"UPDATE predictions "
|
||||||
|
"SET prediction_cutoff_at = cutoff_at "
|
||||||
|
"WHERE prediction_cutoff_at IS NULL AND cutoff_at IS NOT NULL"
|
||||||
|
)
|
||||||
|
op.drop_column("predictions", "cutoff_at")
|
||||||
|
|
||||||
|
# --- 2. 与 ORM 声明对齐的索引(幂等重建) ---
|
||||||
|
stats_idx = {i["name"] for i in inspector.get_indexes("match_stats")}
|
||||||
|
if "ix_match_stats_available_at" not in stats_idx:
|
||||||
|
op.create_index("ix_match_stats_available_at", "match_stats", ["available_at"])
|
||||||
|
|
||||||
|
pred_idx = {i["name"] for i in inspector.get_indexes("predictions")}
|
||||||
|
if "ix_predictions_cutoff_at" not in pred_idx:
|
||||||
|
op.create_index(
|
||||||
|
"ix_predictions_cutoff_at", "predictions", ["prediction_cutoff_at"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
|
||||||
|
# 恢复幽灵列(与 0005 的最终状态一致:列存在但无值)
|
||||||
|
pred_cols = {c["name"] for c in inspector.get_columns("predictions")}
|
||||||
|
if "cutoff_at" not in pred_cols:
|
||||||
|
op.add_column(
|
||||||
|
"predictions",
|
||||||
|
sa.Column("cutoff_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
stats_idx = {i["name"] for i in inspector.get_indexes("match_stats")}
|
||||||
|
if "ix_match_stats_available_at" in stats_idx:
|
||||||
|
op.drop_index("ix_match_stats_available_at", table_name="match_stats")
|
||||||
|
|
||||||
|
pred_idx = {i["name"] for i in inspector.get_indexes("predictions")}
|
||||||
|
if "ix_predictions_cutoff_at" in pred_idx:
|
||||||
|
op.drop_index("ix_predictions_cutoff_at", table_name="predictions")
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""修复 injuries 表约束命名与 ORM 声明不一致
|
||||||
|
|
||||||
|
Revision ID: 0007_injuries_constraint_naming_align
|
||||||
|
Revises: 0006_schema_model_drift_cleanup
|
||||||
|
Create Date: 2026-09-16
|
||||||
|
|
||||||
|
背景(见代码审查报告 P2-5):
|
||||||
|
0003 迁移使用 sa.UniqueConstraint 创建唯一约束,
|
||||||
|
而 ORM models.py 中声明为 Index(..., unique=True)。
|
||||||
|
虽然 PostgreSQL 中两者效果相同(都保证唯一性),
|
||||||
|
但 pg_catalog 中表示不同,会导致:
|
||||||
|
- alembic autogenerate 持续报告漂移
|
||||||
|
- 约束命名约定不一致(uc_ 前缀 vs ix_ 前缀)
|
||||||
|
|
||||||
|
本迁移将 UniqueConstraint 替换为唯一索引,与 ORM 声明对齐。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '0007_injuries_constraint_naming_align'
|
||||||
|
down_revision: Union[str, None] = '0006_schema_model_drift_cleanup'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
|
||||||
|
# 检查当前约束类型
|
||||||
|
constraints = {
|
||||||
|
c["name"]: c
|
||||||
|
for c in inspector.get_unique_constraints("injuries")
|
||||||
|
}
|
||||||
|
indexes = {
|
||||||
|
i["name"]: i
|
||||||
|
for i in inspector.get_indexes("injuries")
|
||||||
|
}
|
||||||
|
|
||||||
|
# 如果存在 UniqueConstraint 形式的 ix_injuries_player_fixture,替换为唯一索引
|
||||||
|
if "ix_injuries_player_fixture" in constraints:
|
||||||
|
# 删除唯一约束
|
||||||
|
op.drop_constraint("ix_injuries_player_fixture", "injuries", type_="unique")
|
||||||
|
|
||||||
|
# 如果不存在同名唯一索引,创建它(与 ORM 声明一致)
|
||||||
|
if "ix_injuries_player_fixture" not in indexes:
|
||||||
|
op.create_index(
|
||||||
|
"ix_injuries_player_fixture",
|
||||||
|
"injuries",
|
||||||
|
["player_id", "fixture_id", "injury_type"],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
bind = op.get_bind()
|
||||||
|
inspector = sa.inspect(bind)
|
||||||
|
|
||||||
|
indexes = {
|
||||||
|
i["name"]: i
|
||||||
|
for i in inspector.get_indexes("injuries")
|
||||||
|
}
|
||||||
|
constraints = {
|
||||||
|
c["name"]: c
|
||||||
|
for c in inspector.get_unique_constraints("injuries")
|
||||||
|
}
|
||||||
|
|
||||||
|
# 恢复为 UniqueConstraint 形式
|
||||||
|
if "ix_injuries_player_fixture" in indexes:
|
||||||
|
op.drop_index("ix_injuries_player_fixture", table_name="injuries")
|
||||||
|
|
||||||
|
if "ix_injuries_player_fixture" not in constraints:
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"ix_injuries_player_fixture",
|
||||||
|
"injuries",
|
||||||
|
["player_id", "fixture_id", "injury_type"],
|
||||||
|
)
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
"""为 predictions 表添加 match_id+provider+model 唯一约束
|
||||||
|
|
||||||
|
Revision ID: 0007_predictions_unique_constraint
|
||||||
|
Revises: 0007_injuries_constraint_naming_align
|
||||||
|
Create Date: 2026-09-16
|
||||||
|
|
||||||
|
背景(见代码审查报告 P1-6):
|
||||||
|
同一 match_id + provider + model 组合不应产生重复预测。
|
||||||
|
当前缺少数据库级唯一约束,回测多次运行或并发采集可能产生重复记录,
|
||||||
|
导致统计偏差。
|
||||||
|
|
||||||
|
先清理已存在的重复记录(保留最早创建的那条),再添加唯一约束。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '0007_predictions_unique_constraint'
|
||||||
|
down_revision: Union[str, None] = '0007_injuries_constraint_naming_align'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# 1. 清理已存在的重复记录(保留 id 最小的)
|
||||||
|
op.execute(
|
||||||
|
"""
|
||||||
|
DELETE FROM predictions
|
||||||
|
WHERE id NOT IN (
|
||||||
|
SELECT MIN(id)
|
||||||
|
FROM predictions
|
||||||
|
GROUP BY match_id, provider, model
|
||||||
|
)
|
||||||
|
AND match_id IN (
|
||||||
|
SELECT match_id
|
||||||
|
FROM predictions
|
||||||
|
GROUP BY match_id, provider, model
|
||||||
|
HAVING COUNT(*) > 1
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. 添加唯一约束
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_predictions_match_provider_model",
|
||||||
|
"predictions",
|
||||||
|
["match_id", "provider", "model"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint(
|
||||||
|
"uq_predictions_match_provider_model",
|
||||||
|
"predictions",
|
||||||
|
type_="unique",
|
||||||
|
)
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
"""新增 Bronze 层 + 死信表 + 数据质量表 + 血缘表
|
||||||
|
|
||||||
|
Revision ID: 0008_raw_event_and_ingest_failure
|
||||||
|
Revises: 0007_predictions_unique_constraint
|
||||||
|
Create Date: 2026-09-17
|
||||||
|
|
||||||
|
架构审查报告 P1 实施:
|
||||||
|
- raw_events: Bronze 层,不可变原始采集记录
|
||||||
|
- ingest_failures: 采集失败死信表
|
||||||
|
- data_quality_checks: 数据质量检查结果记录
|
||||||
|
- data_lineage: ETL 全过程元数据血缘追踪
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '0008_raw_event_and_ingest_failure'
|
||||||
|
down_revision: Union[str, None] = '0007_predictions_unique_constraint'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# ── raw_events: Bronze 层原始记录 ──
|
||||||
|
op.create_table(
|
||||||
|
"raw_events",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("source_system", sa.String(30), nullable=False),
|
||||||
|
sa.Column("source_record_id", sa.String(100), nullable=False),
|
||||||
|
sa.Column("raw_payload", JSONB(), nullable=False),
|
||||||
|
sa.Column("ingested_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("ingest_batch_id", sa.String(64), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index("ix_raw_events_batch", "raw_events", ["ingest_batch_id"])
|
||||||
|
op.create_index("ix_raw_events_source_ingested", "raw_events", ["source_system", "ingested_at"])
|
||||||
|
op.create_unique_constraint("uq_raw_events_source_record", "raw_events", ["source_system", "source_record_id"])
|
||||||
|
|
||||||
|
# ── ingest_failures: 采集失败死信表 ──
|
||||||
|
op.create_table(
|
||||||
|
"ingest_failures",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("source_system", sa.String(30), nullable=False),
|
||||||
|
sa.Column("entity_type", sa.String(30), nullable=False),
|
||||||
|
sa.Column("source_record_id", sa.String(100), nullable=True),
|
||||||
|
sa.Column("error_type", sa.String(50), nullable=False),
|
||||||
|
sa.Column("error_detail", sa.Text(), nullable=True),
|
||||||
|
sa.Column("raw_payload", JSONB(), nullable=True),
|
||||||
|
sa.Column("retry_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
sa.Column("next_retry_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index("ix_ingest_failures_status_next_retry", "ingest_failures", ["status", "next_retry_at"])
|
||||||
|
op.create_index("ix_ingest_failures_source", "ingest_failures", ["source_system", "entity_type"])
|
||||||
|
op.create_check_constraint("ck_ingest_failures_status", "ingest_failures",
|
||||||
|
"status IN ('pending', 'retrying', 'resolved', 'abandoned')")
|
||||||
|
op.create_check_constraint("ck_ingest_failures_retry_nonneg", "ingest_failures", "retry_count >= 0")
|
||||||
|
|
||||||
|
# ── data_quality_checks: 数据质量检查记录 ──
|
||||||
|
op.create_table(
|
||||||
|
"data_quality_checks",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("check_name", sa.String(100), nullable=False),
|
||||||
|
sa.Column("entity_type", sa.String(30), nullable=False),
|
||||||
|
sa.Column("entity_id", sa.String(50), nullable=True),
|
||||||
|
sa.Column("expected_value", sa.Text(), nullable=True),
|
||||||
|
sa.Column("actual_value", sa.Text(), nullable=True),
|
||||||
|
sa.Column("passed", sa.Boolean(), nullable=False),
|
||||||
|
sa.Column("severity", sa.String(10), nullable=False, server_default="warning"),
|
||||||
|
sa.Column("detail", sa.Text(), nullable=True),
|
||||||
|
sa.Column("checked_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_index("ix_dqc_check_time", "data_quality_checks", ["check_name", "checked_at"])
|
||||||
|
op.create_index("ix_dqc_entity", "data_quality_checks", ["entity_type", "entity_id"])
|
||||||
|
op.create_index("ix_dqc_severity_passed", "data_quality_checks", ["severity", "passed"])
|
||||||
|
op.create_check_constraint("ck_dqc_severity", "data_quality_checks",
|
||||||
|
"severity IN ('info', 'warning', 'critical')")
|
||||||
|
|
||||||
|
# ── data_lineage: ETL 血缘追踪 ──
|
||||||
|
op.create_table(
|
||||||
|
"data_lineage",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("source_system", sa.String(30), nullable=False),
|
||||||
|
sa.Column("source_record_id", sa.String(100), nullable=False),
|
||||||
|
sa.Column("target_table", sa.String(50), nullable=False),
|
||||||
|
sa.Column("target_id", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("transform_name", sa.String(100), nullable=False),
|
||||||
|
sa.Column("transform_detail", sa.Text(), nullable=True),
|
||||||
|
sa.Column("batch_id", sa.String(64), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
)
|
||||||
|
op.create_index("ix_lineage_source", "data_lineage", ["source_system", "source_record_id"])
|
||||||
|
op.create_index("ix_lineage_target", "data_lineage", ["target_table", "target_id"])
|
||||||
|
op.create_index("ix_lineage_batch", "data_lineage", ["batch_id"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# 逆序删除
|
||||||
|
op.drop_index("ix_lineage_batch", table_name="data_lineage")
|
||||||
|
op.drop_index("ix_lineage_target", table_name="data_lineage")
|
||||||
|
op.drop_index("ix_lineage_source", table_name="data_lineage")
|
||||||
|
op.drop_table("data_lineage")
|
||||||
|
|
||||||
|
op.drop_index("ix_dqc_severity_passed", table_name="data_quality_checks")
|
||||||
|
op.drop_index("ix_dqc_entity", table_name="data_quality_checks")
|
||||||
|
op.drop_index("ix_dqc_check_time", table_name="data_quality_checks")
|
||||||
|
op.drop_table("data_quality_checks")
|
||||||
|
|
||||||
|
op.drop_index("ix_ingest_failures_source", table_name="ingest_failures")
|
||||||
|
op.drop_index("ix_ingest_failures_status_next_retry", table_name="ingest_failures")
|
||||||
|
op.drop_table("ingest_failures")
|
||||||
|
|
||||||
|
op.drop_constraint("uq_raw_events_source_record", table_name="raw_events", type_="unique")
|
||||||
|
op.drop_index("ix_raw_events_source_ingested", table_name="raw_events")
|
||||||
|
op.drop_index("ix_raw_events_batch", table_name="raw_events")
|
||||||
|
op.drop_table("raw_events")
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""MatchStats 新增 xG 追踪字段
|
||||||
|
|
||||||
|
Revision ID: 0009_match_stats_xg_fields
|
||||||
|
Revises: 0008_raw_event_and_ingest_failure
|
||||||
|
Create Date: 2026-09-17
|
||||||
|
|
||||||
|
架构审查报告 P1-4 实施:
|
||||||
|
understat 允许纠正旧 xG 值。新增字段追踪 xG 具体来源和更新时间,
|
||||||
|
实现全量覆盖模式:当 understat 数据更新时覆盖旧值而非跳过。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
|
revision: str = '0009_match_stats_xg_fields'
|
||||||
|
down_revision: Union[str, None] = '0008_raw_event_and_ingest_failure'
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("match_stats", sa.Column("xg_source", sa.String(30), nullable=True))
|
||||||
|
op.add_column("match_stats", sa.Column("xg_updated_at", sa.DateTime(timezone=True), nullable=True))
|
||||||
|
op.add_column("match_stats", sa.Column("xg_source_record_id", sa.String(100), nullable=True))
|
||||||
|
op.create_index("ix_match_stats_xg_source", "match_stats", ["xg_source", "xg_updated_at"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_match_stats_xg_source", table_name="match_stats")
|
||||||
|
op.drop_column("match_stats", "xg_source_record_id")
|
||||||
|
op.drop_column("match_stats", "xg_updated_at")
|
||||||
|
op.drop_column("match_stats", "xg_source")
|
||||||
+6
-6
@@ -2,15 +2,15 @@ services:
|
|||||||
postgres:
|
postgres:
|
||||||
image: postgres:16-alpine
|
image: postgres:16-alpine
|
||||||
environment:
|
environment:
|
||||||
POSTGRES_USER: football
|
POSTGRES_USER: ${POSTGRES_USER:?POSTGRES_USER 未设置}
|
||||||
POSTGRES_PASSWORD: football
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD 未设置}
|
||||||
POSTGRES_DB: football
|
POSTGRES_DB: ${POSTGRES_DB:-football}
|
||||||
ports:
|
ports:
|
||||||
- "5432:5432"
|
- "${POSTGRES_PORT:-5432}:5432"
|
||||||
volumes:
|
volumes:
|
||||||
- pgdata:/var/lib/postgresql/data
|
- pgdata:/var/lib/postgresql/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "pg_isready -U football"]
|
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:?POSTGRES_USER 未设置}"]
|
||||||
interval: 5s
|
interval: 5s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
@@ -19,7 +19,7 @@ services:
|
|||||||
build: .
|
build: .
|
||||||
command: uvicorn src.api.app:app --host 0.0.0.0 --port 8000 --reload
|
command: uvicorn src.api.app:app --host 0.0.0.0 --port 8000 --reload
|
||||||
ports:
|
ports:
|
||||||
- "8000:8000"
|
- "${API_PORT:-8000}:8000"
|
||||||
env_file: .env
|
env_file: .env
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
|
|||||||
+18
-8
@@ -57,38 +57,48 @@
|
|||||||
Profeto/
|
Profeto/
|
||||||
├── src/
|
├── src/
|
||||||
│ ├── api/
|
│ ├── api/
|
||||||
│ │ ├── app.py # FastAPI 工厂(lifespan 建表)
|
│ │ ├── app.py # FastAPI 工厂(lifespan 仅验证 DB 连接,不建表)
|
||||||
|
│ │ ├── deps.py # 依赖:管理接口鉴权(X-API-Key)
|
||||||
│ │ ├── schemas.py # Pydantic v2 请求/响应
|
│ │ ├── schemas.py # Pydantic v2 请求/响应
|
||||||
│ │ └── routes/
|
│ │ └── routes/
|
||||||
│ │ ├── matches.py # 联赛/比赛查询(游标分页)
|
│ │ ├── matches.py # 联赛/比赛查询(游标分页)
|
||||||
│ │ ├── predict.py # 预测 + 预测历史
|
│ │ ├── predict.py # 预测 + 预测历史
|
||||||
│ │ ├── ingest.py # 采集触发(自管 session)
|
│ │ ├── ingest.py # 采集触发(自管 session,需鉴权)
|
||||||
│ │ └── eval.py # 赛后回填 + 准确率汇总
|
│ │ ├── eval.py # 赛后回填 + 准确率汇总
|
||||||
|
│ │ └── backtest.py # 历史回测(需鉴权)
|
||||||
│ ├── db/
|
│ ├── db/
|
||||||
│ │ ├── base.py # async engine + get_db/get_db_read
|
│ │ ├── base.py # async engine + get_db/get_db_read
|
||||||
│ │ └── models.py # 6 张表 ORM
|
│ │ ├── models.py # 6 张表 ORM
|
||||||
|
│ │ ├── repositories.py # 仓储层
|
||||||
|
│ │ └── unit_of_work.py # 事务边界
|
||||||
│ ├── data/
|
│ ├── data/
|
||||||
│ │ ├── bzzoiro.py # 赛果采集 + 幂等入库
|
│ │ ├── bzzoiro.py # 赛果采集 + 幂等入库
|
||||||
│ │ ├── understat.py # xG 回填
|
│ │ ├── understat.py # xG 回填
|
||||||
│ │ ├── injuries.py # 伤停采集(带文件缓存)
|
│ │ ├── injuries.py # 伤停采集(带文件缓存)
|
||||||
│ │ ├── normalize.py # NormalizedMatch 清洗契约
|
│ │ ├── normalize.py # NormalizedMatch 清洗契约
|
||||||
│ │ ├── team_names.py # 队名归一映射
|
│ │ ├── team_names.py # 队名归一映射
|
||||||
|
│ │ ├── sources.py # 数据源注册表
|
||||||
│ │ └── config.py # 联赛代码映射
|
│ │ └── config.py # 联赛代码映射
|
||||||
│ ├── llm/
|
│ ├── llm/
|
||||||
│ │ ├── provider.py # OpenAI-compatible 抽象(共享连接池/JSON 兜底解析)
|
│ │ ├── provider.py # OpenAI-compatible 抽象(共享连接池/JSON 兜底解析)
|
||||||
│ │ ├── context_builder.py # 数据切片(h2h/form/stats/home_away/injuries)+ 单 agent 拼接
|
│ │ ├── context_builder.py # 数据切片(h2h/form/stats/home_away/injuries)+ 单 agent 拼接
|
||||||
│ │ ├── predict.py # 预测入口(mode 分派 + 缓存)
|
│ │ ├── predict.py # 预测入口(mode 分派 + 缓存)
|
||||||
│ │ ├── eval.py # 准确率统计
|
│ │ ├── eval.py # 准确率统计
|
||||||
|
│ │ ├── validation.py # LLM 输出严格校验(Pydantic)
|
||||||
|
│ │ ├── backtest.py # 回测执行
|
||||||
│ │ ├── agents/
|
│ │ ├── agents/
|
||||||
│ │ │ ├── base.py # AgentSpec / AgentReport / run_agent
|
│ │ │ ├── base.py # AgentSpec / AgentReport / run_agent
|
||||||
│ │ │ └── orchestrator.py # 并行专家 → 终裁 → 存库
|
│ │ │ └── orchestrator.py # 并行专家 → 终裁 → 存库
|
||||||
│ │ └── prompts/
|
│ │ └── prompts/
|
||||||
│ │ ├── match_prediction_v1/v2.md # 单 agent 模板
|
│ │ ├── match_prediction_v1/v2.md # 单 agent 模板
|
||||||
│ │ └── agents/{h2h,form,stats,home_away,injuries,aggregator}_v1.md
|
│ │ └── agents/{h2h,form,stats,home_away,injuries,aggregator}_v1.md
|
||||||
│ └── core/config.py # pydantic-settings
|
│ └── core/
|
||||||
├── alembic/versions/ # 0001 建表 + 0002 agent 字段
|
│ ├── config.py # pydantic-settings
|
||||||
├── frontend/src/pages/Matches.tsx # 单页(预测面板 + 专家报告折叠区)
|
│ ├── http_client.py # 共享 httpx 客户端
|
||||||
├── tests/ # 33 项(核心 13 + agent 20)
|
│ └── retry.py # 重试工具
|
||||||
|
├── alembic/versions/ # 0001~0006(0001 建表 → 0006 漂移清理)
|
||||||
|
├── frontend/src/pages/Matches.tsx # 单页(预测面板 + 专家报告折叠区 + 游标分页)
|
||||||
|
├── tests/ # 核心 + agent 测试
|
||||||
├── docker-compose.yml # api + postgres 两容器
|
├── docker-compose.yml # api + postgres 两容器
|
||||||
└── docs/ # 本文档
|
└── docs/ # 本文档
|
||||||
```
|
```
|
||||||
|
|||||||
+18
-5
@@ -35,27 +35,34 @@ Profeto/
|
|||||||
├── src/ # 后端源码
|
├── src/ # 后端源码
|
||||||
│ ├── api/
|
│ ├── api/
|
||||||
│ │ ├── routes/ # FastAPI 路由
|
│ │ ├── routes/ # FastAPI 路由
|
||||||
│ │ │ ├── leagues.py # 联赛/比赛查询
|
│ │ │ ├── matches.py # 联赛/比赛查询
|
||||||
│ │ │ ├── predict.py # 预测入口
|
│ │ │ ├── predict.py # 预测入口
|
||||||
│ │ │ ├── ingest.py # 数据采集
|
│ │ │ ├── ingest.py # 数据采集
|
||||||
│ │ │ └── eval.py # 评估回填
|
│ │ │ ├── eval.py # 评估回填
|
||||||
|
│ │ │ └── backtest.py # 历史回测
|
||||||
|
│ │ ├── deps.py # 依赖:管理接口鉴权(X-API-Key)
|
||||||
│ │ ├── schemas.py # Pydantic 模型
|
│ │ ├── schemas.py # Pydantic 模型
|
||||||
│ │ └── app.py # FastAPI 工厂
|
│ │ └── app.py # FastAPI 工厂
|
||||||
│ ├── db/
|
│ ├── db/
|
||||||
│ │ ├── base.py # SQLAlchemy async engine + session
|
│ │ ├── base.py # SQLAlchemy async engine + session
|
||||||
│ │ └── models.py # 6 张表 ORM
|
│ │ ├── models.py # 6 张表 ORM
|
||||||
|
│ │ ├── repositories.py # 仓储层(查询封装)
|
||||||
|
│ │ └── unit_of_work.py # 事务边界
|
||||||
│ ├── data/
|
│ ├── data/
|
||||||
│ │ ├── bzzoiro.py # bzzoiro 采集 + 入库
|
│ │ ├── bzzoiro.py # bzzoiro 采集 + 入库
|
||||||
│ │ ├── understat.py # understat xG 回填
|
│ │ ├── understat.py # understat xG 回填
|
||||||
│ │ ├── injuries.py # 伤停采集
|
│ │ ├── injuries.py # 伤停采集
|
||||||
│ │ ├── normalize.py # 数据清洗契约
|
│ │ ├── normalize.py # 数据清洗契约
|
||||||
│ │ ├── team_names.py # 队名归一化映射
|
│ │ ├── team_names.py # 队名归一化映射
|
||||||
|
│ │ ├── sources.py # 数据源注册表
|
||||||
│ │ └── config.py # 联赛映射常量
|
│ │ └── config.py # 联赛映射常量
|
||||||
│ ├── llm/
|
│ ├── llm/
|
||||||
│ │ ├── provider.py # LLM 提供商抽象(OpenAI-compatible)
|
│ │ ├── provider.py # LLM 提供商抽象(OpenAI-compatible)
|
||||||
│ │ ├── context_builder.py # 数据切片 + 拼接
|
│ │ ├── context_builder.py # 数据切片 + 拼接
|
||||||
│ │ ├── predict.py # 预测入口(单/多模式分派)
|
│ │ ├── predict.py # 预测入口(单/多模式分派)
|
||||||
│ │ ├── eval.py # 评估统计
|
│ │ ├── eval.py # 评估统计
|
||||||
|
│ │ ├── validation.py # LLM 输出严格校验
|
||||||
|
│ │ ├── backtest.py # 回测执行
|
||||||
│ │ ├── agents/
|
│ │ ├── agents/
|
||||||
│ │ │ ├── base.py # AgentSpec + run_agent
|
│ │ │ ├── base.py # AgentSpec + run_agent
|
||||||
│ │ │ └── orchestrator.py # 多 agent 编排
|
│ │ │ └── orchestrator.py # 多 agent 编排
|
||||||
@@ -69,12 +76,18 @@ Profeto/
|
|||||||
│ │ ├── h2h_v1.md
|
│ │ ├── h2h_v1.md
|
||||||
│ │ └── aggregator_v1.md
|
│ │ └── aggregator_v1.md
|
||||||
│ └── core/
|
│ └── core/
|
||||||
│ └── config.py # pydantic-settings 配置
|
│ ├── config.py # pydantic-settings 配置
|
||||||
|
│ ├── http_client.py # 共享 httpx 客户端
|
||||||
|
│ └── retry.py # 重试工具
|
||||||
├── frontend/ # React 单页前端
|
├── frontend/ # React 单页前端
|
||||||
├── alembic/ # 数据库迁移
|
├── alembic/ # 数据库迁移
|
||||||
│ └── versions/
|
│ └── versions/
|
||||||
│ ├── 0001_initial.py
|
│ ├── 0001_initial.py
|
||||||
│ └── 0002_agent_outputs.py
|
│ ├── 0002_agent_outputs.py
|
||||||
|
│ ├── 0003_injuries.py
|
||||||
|
│ ├── 0004_snapshot_and_constraints.py
|
||||||
|
│ ├── 0005_prediction_status_and_stats_provenance.py
|
||||||
|
│ └── 0006_schema_model_drift_cleanup.py
|
||||||
├── tests/ # 测试
|
├── tests/ # 测试
|
||||||
│ ├── test_core.py # 核心逻辑测试
|
│ ├── test_core.py # 核心逻辑测试
|
||||||
│ └── test_agents.py # 多 agent 测试
|
│ └── test_agents.py # 多 agent 测试
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
export default {
|
export default {
|
||||||
content: ['./index.html', './src/**/*.{ts,tsx}'],
|
plugins: {
|
||||||
theme: { extend: {} },
|
tailwindcss: {},
|
||||||
plugins: [],
|
autoprefixer: {},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+42
-5
@@ -1,17 +1,54 @@
|
|||||||
import { ErrorBoundary } from './components/ErrorBoundary'
|
import { ErrorBoundary } from './components/ErrorBoundary'
|
||||||
import Matches from './pages/Matches'
|
import Matches from './pages/Matches'
|
||||||
|
|
||||||
|
/** 报眉日期行:2026年9月15日 星期二 */
|
||||||
|
function dateLine(): string {
|
||||||
|
return new Date().toLocaleDateString('zh-CN', {
|
||||||
|
year: 'numeric',
|
||||||
|
month: 'long',
|
||||||
|
day: 'numeric',
|
||||||
|
weekday: 'long',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export default function App() {
|
export default function App() {
|
||||||
return (
|
return (
|
||||||
<ErrorBoundary>
|
<ErrorBoundary>
|
||||||
<div className="min-h-screen bg-gray-50">
|
<div className="min-h-screen bg-paper-50">
|
||||||
<header className="bg-white border-b px-6 py-3 flex items-center justify-between">
|
{/* ── 报头:粗线 + 居中刊名 + 报眉 ── */}
|
||||||
<h1 className="text-xl font-bold text-blue-700">⚽ 先知 Profeto</h1>
|
<header className="masthead-rule">
|
||||||
<span className="text-sm text-gray-500">足球 LLM 预测服务</span>
|
<div className="mx-auto max-w-5xl px-5 sm:px-8">
|
||||||
|
<div className="border-b border-ink-900 py-5 text-center sm:py-6">
|
||||||
|
<h1 className="font-serif text-4xl font-bold tracking-widest text-ink-900">
|
||||||
|
先知
|
||||||
|
<span className="ml-3 align-baseline font-serif text-base font-normal italic tracking-normal text-ink-500">
|
||||||
|
Profeto
|
||||||
|
</span>
|
||||||
|
</h1>
|
||||||
|
<p className="mt-2 text-2xs tracking-[0.4em] text-ink-500">
|
||||||
|
足球比分预测 · 五路专家 · 终裁汇总
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center justify-between border-b border-ink-200 py-2 text-2xs text-ink-500">
|
||||||
|
<span>{dateLine()}</span>
|
||||||
|
<span className="flex items-center gap-1.5">
|
||||||
|
<span className="inline-block h-1.5 w-1.5 bg-emerald-600" aria-hidden="true" />
|
||||||
|
服务运行中
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</header>
|
</header>
|
||||||
<main className="max-w-5xl mx-auto p-6">
|
|
||||||
|
<main className="mx-auto max-w-5xl px-5 py-6 sm:px-8 sm:py-8">
|
||||||
<Matches />
|
<Matches />
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
|
{/* ── 版底 ── */}
|
||||||
|
<footer className="mx-auto max-w-5xl px-5 pb-10 sm:px-8">
|
||||||
|
<div className="border-t border-ink-200 pt-3 text-center text-2xs leading-relaxed text-ink-400">
|
||||||
|
预测结果由大语言模型生成 · 仅供研究参考 · 不构成任何投注建议
|
||||||
|
</div>
|
||||||
|
</footer>
|
||||||
</div>
|
</div>
|
||||||
</ErrorBoundary>
|
</ErrorBoundary>
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,3 +1,102 @@
|
|||||||
@tailwind base;
|
@tailwind base;
|
||||||
@tailwind components;
|
@tailwind components;
|
||||||
@tailwind utilities;
|
@tailwind utilities;
|
||||||
|
|
||||||
|
@layer base {
|
||||||
|
html {
|
||||||
|
-webkit-text-size-adjust: 100%;
|
||||||
|
text-rendering: optimizeLegibility;
|
||||||
|
}
|
||||||
|
|
||||||
|
body {
|
||||||
|
@apply bg-paper-50 text-ink-800 font-sans antialiased;
|
||||||
|
font-feature-settings: 'tnum' 1; /* 数字等宽:比分/百分比不跳动 */
|
||||||
|
}
|
||||||
|
|
||||||
|
::selection {
|
||||||
|
@apply bg-press-wash text-press;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* 统一焦点环:印报红细线,键盘可达 */
|
||||||
|
:focus-visible {
|
||||||
|
outline: 2px solid #9e1b1b;
|
||||||
|
outline-offset: 2px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* 细滚动条 */
|
||||||
|
::-webkit-scrollbar {
|
||||||
|
width: 10px;
|
||||||
|
height: 10px;
|
||||||
|
}
|
||||||
|
::-webkit-scrollbar-track {
|
||||||
|
background: transparent;
|
||||||
|
}
|
||||||
|
::-webkit-scrollbar-thumb {
|
||||||
|
@apply rounded-full bg-ink-300;
|
||||||
|
}
|
||||||
|
::-webkit-scrollbar-thumb:hover {
|
||||||
|
@apply bg-ink-400;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* 尊重「减少动态效果」偏好 */
|
||||||
|
@media (prefers-reduced-motion: reduce) {
|
||||||
|
*,
|
||||||
|
*::before,
|
||||||
|
*::after {
|
||||||
|
animation-duration: 0.01ms !important;
|
||||||
|
animation-iteration-count: 1 !important;
|
||||||
|
transition-duration: 0.01ms !important;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@layer components {
|
||||||
|
/* ── 报头双线:粗线在上、细线在下 ── */
|
||||||
|
.masthead-rule {
|
||||||
|
border-top: 3px solid #17140f;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── 按钮:方正边框式,悬停反白 ── */
|
||||||
|
.btn {
|
||||||
|
@apply inline-flex items-center justify-center gap-1.5 border border-ink-300 bg-transparent px-3 py-1.5
|
||||||
|
text-sm text-ink-700 transition-colors duration-150
|
||||||
|
hover:border-ink-900 hover:bg-ink-900 hover:text-paper-50
|
||||||
|
disabled:cursor-not-allowed disabled:opacity-40
|
||||||
|
disabled:hover:border-ink-300 disabled:hover:bg-transparent disabled:hover:text-ink-700;
|
||||||
|
}
|
||||||
|
.btn-sm {
|
||||||
|
@apply px-2.5 py-1 text-xs;
|
||||||
|
}
|
||||||
|
.btn-solid {
|
||||||
|
@apply border-ink-900 bg-ink-900 text-paper-50 hover:border-press hover:bg-press;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── 表单控件:方正、无圆角 ── */
|
||||||
|
.field {
|
||||||
|
@apply border border-ink-300 bg-transparent px-2.5 py-1.5 text-sm text-ink-800
|
||||||
|
transition-colors hover:border-ink-400
|
||||||
|
focus:border-press focus:outline-none;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── 版面切换文字标签(联赛/状态/模式) ── */
|
||||||
|
.tab {
|
||||||
|
@apply relative whitespace-nowrap px-0.5 py-1 text-sm text-ink-500 transition-colors hover:text-ink-900;
|
||||||
|
}
|
||||||
|
.tab-on {
|
||||||
|
@apply font-medium text-press;
|
||||||
|
}
|
||||||
|
.tab-on::after {
|
||||||
|
content: '';
|
||||||
|
@apply absolute inset-x-0 bottom-0 h-0.5 bg-press;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── 小节标题:宋体加粗 + 墨色底线 ── */
|
||||||
|
.section-head {
|
||||||
|
@apply border-b border-ink-900 pb-1.5 font-serif text-sm font-bold text-ink-900;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── 骨架占位:低调脉动,不用渐变扫光 ── */
|
||||||
|
.skeleton {
|
||||||
|
@apply animate-pulse rounded-none bg-ink-200;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+544
-176
@@ -1,4 +1,4 @@
|
|||||||
import { useCallback, useEffect, useState } from 'react'
|
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||||
|
|
||||||
interface Match {
|
interface Match {
|
||||||
id: number
|
id: number
|
||||||
@@ -65,54 +65,207 @@ const LEAGUES = [
|
|||||||
{ code: 'F1', name: '法甲' },
|
{ code: 'F1', name: '法甲' },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
/** 汉字编号,给专家意见排版用 */
|
||||||
|
const CN_NUM = ['一', '二', '三', '四', '五', '六', '七', '八']
|
||||||
|
|
||||||
|
const STATUS_META: Record<string, { label: string; cls: string }> = {
|
||||||
|
finished: { label: '已完赛', cls: 'text-ink-400' },
|
||||||
|
scheduled: { label: '未开赛', cls: 'text-ink-600' },
|
||||||
|
live: { label: '进行中', cls: 'text-press font-medium' },
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 1x2 → 中文标签 */
|
||||||
|
const OUTCOME_LABEL: Record<string, string> = { '1': '主胜', X: '平局', '2': '客胜' }
|
||||||
|
|
||||||
|
/** 置信度细线:0~1 数值的低调可视化 */
|
||||||
|
function Meter({ value }: { value: number }) {
|
||||||
|
const pct = Math.max(0, Math.min(100, Math.round(value * 100)))
|
||||||
|
return (
|
||||||
|
<div className="h-px w-full bg-ink-200" role="presentation">
|
||||||
|
<div className="h-px bg-press transition-[width] duration-500" style={{ width: `${pct}%` }} />
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** home_edge(-1~1,正=利主队)的可视化:以中线为原点的双向细条 */
|
||||||
|
function EdgeBar({ value }: { value: number }) {
|
||||||
|
const v = Math.max(-1, Math.min(1, value))
|
||||||
|
const half = Math.abs(v) * 50
|
||||||
|
return (
|
||||||
|
<div className="relative h-px w-full bg-ink-200" role="presentation">
|
||||||
|
<span className="absolute left-1/2 top-1/2 h-2 w-px -translate-x-1/2 -translate-y-1/2 bg-ink-400" />
|
||||||
|
<span
|
||||||
|
className={`absolute top-0 h-px transition-all duration-500 ${v >= 0 ? 'bg-press' : 'bg-ink-600'}`}
|
||||||
|
style={
|
||||||
|
v >= 0
|
||||||
|
? { left: '50%', width: `${half}%` }
|
||||||
|
: { right: '50%', width: `${half}%` }
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 骨架占位行:低调脉动灰块 */
|
||||||
|
function SkeletonRows({ n = 4 }: { n?: number }) {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
{Array.from({ length: n }).map((_, i) => (
|
||||||
|
<div key={i} className="flex items-center gap-4 border-b border-ink-200 px-1 py-3.5">
|
||||||
|
<div className="skeleton h-3 w-16" />
|
||||||
|
<div className="skeleton h-3 flex-1" />
|
||||||
|
<div className="skeleton h-3 w-10" />
|
||||||
|
<div className="skeleton h-3 flex-1" />
|
||||||
|
<div className="skeleton h-3 w-16" />
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function Spinner({ className = '' }: { className?: string }) {
|
||||||
|
return (
|
||||||
|
<svg
|
||||||
|
viewBox="0 0 20 20"
|
||||||
|
className={`h-3.5 w-3.5 animate-spin ${className}`}
|
||||||
|
fill="none"
|
||||||
|
aria-hidden="true"
|
||||||
|
>
|
||||||
|
<circle cx="10" cy="10" r="7.5" stroke="currentColor" strokeWidth="1.5" strokeOpacity="0.25" />
|
||||||
|
<path d="M17.5 10A7.5 7.5 0 0010 2.5" stroke="currentColor" strokeWidth="1.5" strokeLinecap="round" />
|
||||||
|
</svg>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 胜平负一行文字:选中的红字加方块标记,未选中的退灰 */
|
||||||
|
function OutcomeLine({
|
||||||
|
pick,
|
||||||
|
confidence,
|
||||||
|
}: {
|
||||||
|
pick: string | null
|
||||||
|
confidence: number | null
|
||||||
|
}) {
|
||||||
|
const options = ['1', 'X', '2'] as const
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<div className="flex items-baseline justify-center gap-6 sm:gap-10">
|
||||||
|
{options.map(o => {
|
||||||
|
const on = pick === o
|
||||||
|
return (
|
||||||
|
<div key={o} className="flex flex-col items-center gap-1">
|
||||||
|
<span className={`flex items-center gap-1.5 text-sm ${on ? 'font-semibold text-press' : 'text-ink-400'}`}>
|
||||||
|
{on && <span className="inline-block h-2 w-2 bg-press" aria-hidden="true" />}
|
||||||
|
{OUTCOME_LABEL[o]}
|
||||||
|
</span>
|
||||||
|
{on && confidence !== null && (
|
||||||
|
<span className="text-2xs tabular-nums text-ink-500">
|
||||||
|
置信 {Math.round(confidence * 100)}%
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
{pick && confidence !== null && (
|
||||||
|
<div className="mx-auto mt-3 max-w-xs">
|
||||||
|
<Meter value={confidence} />
|
||||||
|
<p className="mt-1 text-center text-2xs text-ink-400">主观置信度,非统计概率</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
export default function Matches() {
|
export default function Matches() {
|
||||||
const [league, setLeague] = useState('E0')
|
const [league, setLeague] = useState('E0')
|
||||||
const [status, setStatus] = useState('scheduled')
|
const [status, setStatus] = useState('scheduled')
|
||||||
const [matches, setMatches] = useState<Match[]>([])
|
const [matches, setMatches] = useState<Match[]>([])
|
||||||
|
const [nextCursor, setNextCursor] = useState<string | null>(null)
|
||||||
|
const [loadingMore, setLoadingMore] = useState(false)
|
||||||
const [loading, setLoading] = useState(false)
|
const [loading, setLoading] = useState(false)
|
||||||
const [predictingId, setPredictingId] = useState<number | null>(null)
|
const [predictingId, setPredictingId] = useState<number | null>(null)
|
||||||
const [prediction, setPrediction] = useState<Prediction | null>(null)
|
const [prediction, setPrediction] = useState<Prediction | null>(null)
|
||||||
|
const [predictionFor, setPredictionFor] = useState<Match | null>(null)
|
||||||
const [error, setError] = useState<string | null>(null)
|
const [error, setError] = useState<string | null>(null)
|
||||||
const [mode, setMode] = useState<'single' | 'multi'>('multi')
|
const [mode, setMode] = useState<'single' | 'multi'>('multi')
|
||||||
|
|
||||||
|
// 请求竞态防护:切换联赛/状态很快时,先发的慢请求可能后返回,
|
||||||
|
// 把旧结果覆盖到新筛选上。用递增序号只认最后一次请求的响应。
|
||||||
|
const loadSeq = useRef(0)
|
||||||
|
const predictSeq = useRef(0)
|
||||||
|
|
||||||
const load = useCallback(async () => {
|
const load = useCallback(async () => {
|
||||||
|
const seq = ++loadSeq.current
|
||||||
setLoading(true)
|
setLoading(true)
|
||||||
|
// 切换筛选时作废进行中的「加载更多」,避免其标志位卡住
|
||||||
|
setLoadingMore(false)
|
||||||
setError(null)
|
setError(null)
|
||||||
try {
|
try {
|
||||||
const params = new URLSearchParams({ league, status, limit: '50' })
|
const params = new URLSearchParams({ league, status, limit: '50' })
|
||||||
const res = await fetch(`/api/v1/matches?${params}`)
|
const res = await fetch(`/api/v1/matches?${params}`)
|
||||||
|
if (seq !== loadSeq.current) return // 已有更新的请求,丢弃本次结果
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||||
const data = await res.json()
|
const data = await res.json()
|
||||||
|
if (seq !== loadSeq.current) return
|
||||||
setMatches(data.items)
|
setMatches(data.items)
|
||||||
|
setNextCursor(data.next_cursor ?? null)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (seq !== loadSeq.current) return
|
||||||
setError(e instanceof Error ? e.message : String(e))
|
setError(e instanceof Error ? e.message : String(e))
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false)
|
if (seq === loadSeq.current) setLoading(false)
|
||||||
}
|
}
|
||||||
}, [league, status])
|
}, [league, status])
|
||||||
|
|
||||||
|
// 加载下一页(游标分页)
|
||||||
|
const loadMore = async () => {
|
||||||
|
if (!nextCursor || loadingMore) return
|
||||||
|
const seq = loadSeq.current // 不做自增:切换筛选会自增,这里只跟随当前序列
|
||||||
|
setLoadingMore(true)
|
||||||
|
try {
|
||||||
|
const params = new URLSearchParams({ league, status, limit: '50', cursor: nextCursor })
|
||||||
|
const res = await fetch(`/api/v1/matches?${params}`)
|
||||||
|
if (seq !== loadSeq.current) return
|
||||||
|
if (!res.ok) throw new Error(`HTTP ${res.status}`)
|
||||||
|
const data = await res.json()
|
||||||
|
if (seq !== loadSeq.current) return
|
||||||
|
setMatches(prev => [...prev, ...data.items])
|
||||||
|
setNextCursor(data.next_cursor ?? null)
|
||||||
|
} catch (e) {
|
||||||
|
if (seq !== loadSeq.current) return
|
||||||
|
setError(e instanceof Error ? e.message : String(e))
|
||||||
|
} finally {
|
||||||
|
if (seq === loadSeq.current) setLoadingMore(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
useEffect(() => { load() }, [load])
|
useEffect(() => { load() }, [load])
|
||||||
|
|
||||||
const predict = async (matchId: number) => {
|
const predict = async (m: Match) => {
|
||||||
setPredictingId(matchId)
|
const seq = ++predictSeq.current
|
||||||
|
setPredictingId(m.id)
|
||||||
setError(null)
|
setError(null)
|
||||||
setPrediction(null)
|
setPrediction(null)
|
||||||
|
setPredictionFor(m)
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/v1/predict', {
|
const res = await fetch('/api/v1/predict', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ match_id: matchId, mode }),
|
body: JSON.stringify({ match_id: m.id, mode }),
|
||||||
})
|
})
|
||||||
|
if (seq !== predictSeq.current) return
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const t = await res.text()
|
const t = await res.text()
|
||||||
throw new Error(`HTTP ${res.status}: ${t}`)
|
throw new Error(`HTTP ${res.status}: ${t}`)
|
||||||
}
|
}
|
||||||
const data = await res.json()
|
const data = await res.json()
|
||||||
|
if (seq !== predictSeq.current) return
|
||||||
setPrediction(data)
|
setPrediction(data)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (seq !== predictSeq.current) return
|
||||||
setError(e instanceof Error ? e.message : String(e))
|
setError(e instanceof Error ? e.message : String(e))
|
||||||
} finally {
|
} finally {
|
||||||
setPredictingId(null)
|
if (seq === predictSeq.current) setPredictingId(null)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,195 +274,410 @@ export default function Matches() {
|
|||||||
return d.toLocaleString('zh-CN', { month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit' })
|
return d.toLocaleString('zh-CN', { month: '2-digit', day: '2-digit', hour: '2-digit', minute: '2-digit' })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const leagueName = LEAGUES.find(l => l.code === league)?.name ?? league
|
||||||
|
|
||||||
|
/** 状态/模式一组的文字切换 */
|
||||||
|
const Switch = ({ value, onChange, items }: {
|
||||||
|
value: string
|
||||||
|
onChange: (v: string) => void
|
||||||
|
items: { v: string; label: string; title?: string }[]
|
||||||
|
}) => (
|
||||||
|
<span className="inline-flex items-center gap-2.5">
|
||||||
|
{items.map((it, i) => (
|
||||||
|
<span key={it.v} className="inline-flex items-center gap-2.5">
|
||||||
|
{i > 0 && <span className="text-ink-300" aria-hidden="true">/</span>}
|
||||||
|
<button
|
||||||
|
onClick={() => onChange(it.v)}
|
||||||
|
title={it.title}
|
||||||
|
className={`relative tab ${value === it.v ? 'tab-on' : ''} text-xs`}
|
||||||
|
>
|
||||||
|
{it.label}
|
||||||
|
</button>
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</span>
|
||||||
|
)
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-4">
|
<div className="space-y-5">
|
||||||
{/* 筛选 */}
|
{/* ── 联赛版面切换 ── */}
|
||||||
<div className="flex gap-3 items-center flex-wrap">
|
<nav className="flex items-center gap-6 overflow-x-auto border-b border-ink-900" aria-label="联赛">
|
||||||
<select value={league} onChange={e => setLeague(e.target.value)}
|
{LEAGUES.map(l => (
|
||||||
className="border rounded px-3 py-1.5 text-sm">
|
|
||||||
{LEAGUES.map(l => <option key={l.code} value={l.code}>{l.name}</option>)}
|
|
||||||
</select>
|
|
||||||
<select value={status} onChange={e => setStatus(e.target.value)}
|
|
||||||
className="border rounded px-3 py-1.5 text-sm">
|
|
||||||
<option value="scheduled">未开赛</option>
|
|
||||||
<option value="finished">已完赛</option>
|
|
||||||
<option value="">全部</option>
|
|
||||||
</select>
|
|
||||||
<div className="flex items-center gap-2 text-sm">
|
|
||||||
<span className="text-gray-500">模式:</span>
|
|
||||||
<button
|
<button
|
||||||
onClick={() => setMode('single')}
|
key={l.code}
|
||||||
className={`px-2 py-1 rounded text-xs ${mode === 'single' ? 'bg-blue-600 text-white' : 'bg-gray-100 text-gray-600'}`}
|
onClick={() => setLeague(l.code)}
|
||||||
>单次</button>
|
className={`relative tab ${league === l.code ? 'tab-on' : ''} font-serif`}
|
||||||
<button
|
>
|
||||||
onClick={() => setMode('multi')}
|
{l.name}
|
||||||
className={`px-2 py-1 rounded text-xs ${mode === 'multi' ? 'bg-blue-600 text-white' : 'bg-gray-100 text-gray-600'}`}
|
|
||||||
>多 Agent</button>
|
|
||||||
</div>
|
|
||||||
<button onClick={load} disabled={loading}
|
|
||||||
className="bg-blue-600 text-white text-sm px-4 py-1.5 rounded hover:bg-blue-700 disabled:opacity-50">
|
|
||||||
{loading ? '加载中...' : '刷新'}
|
|
||||||
</button>
|
</button>
|
||||||
<span className="text-sm text-gray-500">共 {matches.length} 场</span>
|
))}
|
||||||
|
</nav>
|
||||||
|
|
||||||
|
{/* ── 第二行:状态 / 模式 / 计数 / 刷新 ── */}
|
||||||
|
<div className="flex flex-wrap items-center gap-x-5 gap-y-2 text-xs text-ink-500">
|
||||||
|
<span className="inline-flex items-center gap-2.5">
|
||||||
|
<span className="text-2xs text-ink-400">状态</span>
|
||||||
|
<Switch
|
||||||
|
value={status}
|
||||||
|
onChange={setStatus}
|
||||||
|
items={[
|
||||||
|
{ v: 'scheduled', label: '未开赛' },
|
||||||
|
{ v: 'finished', label: '已完赛' },
|
||||||
|
{ v: '', label: '全部' },
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span className="inline-flex items-center gap-2.5">
|
||||||
|
<span className="text-2xs text-ink-400">模式</span>
|
||||||
|
<Switch
|
||||||
|
value={mode}
|
||||||
|
onChange={v => setMode(v as 'single' | 'multi')}
|
||||||
|
items={[
|
||||||
|
{ v: 'single', label: '单次', title: '单次调用,快但只有一个模型看全部数据' },
|
||||||
|
{ v: 'multi', label: '多专家', title: '5 个专家并行分析后由终裁汇总,质量更高' },
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<span className="ml-auto inline-flex items-center gap-3">
|
||||||
|
<span className="tabular-nums">共 {matches.length} 场</span>
|
||||||
|
<button onClick={load} disabled={loading} className="btn btn-sm">
|
||||||
|
{loading ? (<><Spinner /> 获取中</>) : '刷新'}
|
||||||
|
</button>
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* ── 错误提示 ── */}
|
||||||
{error && (
|
{error && (
|
||||||
<div className="bg-red-50 border border-red-200 text-red-700 px-4 py-2 rounded text-sm flex items-center justify-between">
|
<div className="flex items-start justify-between gap-3 border border-press bg-press-wash px-4 py-3">
|
||||||
<span>❌ {error}</span>
|
<div>
|
||||||
<button onClick={() => setError(null)} className="text-red-500 hover:text-red-700 text-xs">✕</button>
|
<p className="text-sm font-medium text-press">请求失败</p>
|
||||||
|
<p className="mt-0.5 text-xs text-ink-600">{error}</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
<button onClick={() => setError(null)} className="text-ink-400 transition-colors hover:text-ink-900" aria-label="关闭">
|
||||||
|
<svg viewBox="0 0 20 20" className="h-4 w-4" fill="currentColor" aria-hidden="true">
|
||||||
{/* 比赛表 */}
|
<path d="M6.3 5.3a1 1 0 011.4 0L10 7.6l2.3-2.3a1 1 0 111.4 1.4L11.4 9l2.3 2.3a1 1 0 01-1.4 1.4L10 10.4l-2.3 2.3a1 1 0 01-1.4-1.4L8.6 9 6.3 6.7a1 1 0 010-1.4z" />
|
||||||
<div className="bg-white rounded border overflow-hidden">
|
</svg>
|
||||||
<table className="w-full text-sm">
|
|
||||||
<thead className="bg-gray-100 text-gray-600">
|
|
||||||
<tr>
|
|
||||||
<th className="text-left px-4 py-2">日期</th>
|
|
||||||
<th className="text-left px-4 py-2">主队</th>
|
|
||||||
<th className="text-left px-4 py-2">客队</th>
|
|
||||||
<th className="text-center px-4 py-2">比分</th>
|
|
||||||
<th className="text-center px-4 py-2">状态</th>
|
|
||||||
<th className="text-center px-4 py-2">操作</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
{loading && (
|
|
||||||
<tr><td colSpan={6} className="text-center text-gray-400 py-8">
|
|
||||||
<span className="inline-block animate-spin mr-2">⏳</span>加载中...
|
|
||||||
</td></tr>
|
|
||||||
)}
|
|
||||||
{!loading && matches.length === 0 && (
|
|
||||||
<tr><td colSpan={6} className="text-center text-gray-400 py-8">暂无数据,请先采集</td></tr>
|
|
||||||
)}
|
|
||||||
{matches.map(m => (
|
|
||||||
<tr key={m.id} className="border-t hover:bg-gray-50">
|
|
||||||
<td className="px-4 py-2 text-gray-600">{fmtDate(m.match_date)}</td>
|
|
||||||
<td className="px-4 py-2 font-medium">{m.home_team_zh || m.home_team}</td>
|
|
||||||
<td className="px-4 py-2 font-medium">{m.away_team_zh || m.away_team}</td>
|
|
||||||
<td className="px-4 py-2 text-center">
|
|
||||||
{m.home_goals !== null ? `${m.home_goals} - ${m.away_goals}` : '-'}
|
|
||||||
</td>
|
|
||||||
<td className="px-4 py-2 text-center">
|
|
||||||
<span className={`text-xs px-2 py-0.5 rounded ${
|
|
||||||
m.match_status === 'finished' ? 'bg-green-100 text-green-700' :
|
|
||||||
m.match_status === 'scheduled' ? 'bg-blue-100 text-blue-700' : 'bg-gray-100'
|
|
||||||
}`}>
|
|
||||||
{m.match_status === 'finished' ? '完赛' : m.match_status === 'scheduled' ? '未开赛' : m.match_status}
|
|
||||||
</span>
|
|
||||||
</td>
|
|
||||||
<td className="px-4 py-2 text-center">
|
|
||||||
<button onClick={() => predict(m.id)}
|
|
||||||
disabled={predictingId === m.id}
|
|
||||||
className="text-blue-600 hover:underline text-xs disabled:opacity-50">
|
|
||||||
{predictingId === m.id ? '预测中...' : 'LLM 预测'}
|
|
||||||
</button>
|
</button>
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
))}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* 预测中指示 */}
|
|
||||||
{predictingId && (
|
|
||||||
<div className="bg-blue-50 border border-blue-200 text-blue-700 px-4 py-3 rounded text-sm flex items-center gap-2">
|
|
||||||
<span className="inline-block animate-spin">⏳</span>
|
|
||||||
LLM 预测中,请稍候...
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{/* 预测结果 */}
|
{/* ── 赛程栏:表格化,行间细线 ── */}
|
||||||
{prediction && (
|
<section aria-label="赛程">
|
||||||
<div className="bg-white rounded border p-5 space-y-3">
|
{loading && <SkeletonRows n={4} />}
|
||||||
<h3 className="font-bold text-lg">🤖 LLM 预测结果</h3>
|
|
||||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm">
|
|
||||||
<div className="bg-blue-50 rounded p-3">
|
|
||||||
<div className="text-gray-500 text-xs">主进球</div>
|
|
||||||
<div className="text-xl font-bold">{prediction.pred_home_goals ?? '-'}</div>
|
|
||||||
</div>
|
|
||||||
<div className="bg-blue-50 rounded p-3">
|
|
||||||
<div className="text-gray-500 text-xs">客进球</div>
|
|
||||||
<div className="text-xl font-bold">{prediction.pred_away_goals ?? '-'}</div>
|
|
||||||
</div>
|
|
||||||
<div className="bg-amber-50 rounded p-3">
|
|
||||||
<div className="text-gray-500 text-xs">胜平负</div>
|
|
||||||
<div className="text-xl font-bold">{prediction.pred_1x2 ?? '-'}</div>
|
|
||||||
</div>
|
|
||||||
<div className="bg-green-50 rounded p-3">
|
|
||||||
<div className="text-gray-500 text-xs">置信度</div>
|
|
||||||
<div className="text-xl font-bold">
|
|
||||||
{prediction.subjective_confidence !== null ? `${(prediction.subjective_confidence * 100).toFixed(0)}%` : '-'}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div className="text-xs text-gray-400">
|
|
||||||
{prediction.provider} / {prediction.model} · 耗时 {prediction.latency_ms}ms
|
|
||||||
{prediction.mode === 'multi' && ' · 多 Agent 模式'}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* 各专家 agent 报告 */}
|
{!loading && matches.length === 0 && (
|
||||||
{prediction.agent_outputs && prediction.agent_outputs.length > 0 && (
|
<div className="border-y border-ink-200 py-14 text-center">
|
||||||
<div className="space-y-2">
|
<p className="font-serif text-sm text-ink-600">本版暂无赛程</p>
|
||||||
<div className="text-sm font-medium text-gray-700">专家 Agent 报告</div>
|
<p className="mt-1.5 text-xs text-ink-400">请先通过采集接口导入 {leagueName} 的比赛数据</p>
|
||||||
{prediction.agent_outputs.map((r) => (
|
|
||||||
<details key={r.agent} className="bg-white border rounded">
|
|
||||||
<summary className="cursor-pointer px-3 py-2 text-sm flex items-center justify-between">
|
|
||||||
<span className="font-medium">
|
|
||||||
{AGENT_LABELS[r.agent] || r.agent}
|
|
||||||
{r.status !== 'ok' && (
|
|
||||||
<span className={`ml-2 text-xs px-1.5 py-0.5 rounded ${
|
|
||||||
r.status === 'no_data' ? 'bg-gray-100 text-gray-500' : 'bg-red-100 text-red-600'
|
|
||||||
}`}>
|
|
||||||
{r.status === 'no_data' ? '无数据' : '失败'}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</span>
|
|
||||||
<span className="flex gap-3 text-xs text-gray-500">
|
|
||||||
{r.home_edge !== null && (
|
|
||||||
<span className={r.home_edge > 0 ? 'text-blue-600' : r.home_edge < 0 ? 'text-amber-600' : ''}>
|
|
||||||
主队优势 {r.home_edge > 0 ? '+' : ''}{r.home_edge.toFixed(2)}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
{r.subjective_confidence !== null && <span>信心 {(r.subjective_confidence * 100).toFixed(0)}%</span>}
|
|
||||||
{r.probable_score && <span>比分 {r.probable_score}</span>}
|
|
||||||
</span>
|
|
||||||
</summary>
|
|
||||||
<div className="px-3 pb-3 pt-1 space-y-2 text-sm">
|
|
||||||
{r.analysis && <p className="text-gray-700">{r.analysis}</p>}
|
|
||||||
{r.key_evidence.length > 0 && (
|
|
||||||
<ul className="text-xs text-gray-500 list-disc pl-4">
|
|
||||||
{r.key_evidence.map((e, i) => <li key={i}>{e}</li>)}
|
|
||||||
</ul>
|
|
||||||
)}
|
|
||||||
{r.exp_home_goals !== null && r.exp_away_goals !== null && (
|
|
||||||
<div className="text-xs text-gray-500">
|
|
||||||
进球期望: {r.exp_home_goals.toFixed(1)} - {r.exp_away_goals.toFixed(1)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
<div className="text-xs text-gray-400">
|
|
||||||
数据充分度 {r.data_sufficiency} · {r.model} · {r.latency_ms}ms
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</details>
|
|
||||||
))}
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{prediction.reasoning && (
|
{!loading && matches.map(m => {
|
||||||
<div className="bg-gray-50 rounded p-3">
|
const st = STATUS_META[m.match_status] ?? { label: m.match_status, cls: 'text-ink-400' }
|
||||||
<div className="text-xs text-gray-500 mb-1">推理过程</div>
|
const homeName = m.home_team_zh || m.home_team
|
||||||
<div className="text-sm whitespace-pre-wrap">{prediction.reasoning}</div>
|
const awayName = m.away_team_zh || m.away_team
|
||||||
|
const busy = predictingId === m.id
|
||||||
|
const active = predictionFor?.id === m.id
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={m.id}
|
||||||
|
className={`border-b border-ink-200 px-1 py-3 transition-colors hover:bg-paper-100 ${
|
||||||
|
active ? 'bg-press-wash/50' : ''
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<div className="flex flex-col gap-2 sm:grid sm:grid-cols-[88px_minmax(0,1fr)_64px_minmax(0,1fr)_56px_auto] sm:items-center sm:gap-x-3 sm:gap-y-0">
|
||||||
|
{/* 日期 + 状态:移动端同行,桌面端日期单独归列 */}
|
||||||
|
<div className="flex items-center justify-between sm:contents">
|
||||||
|
<span className="text-2xs tabular-nums text-ink-400">{fmtDate(m.match_date)}</span>
|
||||||
|
<span className={`text-2xs sm:hidden ${st.cls}`}>{st.label}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* 对阵:移动端主队/比分/客队同一行,桌面端 sm:contents 拆回 grid 列 */}
|
||||||
|
<div className="flex items-center gap-2 sm:contents">
|
||||||
|
{/* 主队(右对齐) */}
|
||||||
|
<div className="flex min-w-0 flex-1 items-center justify-end">
|
||||||
|
<span className="truncate text-sm font-medium text-ink-900">{homeName}</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* 比分 / VS */}
|
||||||
|
<div className="flex w-14 flex-shrink-0 flex-col items-center sm:w-auto">
|
||||||
|
{m.home_goals !== null && m.away_goals !== null ? (
|
||||||
|
<span className="font-serif text-base font-bold tabular-nums text-ink-900">
|
||||||
|
{m.home_goals}<span className="mx-0.5 font-normal text-ink-300">:</span>{m.away_goals}
|
||||||
|
</span>
|
||||||
|
) : (
|
||||||
|
<span className="text-2xs tracking-widest text-ink-400">VS</span>
|
||||||
|
)}
|
||||||
|
{m.home_xg !== null && m.away_xg !== null && (
|
||||||
|
<span className="text-2xs tabular-nums text-ink-400">
|
||||||
|
xG {m.home_xg.toFixed(1)}-{m.away_xg.toFixed(1)}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* 客队(左对齐) */}
|
||||||
|
<div className="flex min-w-0 flex-1 items-center">
|
||||||
|
<span className="truncate text-sm font-medium text-ink-900">{awayName}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* 状态列(桌面) */}
|
||||||
|
<span className={`hidden text-right text-2xs sm:block ${st.cls}`}>{st.label}</span>
|
||||||
|
|
||||||
|
{/* 预测按钮 */}
|
||||||
|
<div className="flex justify-end">
|
||||||
|
<button
|
||||||
|
onClick={() => predict(m)}
|
||||||
|
disabled={busy}
|
||||||
|
className="btn btn-sm w-[76px]"
|
||||||
|
title={`以${mode === 'multi' ? '多专家' : '单次'}模式预测这场`}
|
||||||
|
>
|
||||||
|
{busy ? (<><Spinner /> 预测中</>) : '预测'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
|
||||||
|
{!loading && nextCursor && (
|
||||||
|
<div className="flex justify-center pt-4">
|
||||||
|
<button onClick={loadMore} disabled={loadingMore} className="btn btn-sm">
|
||||||
|
{loadingMore ? (<><Spinner /> 获取中</>) : '载入更多'}
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<details className="text-xs">
|
</section>
|
||||||
<summary className="cursor-pointer text-gray-500 hover:text-gray-700">查看完整上下文</summary>
|
|
||||||
<pre className="mt-2 bg-gray-900 text-green-300 p-3 rounded overflow-x-auto text-xs">
|
{/* ── 预测中占位 ── */}
|
||||||
{prediction.context}
|
{predictingId && !prediction && (
|
||||||
</pre>
|
<div className="border border-ink-900">
|
||||||
</details>
|
<div className="flex items-center gap-2 border-b border-ink-900 bg-paper-100 px-4 py-2.5">
|
||||||
|
<Spinner className="text-press" />
|
||||||
|
<span className="text-sm font-medium text-ink-800">正在生成预测</span>
|
||||||
|
<span className="text-2xs text-ink-500">
|
||||||
|
{mode === 'multi' ? '五路专家并行分析后终裁,约需 20-60 秒' : '单次调用,约需 5-15 秒'}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
<div className="space-y-4 px-4 py-6">
|
||||||
|
<div className="flex items-center justify-center gap-6">
|
||||||
|
<div className="skeleton h-4 w-20" />
|
||||||
|
<div className="skeleton h-10 w-24" />
|
||||||
|
<div className="skeleton h-4 w-20" />
|
||||||
|
</div>
|
||||||
|
<div className="skeleton mx-auto h-px w-64" />
|
||||||
|
<div className="skeleton h-16 w-full" />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 预测版 ── */}
|
||||||
|
{prediction && predictionFor && (
|
||||||
|
<PredictionPanel prediction={prediction} match={predictionFor} mode={mode} />
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function PredictionPanel({
|
||||||
|
prediction,
|
||||||
|
match,
|
||||||
|
mode,
|
||||||
|
}: {
|
||||||
|
prediction: Prediction
|
||||||
|
match: Match
|
||||||
|
mode: 'single' | 'multi'
|
||||||
|
}) {
|
||||||
|
const homeName = match.home_team_zh || match.home_team
|
||||||
|
const awayName = match.away_team_zh || match.away_team
|
||||||
|
const okReports = (prediction.agent_outputs ?? []).filter(r => r.status === 'ok')
|
||||||
|
|
||||||
|
return (
|
||||||
|
<article className="border border-ink-900 bg-paper-50">
|
||||||
|
{/* 版头 */}
|
||||||
|
<div className="flex flex-wrap items-baseline justify-between gap-2 border-b border-ink-900 bg-paper-100 px-4 py-2.5 sm:px-5">
|
||||||
|
<h3 className="font-serif text-sm font-bold text-ink-900">
|
||||||
|
预测版 · {homeName} 对 {awayName}
|
||||||
|
</h3>
|
||||||
|
<span className="text-2xs tabular-nums text-ink-500">
|
||||||
|
{prediction.provider} / {prediction.model}
|
||||||
|
{prediction.latency_ms !== null && ` · ${(prediction.latency_ms / 1000).toFixed(1)}s`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="space-y-7 px-4 py-6 sm:px-5">
|
||||||
|
{/* ── 预测比分:版面核心,大号宋体 ── */}
|
||||||
|
<div className="text-center">
|
||||||
|
<p className="font-serif text-5xl font-bold tabular-nums leading-none text-ink-900 sm:text-6xl">
|
||||||
|
{prediction.pred_home_goals ?? '-'}
|
||||||
|
<span className="mx-3 font-normal text-ink-300">:</span>
|
||||||
|
{prediction.pred_away_goals ?? '-'}
|
||||||
|
</p>
|
||||||
|
<p className="mt-3 text-2xs tracking-[0.5em] text-ink-400">预测比分</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* ── 胜平负 ── */}
|
||||||
|
<div className="border-y border-ink-200 py-4">
|
||||||
|
<OutcomeLine pick={prediction.pred_1x2} confidence={prediction.subjective_confidence} />
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* ── 元信息一行 ── */}
|
||||||
|
<p className="text-center text-2xs text-ink-500">
|
||||||
|
{mode === 'multi' ? `多专家模式 · ${okReports.length}/${prediction.agent_outputs?.length ?? 0} 路有效` : '单次模式'}
|
||||||
|
{prediction.prompt_version && ` · prompt ${prediction.prompt_version}`}
|
||||||
|
{prediction.latency_ms !== null && ` · 终裁耗时 ${(prediction.latency_ms / 1000).toFixed(1)} 秒`}
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{/* ── 专家意见 ── */}
|
||||||
|
{prediction.agent_outputs && prediction.agent_outputs.length > 0 && (
|
||||||
|
<section>
|
||||||
|
<div className="section-head flex flex-wrap items-baseline justify-between gap-1">
|
||||||
|
<span>五路专家意见</span>
|
||||||
|
{prediction.agent_weights && (
|
||||||
|
<span className="font-sans text-2xs font-normal text-ink-500">
|
||||||
|
终裁权重:{Object.entries(prediction.agent_weights)
|
||||||
|
.sort((a, b) => b[1] - a[1])
|
||||||
|
.map(([k, v]) => `${AGENT_LABELS[k] ?? k} ${Math.round(v * 100)}%`)
|
||||||
|
.join(' / ')}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
{prediction.agent_outputs.map((r, i) => (
|
||||||
|
<AgentCard key={r.agent} report={r} no={CN_NUM[i] ?? String(i + 1)} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 终裁意见:引文式,红竖线 ── */}
|
||||||
|
{prediction.reasoning && (
|
||||||
|
<section>
|
||||||
|
<h4 className="section-head mb-3">终裁意见</h4>
|
||||||
|
<blockquote className="border-l-2 border-press pl-4">
|
||||||
|
<p className="whitespace-pre-wrap font-serif text-sm leading-loose text-ink-700">
|
||||||
|
{prediction.reasoning}
|
||||||
|
</p>
|
||||||
|
</blockquote>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 原始上下文 ── */}
|
||||||
|
<details className="group">
|
||||||
|
<summary className="flex cursor-pointer list-none items-center gap-1.5 text-xs text-ink-500 transition-colors hover:text-ink-800">
|
||||||
|
<svg viewBox="0 0 20 20" className="h-3 w-3 transition-transform group-open:rotate-90" fill="currentColor" aria-hidden="true">
|
||||||
|
<path d="M7.3 5.3a1 1 0 011.4 0l4 4a1 1 0 010 1.4l-4 4a1 1 0 01-1.4-1.4L10.6 10 7.3 6.7a1 1 0 010-1.4z" />
|
||||||
|
</svg>
|
||||||
|
查看喂给模型的完整数据切片
|
||||||
|
</summary>
|
||||||
|
<pre className="mt-2 max-h-80 overflow-auto border border-ink-200 bg-paper-100 p-3 font-mono text-2xs leading-relaxed text-ink-600">
|
||||||
|
{prediction.context}
|
||||||
|
</pre>
|
||||||
|
</details>
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
const STATUS_BADGE: Record<string, { label: string; cls: string }> = {
|
||||||
|
ok: { label: '正常', cls: 'text-ink-500' },
|
||||||
|
no_data: { label: '无数据', cls: 'text-ink-400' },
|
||||||
|
error: { label: '调用失败', cls: 'text-press' },
|
||||||
|
parse_error: { label: '解析失败', cls: 'text-press' },
|
||||||
|
}
|
||||||
|
|
||||||
|
const SUFFICIENCY_LABEL: Record<string, string> = {
|
||||||
|
high: '充分',
|
||||||
|
medium: '一般',
|
||||||
|
low: '偏少',
|
||||||
|
none: '无',
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单路专家意见:汉字编号 + 细线行 */
|
||||||
|
function AgentCard({ report: r, no }: { report: AgentReport; no: string }) {
|
||||||
|
const badge = STATUS_BADGE[r.status] ?? { label: r.status, cls: 'text-ink-400' }
|
||||||
|
const inactive = r.status !== 'ok'
|
||||||
|
|
||||||
|
return (
|
||||||
|
<details className="group border-b border-ink-200">
|
||||||
|
<summary className="flex cursor-pointer list-none items-baseline gap-2.5 px-1 py-3">
|
||||||
|
<span className="font-serif text-sm text-ink-400">{no}</span>
|
||||||
|
<span className="text-sm font-medium text-ink-900">{AGENT_LABELS[r.agent] ?? r.agent}</span>
|
||||||
|
<span className={`text-2xs ${badge.cls}`}>{badge.label}</span>
|
||||||
|
|
||||||
|
<span className="ml-auto flex items-baseline gap-3 text-2xs tabular-nums text-ink-500">
|
||||||
|
{r.status === 'ok' && r.subjective_confidence !== null && (
|
||||||
|
<span>信心 {Math.round(r.subjective_confidence * 100)}%</span>
|
||||||
|
)}
|
||||||
|
{r.status === 'ok' && r.probable_score && (
|
||||||
|
<span className="font-serif font-bold text-ink-800">{r.probable_score}</span>
|
||||||
|
)}
|
||||||
|
<svg viewBox="0 0 20 20" className="h-3 w-3 self-center text-ink-300 transition-transform group-open:rotate-90" fill="currentColor" aria-hidden="true">
|
||||||
|
<path d="M7.3 5.3a1 1 0 011.4 0l4 4a1 1 0 010 1.4l-4 4a1 1 0 01-1.4-1.4L10.6 10 7.3 6.7a1 1 0 010-1.4z" />
|
||||||
|
</svg>
|
||||||
|
</span>
|
||||||
|
</summary>
|
||||||
|
|
||||||
|
<div className="space-y-3 px-1 pb-4 pl-7">
|
||||||
|
{/* 无数据 / 失败时给出明确说明,避免用户以为是空白 bug */}
|
||||||
|
{inactive && (
|
||||||
|
<p className="text-xs leading-relaxed text-ink-500">
|
||||||
|
{r.status === 'no_data' && '该维度没有可用数据,已跳过 LLM 分析以节省额度(不影响其他专家)。'}
|
||||||
|
{r.status === 'error' && '该专家调用失败,本次结论未纳入其视角(fail-open 设计,不阻断整体预测)。'}
|
||||||
|
{r.status === 'parse_error' && '模型输出未通过格式校验,该报告已丢弃。'}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!inactive && r.home_edge !== null && (
|
||||||
|
<div>
|
||||||
|
<div className="mb-1.5 flex items-baseline justify-between text-2xs">
|
||||||
|
<span className="text-ink-500">主队优势</span>
|
||||||
|
<span className={`font-semibold tabular-nums ${r.home_edge > 0 ? 'text-press' : r.home_edge < 0 ? 'text-ink-700' : 'text-ink-500'}`}>
|
||||||
|
{r.home_edge > 0 ? '+' : ''}{r.home_edge.toFixed(2)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<EdgeBar value={r.home_edge} />
|
||||||
|
<div className="mt-1 flex justify-between text-2xs text-ink-400">
|
||||||
|
<span>利客队</span>
|
||||||
|
<span>利主队</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{r.analysis && (
|
||||||
|
<p className="font-serif text-sm leading-loose text-ink-700">{r.analysis}</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{r.key_evidence.length > 0 && (
|
||||||
|
<ul className="space-y-1.5">
|
||||||
|
{r.key_evidence.map((e, i) => (
|
||||||
|
<li key={i} className="flex gap-2 text-xs leading-relaxed text-ink-600">
|
||||||
|
<span className="flex-shrink-0 text-ink-300" aria-hidden="true">—</span>
|
||||||
|
<span>{e}</span>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{r.exp_home_goals !== null && r.exp_away_goals !== null && (
|
||||||
|
<p className="text-xs text-ink-500">
|
||||||
|
进球期望 <span className="font-serif font-bold tabular-nums text-ink-900">{r.exp_home_goals.toFixed(1)} - {r.exp_away_goals.toFixed(1)}</span>
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!inactive && (
|
||||||
|
<p className="border-t border-ink-100 pt-2.5 text-2xs text-ink-400">
|
||||||
|
数据充分度 {SUFFICIENCY_LABEL[r.data_sufficiency] ?? r.data_sufficiency}
|
||||||
|
<span className="mx-2 text-ink-200">|</span>
|
||||||
|
<span className="font-mono">{r.model}</span>
|
||||||
|
{r.latency_ms !== null && <span className="ml-2 tabular-nums">{r.latency_ms}ms</span>}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,6 +1,61 @@
|
|||||||
/** @type {import('tailwindcss').Config} */
|
/** @type {import('tailwindcss').Config} */
|
||||||
export default {
|
export default {
|
||||||
content: ['./index.html', './src/**/*.{ts,tsx}'],
|
content: ['./index.html', './src/**/*.{ts,tsx}'],
|
||||||
theme: { extend: {} },
|
theme: {
|
||||||
|
extend: {
|
||||||
|
colors: {
|
||||||
|
// 纸白:微暖底色,像新闻纸而不是纯白画布
|
||||||
|
paper: {
|
||||||
|
50: '#FDFCF8',
|
||||||
|
100: '#F7F4EC',
|
||||||
|
200: '#EDE9DE',
|
||||||
|
300: '#DDD7C7',
|
||||||
|
},
|
||||||
|
// 墨色:暖黑灰阶,替代冷调 slate
|
||||||
|
ink: {
|
||||||
|
50: '#FAF9F7',
|
||||||
|
100: '#F0EEE9',
|
||||||
|
200: '#E2DFD7',
|
||||||
|
300: '#C9C4B8',
|
||||||
|
400: '#9C9587',
|
||||||
|
500: '#6E675B',
|
||||||
|
600: '#524C42',
|
||||||
|
700: '#3B362E',
|
||||||
|
800: '#282420',
|
||||||
|
900: '#17140F',
|
||||||
|
},
|
||||||
|
// 印报红:全站唯一强调色,克制使用
|
||||||
|
press: {
|
||||||
|
DEFAULT: '#9E1B1B',
|
||||||
|
dark: '#7C1414',
|
||||||
|
wash: '#F7E9E4',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
fontFamily: {
|
||||||
|
// 标题与比分:宋体血统,报纸版面的骨架
|
||||||
|
serif: [
|
||||||
|
'Georgia',
|
||||||
|
'"Times New Roman"',
|
||||||
|
'"Songti SC"',
|
||||||
|
'STSong',
|
||||||
|
'SimSun',
|
||||||
|
'"Noto Serif CJK SC"',
|
||||||
|
'serif',
|
||||||
|
],
|
||||||
|
// 正文:中文黑体栈
|
||||||
|
sans: [
|
||||||
|
'"PingFang SC"',
|
||||||
|
'"Hiragino Sans GB"',
|
||||||
|
'"Microsoft YaHei"',
|
||||||
|
'"Noto Sans CJK SC"',
|
||||||
|
'sans-serif',
|
||||||
|
],
|
||||||
|
mono: ['ui-monospace', 'SFMono-Regular', 'Menlo', 'Consolas', 'monospace'],
|
||||||
|
},
|
||||||
|
fontSize: {
|
||||||
|
'2xs': ['11px', { lineHeight: '16px' }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
plugins: [],
|
plugins: [],
|
||||||
}
|
}
|
||||||
|
|||||||
+7
-2
@@ -16,6 +16,9 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
|||||||
from src.db.base import init_db
|
from src.db.base import init_db
|
||||||
from src.core.http_client import close_client
|
from src.core.http_client import close_client
|
||||||
await init_db() # 验证连接,不建表
|
await init_db() # 验证连接,不建表
|
||||||
|
# P2-5: 启动时执行一次 ingest failure 重试清理
|
||||||
|
from src.data.retry_worker import run_retry_worker
|
||||||
|
await run_retry_worker()
|
||||||
yield
|
yield
|
||||||
await close_client()
|
await close_client()
|
||||||
|
|
||||||
@@ -29,12 +32,14 @@ def create_app() -> FastAPI:
|
|||||||
)
|
)
|
||||||
|
|
||||||
origins = [o.strip() for o in settings.CORS_ORIGINS.split(",") if o.strip()]
|
origins = [o.strip() for o in settings.CORS_ORIGINS.split(",") if o.strip()]
|
||||||
|
methods = [m.strip() for m in settings.CORS_METHODS.split(",") if m.strip()]
|
||||||
|
headers = [h.strip() for h in settings.CORS_HEADERS.split(",") if h.strip()]
|
||||||
app.add_middleware(
|
app.add_middleware(
|
||||||
CORSMiddleware,
|
CORSMiddleware,
|
||||||
allow_origins=origins,
|
allow_origins=origins,
|
||||||
allow_credentials=True,
|
allow_credentials=True,
|
||||||
allow_methods=["*"],
|
allow_methods=methods,
|
||||||
allow_headers=["*"],
|
allow_headers=headers,
|
||||||
)
|
)
|
||||||
|
|
||||||
from src.api.routes.matches import router as matches_router
|
from src.api.routes.matches import router as matches_router
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""API 依赖:鉴权等横切关注点。
|
||||||
|
|
||||||
|
审查报告 P2-7:ingest / backtest / settle 这类「写入型或高成本」接口此前
|
||||||
|
完全无鉴权 —— 任何能访问到服务的人都可触发采集、或直接烧掉 LLM 额度。
|
||||||
|
|
||||||
|
策略(渐进式,不破坏本地开发):
|
||||||
|
- `ADMIN_API_KEY` 未配置 → 直接放行,并打一次 warning。
|
||||||
|
这样本地 `docker compose up` 无需额外配置即可用。
|
||||||
|
- 已配置 → 必须带匹配的 `X-API-Key` 请求头,否则 401。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import secrets
|
||||||
|
|
||||||
|
from fastapi import Header, HTTPException
|
||||||
|
|
||||||
|
from src.core.config import settings
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
async def require_admin_key(x_api_key: str | None = Header(None, alias="X-API-Key")) -> None:
|
||||||
|
"""保护「写入型 / 高成本」接口的依赖。
|
||||||
|
|
||||||
|
用法: `@router.post("/ingest/bzzoiro", dependencies=[Depends(require_admin_key)])`
|
||||||
|
"""
|
||||||
|
expected = settings.ADMIN_API_KEY
|
||||||
|
if not expected:
|
||||||
|
logger.warning(
|
||||||
|
"ADMIN_API_KEY 未设置,采集/回测接口当前【无鉴权】。"
|
||||||
|
"生产环境请设置该环境变量。"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not x_api_key or not secrets.compare_digest(x_api_key, expected):
|
||||||
|
raise HTTPException(status_code=401, detail="无效或缺失的 X-API-Key")
|
||||||
@@ -3,9 +3,10 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
from src.api.deps import require_admin_key
|
||||||
from src.llm.backtest import run_backtest
|
from src.llm.backtest import run_backtest
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -22,10 +23,13 @@ class BacktestRequest(BaseModel):
|
|||||||
model: str | None = Field(None, description="指定模型 (空=默认)")
|
model: str | None = Field(None, description="指定模型 (空=默认)")
|
||||||
|
|
||||||
|
|
||||||
@router.post("/backtest")
|
@router.post("/backtest", dependencies=[Depends(require_admin_key)])
|
||||||
async def backtest(req: BacktestRequest):
|
async def backtest(req: BacktestRequest):
|
||||||
"""对历史比赛运行回测。
|
"""对历史比赛运行回测。
|
||||||
|
|
||||||
|
该接口会对每场已完赛比赛各发起一次 LLM 预测,成本高 —— 因此需要
|
||||||
|
`X-API-Key` 鉴权(见审查报告 P2-7)。
|
||||||
|
|
||||||
对每场已完赛比赛:
|
对每场已完赛比赛:
|
||||||
1. 用比赛之前的数据构建上下文 (防未来信息泄漏)
|
1. 用比赛之前的数据构建上下文 (防未来信息泄漏)
|
||||||
2. 调 LLM 预测
|
2. 调 LLM 预测
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import logging
|
|||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
|
||||||
|
from src.api.deps import require_admin_key
|
||||||
from src.api.schemas import EvalSummaryOut, SettleRequest
|
from src.api.schemas import EvalSummaryOut, SettleRequest
|
||||||
from src.db.base import AsyncSession, get_db, get_db_read
|
from src.db.base import AsyncSession, get_db, get_db_read
|
||||||
from src.llm.eval import get_eval_summary, settle_prediction
|
from src.llm.eval import get_eval_summary, settle_prediction
|
||||||
@@ -14,7 +15,7 @@ logger = logging.getLogger(__name__)
|
|||||||
router = APIRouter(prefix="/api/v1", tags=["eval"])
|
router = APIRouter(prefix="/api/v1", tags=["eval"])
|
||||||
|
|
||||||
|
|
||||||
@router.post("/eval/settle")
|
@router.post("/eval/settle", dependencies=[Depends(require_admin_key)])
|
||||||
async def settle(req: SettleRequest, db: AsyncSession = Depends(get_db)):
|
async def settle(req: SettleRequest, db: AsyncSession = Depends(get_db)):
|
||||||
"""回填实际结果。"""
|
"""回填实际结果。"""
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -3,8 +3,9 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
from fastapi import APIRouter, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
|
||||||
|
from src.api.deps import require_admin_key
|
||||||
from src.api.schemas import IngestBzzoiroRequest, IngestResponse, IngestUnderstatRequest, IngestInjuriesRequest, IngestSimpleResponse
|
from src.api.schemas import IngestBzzoiroRequest, IngestResponse, IngestUnderstatRequest, IngestInjuriesRequest, IngestSimpleResponse
|
||||||
from src.data.sources import get_source
|
from src.data.sources import get_source
|
||||||
from src.data.injuries import ingest_injuries
|
from src.data.injuries import ingest_injuries
|
||||||
@@ -15,7 +16,7 @@ logger = logging.getLogger(__name__)
|
|||||||
router = APIRouter(prefix="/api/v1", tags=["ingest"])
|
router = APIRouter(prefix="/api/v1", tags=["ingest"])
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ingest/bzzoiro", response_model=IngestResponse)
|
@router.post("/ingest/bzzoiro", response_model=IngestResponse, dependencies=[Depends(require_admin_key)])
|
||||||
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
||||||
"""触发 bzzoiro 采集。"""
|
"""触发 bzzoiro 采集。"""
|
||||||
source = get_source("bzzoiro")
|
source = get_source("bzzoiro")
|
||||||
@@ -34,7 +35,7 @@ async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
|||||||
raise HTTPException(500, "数据采集失败,请查看服务器日志")
|
raise HTTPException(500, "数据采集失败,请查看服务器日志")
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ingest/understat", response_model=IngestSimpleResponse)
|
@router.post("/ingest/understat", response_model=IngestSimpleResponse, dependencies=[Depends(require_admin_key)])
|
||||||
async def ingest_understat_route(req: IngestUnderstatRequest):
|
async def ingest_understat_route(req: IngestUnderstatRequest):
|
||||||
"""触发 understat xG 回填。"""
|
"""触发 understat xG 回填。"""
|
||||||
source = get_source("understat")
|
source = get_source("understat")
|
||||||
@@ -47,7 +48,7 @@ async def ingest_understat_route(req: IngestUnderstatRequest):
|
|||||||
raise HTTPException(500, "xG 回填失败,请查看服务器日志")
|
raise HTTPException(500, "xG 回填失败,请查看服务器日志")
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ingest/injuries", response_model=IngestSimpleResponse)
|
@router.post("/ingest/injuries", response_model=IngestSimpleResponse, dependencies=[Depends(require_admin_key)])
|
||||||
async def ingest_injuries_route(req: IngestInjuriesRequest):
|
async def ingest_injuries_route(req: IngestInjuriesRequest):
|
||||||
"""触发伤停采集。"""
|
"""触发伤停采集。"""
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -32,6 +32,23 @@ class Settings(BaseSettings):
|
|||||||
|
|
||||||
# --- CORS ---
|
# --- CORS ---
|
||||||
CORS_ORIGINS: str = "http://localhost:5173,http://localhost:3000"
|
CORS_ORIGINS: str = "http://localhost:5173,http://localhost:3000"
|
||||||
|
CORS_METHODS: str = "GET,POST,PUT,DELETE,OPTIONS"
|
||||||
|
CORS_HEADERS: str = "Authorization,Content-Type,X-API-Key,Accept"
|
||||||
|
|
||||||
|
# --- HTTP ---
|
||||||
|
HTTP_DEFAULT_TIMEOUT: int = 30
|
||||||
|
|
||||||
|
# --- database pool ---
|
||||||
|
DB_POOL_SIZE: int = 5
|
||||||
|
DB_MAX_OVERFLOW: int = 10
|
||||||
|
DB_POOL_TIMEOUT: int = 30
|
||||||
|
DB_POOL_RECYCLE: int = 1800
|
||||||
|
|
||||||
|
# --- 管理接口鉴权 ---
|
||||||
|
# 采集 / 回测等高成本或写入型接口需要此 Key(请求头 X-API-Key)。
|
||||||
|
# 留空表示「未启用鉴权」(本地开发默认),生产环境必须设置。
|
||||||
|
# 见审查报告 P2-7:ingest/backtest 无鉴权可被任意调用并烧掉 LLM 额度。
|
||||||
|
ADMIN_API_KEY: str = ""
|
||||||
|
|
||||||
|
|
||||||
settings = Settings()
|
settings = Settings()
|
||||||
|
|||||||
@@ -2,24 +2,27 @@
|
|||||||
|
|
||||||
使用方:
|
使用方:
|
||||||
- src/llm/provider.py: LLM 调用
|
- src/llm/provider.py: LLM 调用
|
||||||
|
- src/data/bzzoiro.py: bzzoiro 比赛数据
|
||||||
- src/data/understat.py: xG 抓取
|
- src/data/understat.py: xG 抓取
|
||||||
- src/data/injuries.py: 伤停抓取
|
- src/data/injuries.py: 伤停抓取
|
||||||
|
|
||||||
生命周期由 FastAPI lifespan 管理(关闭时 aclose)。
|
生命周期由 FastAPI lifespan 管理(关闭时 aclose)。
|
||||||
|
调用方可通过 `timeout` 参数覆盖 per-request 超时。
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
from src.core.config import settings
|
||||||
|
|
||||||
_shared_client: httpx.AsyncClient | None = None
|
_shared_client: httpx.AsyncClient | None = None
|
||||||
_default_timeout = 30
|
|
||||||
|
|
||||||
|
|
||||||
def get_client() -> httpx.AsyncClient:
|
def get_client() -> httpx.AsyncClient:
|
||||||
"""获取共享客户端(懒初始化)。"""
|
"""获取共享客户端(懒初始化)。"""
|
||||||
global _shared_client
|
global _shared_client
|
||||||
if _shared_client is None or _shared_client.is_closed:
|
if _shared_client is None or _shared_client.is_closed:
|
||||||
_shared_client = httpx.AsyncClient(timeout=_default_timeout)
|
_shared_client = httpx.AsyncClient(timeout=settings.HTTP_DEFAULT_TIMEOUT)
|
||||||
return _shared_client
|
return _shared_client
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,82 +0,0 @@
|
|||||||
"""重试工具:带指数退避的瞬态错误重试。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import functools
|
|
||||||
import logging
|
|
||||||
import random
|
|
||||||
import time
|
|
||||||
from typing import Callable, Iterable, TypeVar
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
T = TypeVar("T")
|
|
||||||
|
|
||||||
|
|
||||||
def with_retry(
|
|
||||||
*,
|
|
||||||
max_retries: int = 3,
|
|
||||||
base_delay: float = 1.0,
|
|
||||||
max_delay: float = 30.0,
|
|
||||||
retryable_exceptions: Iterable[type[BaseException]] = (Exception,),
|
|
||||||
on_retry: Callable[[Exception, int], None] | None = None,
|
|
||||||
) -> Callable:
|
|
||||||
"""重试装饰器(同步/异步通用,指数退避 + 抖动)。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
max_retries: 最大重试次数
|
|
||||||
base_delay: 基础延迟(秒)
|
|
||||||
max_delay: 最大延迟(秒)
|
|
||||||
retryable_exceptions: 触发重试的异常类型
|
|
||||||
on_retry: 重试回调(exception, attempt)
|
|
||||||
"""
|
|
||||||
retryable = tuple(retryable_exceptions)
|
|
||||||
|
|
||||||
def decorator(func: Callable) -> Callable:
|
|
||||||
@functools.wraps(func)
|
|
||||||
async def async_wrapper(*args, **kwargs):
|
|
||||||
last_exc: Exception | None = None
|
|
||||||
for attempt in range(max_retries + 1):
|
|
||||||
try:
|
|
||||||
return await func(*args, **kwargs)
|
|
||||||
except retryable as e:
|
|
||||||
last_exc = e
|
|
||||||
if attempt == max_retries:
|
|
||||||
break
|
|
||||||
delay = min(base_delay * (2 ** attempt), max_delay)
|
|
||||||
delay += random.uniform(0, delay * 0.1) # 抖动
|
|
||||||
logger.warning(
|
|
||||||
"%s failed (attempt %d/%d), retry in %.1fs: %s",
|
|
||||||
func.__name__, attempt + 1, max_retries, delay, e,
|
|
||||||
)
|
|
||||||
if on_retry:
|
|
||||||
on_retry(e, attempt + 1)
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
raise last_exc # type: ignore[misc]
|
|
||||||
|
|
||||||
@functools.wraps(func)
|
|
||||||
def sync_wrapper(*args, **kwargs):
|
|
||||||
last_exc: Exception | None = None
|
|
||||||
for attempt in range(max_retries + 1):
|
|
||||||
try:
|
|
||||||
return func(*args, **kwargs)
|
|
||||||
except retryable as e:
|
|
||||||
last_exc = e
|
|
||||||
if attempt == max_retries:
|
|
||||||
break
|
|
||||||
delay = min(base_delay * (2 ** attempt), max_delay)
|
|
||||||
delay += random.uniform(0, delay * 0.1)
|
|
||||||
logger.warning(
|
|
||||||
"%s failed (attempt %d/%d), retry in %.1fs: %s",
|
|
||||||
func.__name__, attempt + 1, max_retries, delay, e,
|
|
||||||
)
|
|
||||||
if on_retry:
|
|
||||||
on_retry(e, attempt + 1)
|
|
||||||
time.sleep(delay)
|
|
||||||
raise last_exc # type: ignore[misc]
|
|
||||||
|
|
||||||
if asyncio.iscoroutinefunction(func):
|
|
||||||
return async_wrapper
|
|
||||||
return sync_wrapper
|
|
||||||
|
|
||||||
return decorator
|
|
||||||
+312
-46
@@ -2,6 +2,12 @@
|
|||||||
|
|
||||||
迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。
|
迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。
|
||||||
使用 Repository 模式进行数据访问,不直接控制事务。
|
使用 Repository 模式进行数据访问,不直接控制事务。
|
||||||
|
|
||||||
|
改进(P1):
|
||||||
|
- Bronze 层集成:采集后先存 RawEvent,再规范化
|
||||||
|
- 死信表集成:采集/规范化失败写 IngestFailure
|
||||||
|
- 令牌桶限流:替换固定 REQUEST_INTERVAL sleep
|
||||||
|
- 数据血缘:记录 ETL 全过程
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -9,61 +15,98 @@ import asyncio
|
|||||||
import json as _json
|
import json as _json
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
import time as _time
|
import uuid
|
||||||
import urllib.error
|
|
||||||
import urllib.parse
|
|
||||||
import urllib.request
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|
||||||
from src.core.config import settings
|
from src.core.config import settings
|
||||||
|
from src.core.http_client import get_client
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||||
from src.data.normalize import normalize_bzzoiro
|
from src.data.normalize import normalize_bzzoiro
|
||||||
|
from src.data.rate_limiter import RateLimitedClient, TokenBucket
|
||||||
from src.data.sources import register
|
from src.data.sources import register
|
||||||
from src.db.models import League, Match, MatchStats, Team
|
from src.db.models import DataLineage, DataQualityCheck, IngestFailure, League, Match, MatchStats, RawEvent, Team
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 令牌桶限流器:替代固定 sleep,rate 根据 REQUEST_INTERVAL 计算
|
||||||
|
_bzzoiro_limiter = TokenBucket(rate=1.0 / REQUEST_INTERVAL, capacity=3)
|
||||||
|
# P2-3: RateLimitedClient 包装,在 HTTP 调用层面透明限流
|
||||||
|
_bzzoiro_rl_client: RateLimitedClient | None = None
|
||||||
|
|
||||||
def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
|
||||||
"""同步 HTTP(bzzoiro 客户端保持同步,在 async 函数里 run_in_executor)。"""
|
def _get_bzzoiro_rl_client() -> RateLimitedClient:
|
||||||
|
"""惰性初始化 RateLimitedClient(避免模块导入时创建 client)。"""
|
||||||
|
global _bzzoiro_rl_client
|
||||||
|
if _bzzoiro_rl_client is None:
|
||||||
|
client = get_client()
|
||||||
|
_bzzoiro_rl_client = RateLimitedClient(client, _bzzoiro_limiter)
|
||||||
|
return _bzzoiro_rl_client
|
||||||
|
|
||||||
|
|
||||||
|
def _to_date(value):
|
||||||
|
"""把 datetime / date / str 统一成 `date`。"""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if hasattr(value, "date") and callable(value.date):
|
||||||
|
return value.date()
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]:
|
||||||
|
"""比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。
|
||||||
|
|
||||||
|
统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类
|
||||||
|
隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配,
|
||||||
|
导致所有比赛被判为不存在而重复插入。
|
||||||
|
"""
|
||||||
|
d = _to_date(match_date)
|
||||||
|
return (home_team_id, away_team_id, d.isoformat() if d is not None else "")
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||||
|
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。"""
|
||||||
base = settings.BZZOIRO_BASE.rstrip("/")
|
base = settings.BZZOIRO_BASE.rstrip("/")
|
||||||
url = f"{base}/{path.lstrip('/')}"
|
url = f"{base}/{path.lstrip('/')}"
|
||||||
if params:
|
|
||||||
url += "?" + urllib.parse.urlencode(params)
|
|
||||||
key = settings.BZZOIRO_KEY
|
key = settings.BZZOIRO_KEY
|
||||||
if not key:
|
if not key:
|
||||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Token {key}",
|
||||||
|
"Accept": "application/json",
|
||||||
|
}
|
||||||
|
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(max_retries):
|
for attempt in range(max_retries):
|
||||||
try:
|
try:
|
||||||
req = urllib.request.Request(url)
|
# P2-3: 使用 RateLimitedClient 包装 HTTP 调用,透明限流
|
||||||
req.add_header("Authorization", f"Token {key}")
|
rl_client = _get_bzzoiro_rl_client()
|
||||||
req.add_header("Accept", "application/json")
|
resp = await rl_client.get(url, headers=headers, params=params, timeout=30)
|
||||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
resp.raise_for_status()
|
||||||
return _json.loads(resp.read().decode("utf-8"))
|
return resp.json()
|
||||||
except urllib.error.HTTPError as e:
|
except Exception as e:
|
||||||
last_exc = e
|
last_exc = e
|
||||||
if e.code == 429:
|
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||||
# 指数退避: 429 通常意味着限速
|
if status == 429:
|
||||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||||
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
||||||
_time.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
continue
|
continue
|
||||||
if 500 <= e.code < 600:
|
if 500 <= (status or 0) < 600:
|
||||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||||
logger.warning("bzzoiro %d, retry %d in %.1fs", e.code, attempt + 1, delay)
|
logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay)
|
||||||
_time.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
continue
|
continue
|
||||||
raise # 4xx 直接抛
|
# 网络错误(连接失败/超时)也退避重试
|
||||||
except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
|
if isinstance(e, (TimeoutError, ConnectionError, OSError)):
|
||||||
last_exc = e
|
|
||||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||||
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||||
_time.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
|
continue
|
||||||
|
raise
|
||||||
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
||||||
|
|
||||||
|
|
||||||
@@ -75,15 +118,13 @@ async def fetch_bzzoiro_events(
|
|||||||
date_to: str | None = None,
|
date_to: str | None = None,
|
||||||
limit: int = 200,
|
limit: int = 200,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""抓取 bzzoiro 原始事件(异步包装)。"""
|
"""抓取 bzzoiro 原始事件(纯异步,无需 run_in_executor)。"""
|
||||||
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
||||||
if league_id is None:
|
if league_id is None:
|
||||||
raise ValueError(f"未知联赛代码: {league_code}")
|
raise ValueError(f"未知联赛代码: {league_code}")
|
||||||
|
|
||||||
loop = asyncio.get_event_loop()
|
|
||||||
rows: list[dict] = []
|
rows: list[dict] = []
|
||||||
offset = 0
|
offset = 0
|
||||||
payload: dict | list = {}
|
|
||||||
while True:
|
while True:
|
||||||
params: dict = {
|
params: dict = {
|
||||||
"league_id": league_id,
|
"league_id": league_id,
|
||||||
@@ -95,8 +136,7 @@ async def fetch_bzzoiro_events(
|
|||||||
params["date_from"] = str(date_from)[:10]
|
params["date_from"] = str(date_from)[:10]
|
||||||
if date_to:
|
if date_to:
|
||||||
params["date_to"] = str(date_to)[:10]
|
params["date_to"] = str(date_to)[:10]
|
||||||
# 显式位置参数,避免 lambda 闭包捕获循环变量
|
payload = await _fetch_json_async("/events/", params)
|
||||||
payload = await loop.run_in_executor(None, _fetch_json_sync, "/events/", params)
|
|
||||||
batch = payload.get("results") or []
|
batch = payload.get("results") or []
|
||||||
if not batch:
|
if not batch:
|
||||||
break
|
break
|
||||||
@@ -107,10 +147,141 @@ async def fetch_bzzoiro_events(
|
|||||||
break
|
break
|
||||||
if len(batch) < limit:
|
if len(batch) < limit:
|
||||||
break
|
break
|
||||||
await asyncio.sleep(REQUEST_INTERVAL)
|
# 令牌桶已在 _fetch_json_async 内部处理,这里不再需要固定 sleep
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_bronze_events(db, raw_events: list[dict], batch_id: str) -> None:
|
||||||
|
"""将原始事件写入 Bronze 层(RawEvent 表)。
|
||||||
|
|
||||||
|
P2-1: 写入血缘记录,追踪 Bronze 层摄取过程。
|
||||||
|
"""
|
||||||
|
for raw in raw_events:
|
||||||
|
try:
|
||||||
|
source_record_id = str(raw.get("id", ""))
|
||||||
|
if not source_record_id:
|
||||||
|
# P1-fix: 不再静默跳过,记录到死信表
|
||||||
|
logger.warning("bzzoiro raw event missing id: %s", str(raw)[:200])
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "bzzoiro", "match", "",
|
||||||
|
"normalize_error", "raw event missing id", raw,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
bronze = RawEvent(
|
||||||
|
source_system="bzzoiro",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
raw_payload=raw,
|
||||||
|
ingest_batch_id=batch_id,
|
||||||
|
)
|
||||||
|
db.add(bronze)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("bzzoiro raw_event write failed: %s", e)
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("bzzoiro raw_events flush failed: %s", e)
|
||||||
|
|
||||||
|
# P2-1: 写入 Bronze 层血缘记录
|
||||||
|
for raw in raw_events:
|
||||||
|
source_record_id = str(raw.get("id", ""))
|
||||||
|
if not source_record_id:
|
||||||
|
continue
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="bzzoiro",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
target_table="raw_events",
|
||||||
|
transform_name="bronze_ingest",
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_ingest_failure(
|
||||||
|
db,
|
||||||
|
source_system: str,
|
||||||
|
entity_type: str,
|
||||||
|
source_record_id: str,
|
||||||
|
error_type: str,
|
||||||
|
error_detail: str,
|
||||||
|
raw_payload: dict | None,
|
||||||
|
) -> None:
|
||||||
|
"""写入采集失败到死信表(IngestFailure)。"""
|
||||||
|
try:
|
||||||
|
from datetime import timedelta
|
||||||
|
failure = IngestFailure(
|
||||||
|
source_system=source_system,
|
||||||
|
entity_type=entity_type,
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
error_type=error_type,
|
||||||
|
error_detail=error_detail[:2000] if error_detail else None,
|
||||||
|
raw_payload=raw_payload,
|
||||||
|
status="pending",
|
||||||
|
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
db.add(failure)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("bzzoiro ingest_failure write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_data_lineage(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
source_system: str,
|
||||||
|
source_record_id: str,
|
||||||
|
target_table: str,
|
||||||
|
target_id: int | None = None,
|
||||||
|
transform_name: str,
|
||||||
|
transform_detail: str | None = None,
|
||||||
|
batch_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""写入数据血缘记录(DataLineage 表)。"""
|
||||||
|
try:
|
||||||
|
lineage = DataLineage(
|
||||||
|
source_system=source_system,
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
target_table=target_table,
|
||||||
|
target_id=target_id,
|
||||||
|
transform_name=transform_name,
|
||||||
|
transform_detail=transform_detail,
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
|
db.add(lineage)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("bzzoiro data_lineage write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_data_quality_check(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
check_name: str,
|
||||||
|
entity_type: str,
|
||||||
|
entity_id: str | None = None,
|
||||||
|
expected_value: str | None = None,
|
||||||
|
actual_value: str | None = None,
|
||||||
|
passed: bool,
|
||||||
|
severity: str = "warning",
|
||||||
|
detail: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""写入数据质量检查结果(DataQualityCheck 表)。"""
|
||||||
|
try:
|
||||||
|
check = DataQualityCheck(
|
||||||
|
check_name=check_name,
|
||||||
|
entity_type=entity_type,
|
||||||
|
entity_id=entity_id,
|
||||||
|
expected_value=expected_value,
|
||||||
|
actual_value=actual_value,
|
||||||
|
passed=passed,
|
||||||
|
severity=severity,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
db.add(check)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("bzzoiro data_quality_check write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
class BzzoiroSource:
|
class BzzoiroSource:
|
||||||
"""bzzoiro 数据源(实现 DataSource 协议)。"""
|
"""bzzoiro 数据源(实现 DataSource 协议)。"""
|
||||||
@@ -131,6 +302,7 @@ class BzzoiroSource:
|
|||||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||||
"""
|
"""
|
||||||
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||||
|
batch_id = uuid.uuid4().hex[:16]
|
||||||
|
|
||||||
for code in leagues:
|
for code in leagues:
|
||||||
league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
|
league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
|
||||||
@@ -139,9 +311,17 @@ class BzzoiroSource:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("bzzoiro fetch failed for %s", code)
|
logger.exception("bzzoiro fetch failed for %s", code)
|
||||||
league_r["errors"].append(f"fetch failed: {e}")
|
league_r["errors"].append(f"fetch failed: {e}")
|
||||||
|
# 写死信表
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "bzzoiro", "match", f"fetch_{code}_{batch_id}",
|
||||||
|
"fetch_error", str(e), {"league": code, "date_from": date_from, "date_to": date_to},
|
||||||
|
)
|
||||||
result["leagues"][code] = league_r
|
result["leagues"][code] = league_r
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# === Bronze 层:先存 RawEvent ===
|
||||||
|
await _write_bronze_events(db, raw_events, batch_id)
|
||||||
|
|
||||||
# 获取或创建联赛
|
# 获取或创建联赛
|
||||||
stmt = select(League).where(League.code == code)
|
stmt = select(League).where(League.code == code)
|
||||||
league = (await db.execute(stmt)).scalar_one_or_none()
|
league = (await db.execute(stmt)).scalar_one_or_none()
|
||||||
@@ -153,7 +333,9 @@ class BzzoiroSource:
|
|||||||
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
||||||
team_name_to_id: dict[str, int] = {}
|
team_name_to_id: dict[str, int] = {}
|
||||||
existing_matches: dict[tuple[int, int, str], Match] = {} # 完整对象,避免重复查询
|
existing_matches: dict[tuple[int, int, str], Match] = {} # 完整对象,避免重复查询
|
||||||
normalized_matches: list = [] # 缓存规范化结果,避免重复调用
|
# (NormalizedMatch, 原始 event) 成对保存:后续写 source_record_id 时
|
||||||
|
# 必须用配对的那条 event,不能依赖外层循环变量残留值。
|
||||||
|
normalized_matches: list[tuple] = []
|
||||||
|
|
||||||
if raw_events:
|
if raw_events:
|
||||||
# 一次遍历: 收集球队名 + 规范化
|
# 一次遍历: 收集球队名 + 规范化
|
||||||
@@ -164,10 +346,15 @@ class BzzoiroSource:
|
|||||||
try:
|
try:
|
||||||
nm.validate()
|
nm.validate()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("normalize skip: %s", e)
|
# P1-3: 统一使用 warning,不追加到 errors(仅运行时错误入 errors)
|
||||||
league_r["errors"].append(f"normalize: {e}")
|
logger.warning("normalize skip: %s", e)
|
||||||
|
# 写死信表:规范化失败
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "bzzoiro", "match", str(raw.get("id", "")),
|
||||||
|
"normalize_error", str(e), raw,
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
normalized_matches.append(nm)
|
normalized_matches.append((nm, raw))
|
||||||
all_team_names.add(nm.home_team)
|
all_team_names.add(nm.home_team)
|
||||||
all_team_names.add(nm.away_team)
|
all_team_names.add(nm.away_team)
|
||||||
|
|
||||||
@@ -176,13 +363,27 @@ class BzzoiroSource:
|
|||||||
teams = (await db.execute(stmt)).scalars().all()
|
teams = (await db.execute(stmt)).scalars().all()
|
||||||
team_name_to_id = {t.name: t.id for t in teams}
|
team_name_to_id = {t.name: t.id for t in teams}
|
||||||
|
|
||||||
# 预加载已有比赛(完整对象)
|
# P1-2: 按需加载,只加载 raw_events 涉及日期范围的比赛(加 30 天缓冲)
|
||||||
stmt = select(Match).where(Match.league_id == league.id)
|
# 避免加载联赛全部历史比赛到内存(多赛季采集时内存溢出)
|
||||||
for m in (await db.execute(stmt)).scalars():
|
if normalized_matches:
|
||||||
key = (m.home_team_id, m.away_team_id, str(m.match_date_date))
|
from datetime import timedelta
|
||||||
existing_matches[key] = m
|
dates = [nm.date for nm in normalized_matches if nm.date is not None]
|
||||||
|
if dates:
|
||||||
|
min_dt = min(dates) - timedelta(days=30)
|
||||||
|
max_dt = max(dates) + timedelta(days=30)
|
||||||
|
stmt = (
|
||||||
|
select(Match)
|
||||||
|
.where(Match.league_id == league.id)
|
||||||
|
.where(Match.match_date >= min_dt)
|
||||||
|
.where(Match.match_date <= max_dt)
|
||||||
|
)
|
||||||
|
existing_matches = {
|
||||||
|
_match_key(m.home_team_id, m.away_team_id, m.match_date_date): m
|
||||||
|
for m in (await db.execute(stmt)).scalars()
|
||||||
|
}
|
||||||
|
# else: existing_matches 保持空 dict(全量新比赛)
|
||||||
|
|
||||||
for nm in normalized_matches:
|
for nm, raw in normalized_matches:
|
||||||
# 球队: 内存查找 + 按需创建
|
# 球队: 内存查找 + 按需创建
|
||||||
home_team_id = team_name_to_id.get(nm.home_team)
|
home_team_id = team_name_to_id.get(nm.home_team)
|
||||||
if home_team_id is None:
|
if home_team_id is None:
|
||||||
@@ -201,8 +402,7 @@ class BzzoiroSource:
|
|||||||
team_name_to_id[nm.away_team] = away_team_id
|
team_name_to_id[nm.away_team] = away_team_id
|
||||||
|
|
||||||
# 查找已有比赛: 内存查找
|
# 查找已有比赛: 内存查找
|
||||||
date_key = nm.date.date().isoformat() if hasattr(nm.date, "date") else str(nm.date)
|
match_key = _match_key(home_team_id, away_team_id, nm.date)
|
||||||
match_key = (home_team_id, away_team_id, date_key)
|
|
||||||
existing_match = existing_matches.get(match_key)
|
existing_match = existing_matches.get(match_key)
|
||||||
|
|
||||||
if existing_match is None:
|
if existing_match is None:
|
||||||
@@ -212,7 +412,7 @@ class BzzoiroSource:
|
|||||||
home_team_id=home_team_id,
|
home_team_id=home_team_id,
|
||||||
away_team_id=away_team_id,
|
away_team_id=away_team_id,
|
||||||
match_date=nm.date,
|
match_date=nm.date,
|
||||||
match_date_date=nm.date.date() if hasattr(nm.date, "date") else nm.date,
|
match_date_date=_to_date(nm.date),
|
||||||
match_status=nm.match_status,
|
match_status=nm.match_status,
|
||||||
home_goals=nm.home_goals,
|
home_goals=nm.home_goals,
|
||||||
away_goals=nm.away_goals,
|
away_goals=nm.away_goals,
|
||||||
@@ -223,6 +423,16 @@ class BzzoiroSource:
|
|||||||
db.add(m)
|
db.add(m)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
existing_matches[match_key] = m # 防止同批重复
|
existing_matches[match_key] = m # 防止同批重复
|
||||||
|
# P2-1: Silver 层血缘 — Match 创建
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="bzzoiro",
|
||||||
|
source_record_id=str(raw.get("id", "")),
|
||||||
|
target_table="matches",
|
||||||
|
target_id=m.id,
|
||||||
|
transform_name="silver_upsert",
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
if nm.home_xg is not None or nm.away_xg is not None:
|
if nm.home_xg is not None or nm.away_xg is not None:
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
stats = MatchStats(
|
stats = MatchStats(
|
||||||
@@ -241,11 +451,22 @@ class BzzoiroSource:
|
|||||||
home_red_cards=nm.home_red_cards,
|
home_red_cards=nm.home_red_cards,
|
||||||
away_red_cards=nm.away_red_cards,
|
away_red_cards=nm.away_red_cards,
|
||||||
source="bzzoiro",
|
source="bzzoiro",
|
||||||
source_event_id=str(raw.get("id", "")),
|
source_record_id=str(raw.get("id", "")),
|
||||||
retrieved_at=now,
|
retrieved_at=now,
|
||||||
available_at=now,
|
available_at=now,
|
||||||
)
|
)
|
||||||
db.add(stats)
|
db.add(stats)
|
||||||
|
await db.flush()
|
||||||
|
# P2-1: Silver 层血缘 — MatchStats 创建
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="bzzoiro",
|
||||||
|
source_record_id=str(raw.get("id", "")),
|
||||||
|
target_table="match_stats",
|
||||||
|
target_id=m.id,
|
||||||
|
transform_name="silver_upsert",
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
league_r["inserted"] += 1
|
league_r["inserted"] += 1
|
||||||
else:
|
else:
|
||||||
# 已有比赛: 直接从内存获取对象更新(无需再查询)
|
# 已有比赛: 直接从内存获取对象更新(无需再查询)
|
||||||
@@ -263,9 +484,39 @@ class BzzoiroSource:
|
|||||||
existing_match.match_stage = nm.match_stage
|
existing_match.match_stage = nm.match_stage
|
||||||
changed = True
|
changed = True
|
||||||
if existing_match.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
if existing_match.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||||
existing_match.stats = MatchStats(match_id=existing_match.id)
|
now = datetime.now(timezone.utc)
|
||||||
|
existing_match.stats = MatchStats(
|
||||||
|
match_id=existing_match.id,
|
||||||
|
home_xg=nm.home_xg,
|
||||||
|
away_xg=nm.away_xg,
|
||||||
|
home_shots=nm.home_shots,
|
||||||
|
away_shots=nm.away_shots,
|
||||||
|
home_shots_on_target=nm.home_shots_on_target,
|
||||||
|
away_shots_on_target=nm.away_shots_on_target,
|
||||||
|
home_corners=nm.home_corners,
|
||||||
|
away_corners=nm.away_corners,
|
||||||
|
home_possession=nm.home_possession,
|
||||||
|
home_yellow_cards=nm.home_yellow_cards,
|
||||||
|
away_yellow_cards=nm.away_yellow_cards,
|
||||||
|
home_red_cards=nm.home_red_cards,
|
||||||
|
away_red_cards=nm.away_red_cards,
|
||||||
|
source="bzzoiro",
|
||||||
|
source_record_id=str(raw.get("id", "")),
|
||||||
|
retrieved_at=now,
|
||||||
|
available_at=now,
|
||||||
|
)
|
||||||
db.add(existing_match.stats)
|
db.add(existing_match.stats)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
|
# P2-1: Silver 层血缘 — MatchStats 创建(更新路径)
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="bzzoiro",
|
||||||
|
source_record_id=str(raw.get("id", "")),
|
||||||
|
target_table="match_stats",
|
||||||
|
target_id=existing_match.id,
|
||||||
|
transform_name="silver_upsert",
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
if existing_match.stats is not None:
|
if existing_match.stats is not None:
|
||||||
for fld in ("home_xg", "away_xg", "home_shots", "away_shots",
|
for fld in ("home_xg", "away_xg", "home_shots", "away_shots",
|
||||||
"home_shots_on_target", "away_shots_on_target",
|
"home_shots_on_target", "away_shots_on_target",
|
||||||
@@ -284,4 +535,19 @@ class BzzoiroSource:
|
|||||||
result["leagues"][code] = league_r
|
result["leagues"][code] = league_r
|
||||||
result["total_inserted"] += league_r["inserted"]
|
result["total_inserted"] += league_r["inserted"]
|
||||||
result["total_updated"] += league_r["updated"]
|
result["total_updated"] += league_r["updated"]
|
||||||
|
|
||||||
|
# P2-2: 数据质量检查 — 行计数合理性(某联赛比赛数不应为 0)
|
||||||
|
total_records = league_r["inserted"] + league_r["updated"]
|
||||||
|
await _write_data_quality_check(
|
||||||
|
db,
|
||||||
|
check_name="bzzoiro_row_count",
|
||||||
|
entity_type="match",
|
||||||
|
entity_id=code,
|
||||||
|
expected_value=">=1",
|
||||||
|
actual_value=str(total_records),
|
||||||
|
passed=total_records > 0,
|
||||||
|
severity="critical" if total_records == 0 else "info",
|
||||||
|
detail=f"league={code} inserted={league_r['inserted']} updated={league_r['updated']}",
|
||||||
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|||||||
+346
-51
@@ -1,15 +1,25 @@
|
|||||||
"""伤停数据采集器(api-football / api-sports.io)。
|
"""伤停数据采集器(api-football / api-sports.io)。
|
||||||
|
|
||||||
采集伤停数据并入库(injuries 表),供 injuries agent 使用。
|
采集伤停数据并入库(injuries 表),供 injuries agent 使用。
|
||||||
|
|
||||||
|
改进(P0):
|
||||||
|
- 原子写缓存(tempfile + os.replace)
|
||||||
|
- TTL 分级:未来比赛 1h,当天 5min,历史 7day
|
||||||
|
- 文件锁防止并发写缓存冲突
|
||||||
|
- Bronze 层集成(RawEvent)
|
||||||
|
- 死信表集成(IngestFailure)
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import random
|
import random
|
||||||
|
import tempfile
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
import uuid
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -17,14 +27,147 @@ import httpx
|
|||||||
|
|
||||||
from src.core.config import settings
|
from src.core.config import settings
|
||||||
from src.core.http_client import get_client
|
from src.core.http_client import get_client
|
||||||
|
from src.db.models import DataQualityCheck
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
API_BASE = "https://v3.football.api-sports.io"
|
API_BASE = "https://v3.football.api-sports.io"
|
||||||
DEFAULT_HOST = "v3.football.api-sports.io"
|
DEFAULT_HOST = "v3.football.api-sports.io"
|
||||||
|
|
||||||
# 缓存目录
|
# P2-3: 缓存目录改用系统临时目录,避免源码树内写入
|
||||||
_CACHE_DIR = Path(__file__).resolve().parent.parent.parent / "data" / "injuries_cache"
|
_CACHE_DIR = Path(tempfile.gettempdir()) / "profeto_injuries"
|
||||||
|
|
||||||
|
# 模块级锁:防止并发写同一缓存文件
|
||||||
|
_cache_lock = asyncio.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _sanitize_cache_component(value: str | int | None) -> str:
|
||||||
|
"""消毒缓存键组件,防止路径遍历。
|
||||||
|
|
||||||
|
只允许 [A-Za-z0-9_.-],其余字符替换为 '_'。
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
s = str(value) if value is not None else "none"
|
||||||
|
return re.sub(r'[^A-Za-z0-9_.]', '_', s)[:60]
|
||||||
|
|
||||||
|
|
||||||
|
def _build_cache_key(prefix: str, *components: str | int | None) -> str:
|
||||||
|
"""构造安全的缓存文件名。"""
|
||||||
|
parts = [_sanitize_cache_component(c) for c in components]
|
||||||
|
return f"{prefix}_{'_'.join(parts)}.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _compute_ttl_hours(date_str: str | None, fixture_id: int | None, league_id: int | None) -> float:
|
||||||
|
"""根据查询参数计算缓存 TTL(小时)。
|
||||||
|
|
||||||
|
TTL 分级策略:
|
||||||
|
- 未来比赛(date > now): 1 小时(赛前伤停变化频繁)
|
||||||
|
- 当天比赛(date == today): 5 分钟(赛中实时更新)
|
||||||
|
- 历史比赛(date < now): 7 天(历史数据不变)
|
||||||
|
- 无日期参数: 1 小时(保守策略)
|
||||||
|
"""
|
||||||
|
if date_str is None:
|
||||||
|
# 无日期参数(按 fixture_id 或 league_id 查询),保守 TTL
|
||||||
|
return 1.0
|
||||||
|
try:
|
||||||
|
query_date = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
today = now.date()
|
||||||
|
if query_date.date() > today:
|
||||||
|
return 1.0 # 未来
|
||||||
|
elif query_date.date() == today:
|
||||||
|
return 5.0 / 60 # 当天:5 分钟
|
||||||
|
else:
|
||||||
|
return 168.0 # 历史:7 天
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return 1.0 # 解析失败,保守 TTL
|
||||||
|
|
||||||
|
|
||||||
|
def _write_cache_atomic(cache_file: Path, data: Any) -> None:
|
||||||
|
"""原子写缓存文件。
|
||||||
|
|
||||||
|
使用 tempfile + os.replace 实现原子写,防止读到写了一半的文件。
|
||||||
|
配合模块级 asyncio.Lock,杜绝并发写冲突。
|
||||||
|
"""
|
||||||
|
cache_file.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
# 在同一目录创建临时文件(保证 os.replace 是原子操作)
|
||||||
|
fd, tmp_path = tempfile.mkstemp(
|
||||||
|
dir=str(cache_file.parent),
|
||||||
|
prefix=f".{cache_file.name}.tmp_",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(data, f, ensure_ascii=False, default=str)
|
||||||
|
os.replace(tmp_path, cache_file)
|
||||||
|
except BaseException:
|
||||||
|
# 失败时清理临时文件
|
||||||
|
try:
|
||||||
|
os.unlink(tmp_path)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def _cleanup_expired_cache() -> int:
|
||||||
|
"""P2-4: 扫描缓存目录,删除过期的 .json 文件。
|
||||||
|
|
||||||
|
TTL 策略(保守取最大值,避免误删有效缓存):
|
||||||
|
- 最短 TTL 为 5 分钟(当天比赛),但清理阈值用 1 小时
|
||||||
|
- 超过 1 小时的缓存文件视为过期
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
删除的文件数。
|
||||||
|
"""
|
||||||
|
import glob as _glob
|
||||||
|
|
||||||
|
cache_dir = _CACHE_DIR
|
||||||
|
if not cache_dir.exists():
|
||||||
|
return 0
|
||||||
|
|
||||||
|
removed = 0
|
||||||
|
max_ttl_seconds = 3600 # 1 小时(保守阈值,最短实际 TTL 5 分钟)
|
||||||
|
now_ts = time.time()
|
||||||
|
for cache_file in _glob.glob(str(cache_dir / "*.json")):
|
||||||
|
try:
|
||||||
|
file_age = now_ts - os.path.getmtime(cache_file)
|
||||||
|
if file_age > max_ttl_seconds:
|
||||||
|
os.unlink(cache_file)
|
||||||
|
removed += 1
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if removed > 0:
|
||||||
|
logger.info("injuries cache cleanup: removed %d expired files", removed)
|
||||||
|
return removed
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_data_quality_check(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
check_name: str,
|
||||||
|
entity_type: str,
|
||||||
|
entity_id: str | None = None,
|
||||||
|
expected_value: str | None = None,
|
||||||
|
actual_value: str | None = None,
|
||||||
|
passed: bool,
|
||||||
|
severity: str = "warning",
|
||||||
|
detail: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""写入数据质量检查结果(DataQualityCheck 表)。"""
|
||||||
|
try:
|
||||||
|
check = DataQualityCheck(
|
||||||
|
check_name=check_name,
|
||||||
|
entity_type=entity_type,
|
||||||
|
entity_id=entity_id,
|
||||||
|
expected_value=expected_value,
|
||||||
|
actual_value=actual_value,
|
||||||
|
passed=passed,
|
||||||
|
severity=severity,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
db.add(check)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("injuries data_quality_check write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = None, league_id: int | None = None) -> list[dict]:
|
async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = None, league_id: int | None = None) -> list[dict]:
|
||||||
@@ -42,20 +185,25 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
|
|||||||
if not api_key:
|
if not api_key:
|
||||||
raise RuntimeError("API_FOOTBALL_KEY 未设置")
|
raise RuntimeError("API_FOOTBALL_KEY 未设置")
|
||||||
|
|
||||||
|
# P2-4: 清理过期缓存文件
|
||||||
|
_cleanup_expired_cache()
|
||||||
|
|
||||||
cache_dir = _CACHE_DIR
|
cache_dir = _CACHE_DIR
|
||||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
# 缓存命中 (7 天内有效)
|
# TTL 分级缓存
|
||||||
cache_key = f"injuries_{date}_{fixture_id}_{league_id}.json"
|
ttl_hours = _compute_ttl_hours(date, fixture_id, league_id)
|
||||||
|
# P0-fix: 消毒缓存键,防止路径遍历(date 参数可能含 '../' 等)
|
||||||
|
cache_key = _build_cache_key("injuries", date, fixture_id, league_id)
|
||||||
cache_file = cache_dir / cache_key
|
cache_file = cache_dir / cache_key
|
||||||
if cache_file.exists():
|
if cache_file.exists():
|
||||||
age_hours = (time.time() - cache_file.stat().st_mtime) / 3600
|
age_hours = (time.time() - cache_file.stat().st_mtime) / 3600
|
||||||
if age_hours < 168: # 7 天
|
if age_hours < ttl_hours:
|
||||||
logger.debug("injuries cache hit: %s (%.1fh old)", cache_key, age_hours)
|
logger.debug("injuries cache hit: %s (%.1fh old, ttl=%.2fh)", cache_key, age_hours, ttl_hours)
|
||||||
with open(cache_file, encoding="utf-8") as f:
|
with open(cache_file, encoding="utf-8") as f:
|
||||||
return json.load(f)
|
return json.load(f)
|
||||||
else:
|
else:
|
||||||
logger.debug("injuries cache expired: %s (%.1fh old)", cache_key, age_hours)
|
logger.debug("injuries cache expired: %s (%.1fh old, ttl=%.2fh)", cache_key, age_hours, ttl_hours)
|
||||||
|
|
||||||
headers = {
|
headers = {
|
||||||
"x-apisports-key": api_key,
|
"x-apisports-key": api_key,
|
||||||
@@ -92,9 +240,9 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
|
|||||||
data = resp.json()
|
data = resp.json()
|
||||||
injuries = data.get("response", [])
|
injuries = data.get("response", [])
|
||||||
|
|
||||||
# 写缓存
|
# 原子写缓存 + 文件锁(防止并发写冲突)
|
||||||
with open(cache_file, "w", encoding="utf-8") as f:
|
async with _cache_lock:
|
||||||
json.dump(injuries, ensure_ascii=False, default=str, fp=f)
|
_write_cache_atomic(cache_file, injuries)
|
||||||
|
|
||||||
return injuries
|
return injuries
|
||||||
|
|
||||||
@@ -103,27 +251,74 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
|||||||
"""采集伤停数据并入库(injuries 表)。
|
"""采集伤停数据并入库(injuries 表)。
|
||||||
|
|
||||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||||
|
|
||||||
|
P1-4: 批量幂等检查,避免逐条查询的竞态条件(并发采集时 IntegrityError)。
|
||||||
|
P1 Bronze: 采集成功后先存 RawEvent,再规范化。
|
||||||
|
P1 死信: 采集/规范化失败时写 IngestFailure。
|
||||||
"""
|
"""
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from src.data.team_names import normalize as normalize_name
|
from src.data.team_names import normalize as normalize_name
|
||||||
from src.db.models import Injury, Team
|
from src.db.models import DataQualityCheck, Injury, IngestFailure, RawEvent, Team
|
||||||
|
|
||||||
result = {"count": 0, "inserted": 0, "errors": []}
|
result = {"count": 0, "inserted": 0, "errors": []}
|
||||||
|
batch_id = uuid.uuid4().hex[:16]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
raw_injuries = await fetch_injuries(date=date)
|
raw_injuries = await fetch_injuries(date=date)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("injuries fetch failed")
|
logger.exception("injuries fetch failed")
|
||||||
result["errors"].append(f"fetch failed: {e}")
|
result["errors"].append(f"fetch failed: {e}")
|
||||||
|
# 写死信表(IngestFailure)
|
||||||
|
try:
|
||||||
|
failure = IngestFailure(
|
||||||
|
source_system="api-football",
|
||||||
|
entity_type="injury",
|
||||||
|
source_record_id=f"fetch_{batch_id}",
|
||||||
|
error_type="fetch_error",
|
||||||
|
error_detail=str(e)[:2000],
|
||||||
|
raw_payload={"date": date},
|
||||||
|
status="pending",
|
||||||
|
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
db.add(failure)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("failed to write fetch error to ingest_failures", exc_info=True)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
result["count"] = len(raw_injuries)
|
result["count"] = len(raw_injuries)
|
||||||
|
|
||||||
|
# === Bronze 层:写 RawEvent ===
|
||||||
|
for raw in raw_injuries:
|
||||||
|
try:
|
||||||
|
player = raw.get("player", {}) or {}
|
||||||
|
fixture = raw.get("fixture", {}) or {}
|
||||||
|
source_record_id = f"inj_{player.get('id')}_{fixture.get('id')}"
|
||||||
|
bronze = RawEvent(
|
||||||
|
source_system="api-football",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
raw_payload=raw,
|
||||||
|
ingest_batch_id=batch_id,
|
||||||
|
)
|
||||||
|
db.add(bronze)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("raw_event write failed for injury: %s", e)
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("raw_events flush failed: %s", e)
|
||||||
|
|
||||||
# 预加载所有球队(用于按名匹配)
|
# 预加载所有球队(用于按名匹配)
|
||||||
teams = (await db.execute(select(Team))).scalars().all()
|
teams = (await db.execute(select(Team))).scalars().all()
|
||||||
team_by_name = {t.name: t.id for t in teams}
|
team_by_name = {t.name: t.id for t in teams}
|
||||||
|
|
||||||
|
# P1-4: 收集所有待插入记录的键,批量查询已存在的记录
|
||||||
|
# 避免逐条查询 + 插入的竞态条件(两个并发请求同时通过检查 → IntegrityError)
|
||||||
|
pending_records: list[dict] = []
|
||||||
|
parse_failures: list[dict] = [] # 规范化失败的原始数据,用于写死信表
|
||||||
for raw in raw_injuries:
|
for raw in raw_injuries:
|
||||||
try:
|
try:
|
||||||
player = raw.get("player", {}) or {}
|
player = raw.get("player", {}) or {}
|
||||||
@@ -144,43 +339,145 @@ async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
|||||||
except (ValueError, AttributeError):
|
except (ValueError, AttributeError):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# 强制 int 转换,API 可能返回字符串
|
||||||
player_id = player.get("id")
|
player_id = player.get("id")
|
||||||
|
try:
|
||||||
|
player_id = int(player_id) if player_id is not None else None
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
player_id = None
|
||||||
fixture_id = fixture.get("id")
|
fixture_id = fixture.get("id")
|
||||||
|
try:
|
||||||
|
fixture_id = int(fixture_id) if fixture_id is not None else None
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
fixture_id = None
|
||||||
|
|
||||||
# 幂等: 已存在则跳过
|
pending_records.append({
|
||||||
existing = (
|
"player_id": player_id,
|
||||||
await db.execute(
|
"player_name": player_name,
|
||||||
select(Injury).where(
|
"team_id": team_id,
|
||||||
Injury.player_id == player_id,
|
"fixture_id": fixture_id,
|
||||||
Injury.fixture_id == fixture_id,
|
"league_id": (raw.get("league") or {}).get("id"),
|
||||||
Injury.injury_type == player.get("type"),
|
"injury_type": player.get("type"),
|
||||||
)
|
"reason": player.get("reason"),
|
||||||
)
|
"injury_date": injury_date,
|
||||||
).scalar_one_or_none()
|
})
|
||||||
|
|
||||||
if existing is not None:
|
|
||||||
continue
|
|
||||||
|
|
||||||
injury = Injury(
|
|
||||||
player_id=player_id,
|
|
||||||
player_name=player_name,
|
|
||||||
team_id=team_id,
|
|
||||||
fixture_id=fixture_id,
|
|
||||||
league_id=(raw.get("league") or {}).get("id"),
|
|
||||||
injury_type=player.get("type"),
|
|
||||||
reason=player.get("reason"),
|
|
||||||
injury_date=injury_date,
|
|
||||||
)
|
|
||||||
db.add(injury)
|
|
||||||
result["inserted"] += 1
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result["errors"].append(f"parse error: {e}")
|
result["errors"].append(f"parse error: {e}")
|
||||||
|
parse_failures.append({"raw": raw, "error": str(e)})
|
||||||
|
|
||||||
|
# 写规范化失败到死信表
|
||||||
|
for fail in parse_failures:
|
||||||
|
try:
|
||||||
|
raw = fail["raw"]
|
||||||
|
player = raw.get("player", {}) or {}
|
||||||
|
failure = IngestFailure(
|
||||||
|
source_system="api-football",
|
||||||
|
entity_type="injury",
|
||||||
|
source_record_id=f"inj_{player.get('id', 'unknown')}",
|
||||||
|
error_type="normalize_error",
|
||||||
|
error_detail=fail["error"][:2000],
|
||||||
|
raw_payload=raw,
|
||||||
|
status="pending",
|
||||||
|
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
db.add(failure)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("failed to write parse error to ingest_failures", exc_info=True)
|
||||||
|
if parse_failures:
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("ingest_failures flush failed", exc_info=True)
|
||||||
|
|
||||||
|
# P1-4: 批量查询已存在的记录(1 次 DB 往返)
|
||||||
|
existing_keys: set[tuple] = set()
|
||||||
|
if pending_records:
|
||||||
|
# 构造查询条件:所有 (player_id, fixture_id, injury_type) 组合
|
||||||
|
# 使用 OR 条件批量查询
|
||||||
|
conditions = []
|
||||||
|
for rec in pending_records:
|
||||||
|
conditions.append(
|
||||||
|
(Injury.player_id == rec["player_id"])
|
||||||
|
& (Injury.fixture_id == rec["fixture_id"])
|
||||||
|
& (Injury.injury_type == rec["injury_type"])
|
||||||
|
)
|
||||||
|
if conditions:
|
||||||
|
from sqlalchemy import or_
|
||||||
|
stmt = select(Injury.player_id, Injury.fixture_id, Injury.injury_type).where(or_(*conditions))
|
||||||
|
rows = (await db.execute(stmt)).all()
|
||||||
|
existing_keys = {(r[0], r[1], r[2]) for r in rows}
|
||||||
|
|
||||||
|
# P1-4: 批量插入(跳过已存在的)
|
||||||
|
for rec in pending_records:
|
||||||
|
key = (rec["player_id"], rec["fixture_id"], rec["injury_type"])
|
||||||
|
if key in existing_keys:
|
||||||
|
continue
|
||||||
|
|
||||||
|
injury = Injury(**rec)
|
||||||
|
db.add(injury)
|
||||||
|
result["inserted"] += 1
|
||||||
|
|
||||||
|
# 每 50 条 flush 一次,减少内存压力,同时捕获 IntegrityError
|
||||||
|
if result["inserted"] % 50 == 0:
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
except IntegrityError:
|
||||||
|
# P1-4: 并发采集时可能仍有竞态,回退到逐条插入
|
||||||
|
await db.rollback()
|
||||||
|
logger.warning("injuries batch IntegrityError, falling back to per-record insert")
|
||||||
|
return await _ingest_injuries_fallback(db, pending_records, result)
|
||||||
|
|
||||||
|
# 最终 flush
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
except IntegrityError:
|
||||||
|
await db.rollback()
|
||||||
|
logger.warning("injuries final flush IntegrityError, falling back to per-record insert")
|
||||||
|
return await _ingest_injuries_fallback(db, pending_records, result)
|
||||||
|
|
||||||
|
# P2-2: 数据质量检查 — 行计数 / 新增比例
|
||||||
|
total = result["count"]
|
||||||
|
inserted = result["inserted"]
|
||||||
|
# 某日期伤停数不应为 0(除非历史日期)
|
||||||
|
await _write_data_quality_check(
|
||||||
|
db,
|
||||||
|
check_name="injuries_row_count",
|
||||||
|
entity_type="injury",
|
||||||
|
entity_id=date or "unknown",
|
||||||
|
expected_value=">=0",
|
||||||
|
actual_value=str(total),
|
||||||
|
passed=True,
|
||||||
|
severity="info",
|
||||||
|
detail=f"date={date} fetched={total} inserted={inserted}",
|
||||||
|
)
|
||||||
|
|
||||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||||
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
|
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def _ingest_injuries_fallback(db, pending_records: list[dict], result: dict) -> dict:
|
||||||
|
"""P1-4: 逐条插入回退,捕获每条 IntegrityError 避免整批回滚。"""
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from src.db.models import Injury
|
||||||
|
|
||||||
|
inserted = 0
|
||||||
|
for rec in pending_records:
|
||||||
|
injury = Injury(**rec)
|
||||||
|
db.add(injury)
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
inserted += 1
|
||||||
|
except IntegrityError:
|
||||||
|
await db.rollback()
|
||||||
|
# 已存在或其他冲突,跳过
|
||||||
|
continue
|
||||||
|
|
||||||
|
result["inserted"] = inserted
|
||||||
|
logger.info("injuries fallback: inserted %d records", inserted)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> list[Injury]:
|
async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> list[Injury]:
|
||||||
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。
|
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。
|
||||||
|
|
||||||
@@ -188,31 +485,29 @@ async def get_injuries_for_match(db, team_id: int, match_date, as_of=None) -> li
|
|||||||
db: 数据库 session
|
db: 数据库 session
|
||||||
team_id: 球队 ID
|
team_id: 球队 ID
|
||||||
match_date: 比赛日期
|
match_date: 比赛日期
|
||||||
as_of: 截止时间(cutoff)。只返回 retrieved_at <= as_of 的记录。
|
as_of: 数据截止时间(用于回测防泄漏)
|
||||||
用于回测时防止"未来采集的数据"泄漏到历史预测。
|
|
||||||
必须保持 timezone-aware datetime,不会截断为 date。
|
|
||||||
"""
|
|
||||||
from sqlalchemy import or_, select
|
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
伤停记录列表
|
||||||
|
"""
|
||||||
|
from sqlalchemy import select
|
||||||
from src.db.models import Injury
|
from src.db.models import Injury
|
||||||
|
|
||||||
# 只处理 match_date:去掉时间部分,仅比较日期
|
if hasattr(match_date, "date") and callable(match_date.date):
|
||||||
if hasattr(match_date, "date"):
|
|
||||||
match_date = match_date.date()
|
match_date = match_date.date()
|
||||||
|
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Injury)
|
select(Injury)
|
||||||
.where(Injury.team_id == team_id)
|
.where(Injury.team_id == team_id)
|
||||||
.where(Injury.injury_date <= match_date)
|
.where(Injury.injury_date <= match_date)
|
||||||
.where(or_(Injury.return_date.is_(None), Injury.return_date >= match_date))
|
.where(
|
||||||
|
(Injury.return_date.is_(None)) | (Injury.return_date >= match_date)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 回测防泄漏: 只使用 as_of 时间点之前已采集的数据
|
|
||||||
# 注意: as_of 保持 datetime,不截断为 date,避免错误排除同日合法数据
|
|
||||||
if as_of is not None:
|
if as_of is not None:
|
||||||
stmt = stmt.where(Injury.retrieved_at.is_not(None))
|
if hasattr(as_of, "date") and callable(as_of.date):
|
||||||
|
as_of = as_of.date()
|
||||||
stmt = stmt.where(Injury.retrieved_at <= as_of)
|
stmt = stmt.where(Injury.retrieved_at <= as_of)
|
||||||
|
|
||||||
stmt = stmt.order_by(Injury.injury_date.desc())
|
|
||||||
result = await db.execute(stmt)
|
result = await db.execute(stmt)
|
||||||
return list(result.scalars().all())
|
return list(result.scalars().all())
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""数据规范化:任意数据源原始记录 → NormalizedMatch。
|
"""数据规范化:任意数据源原始记录 → NormalizedMatch。
|
||||||
|
|
||||||
迁移自旧项目 app/data/normalize.py,简化:
|
迁移自旧项目 app/data/normalize.py,简化:
|
||||||
- 去掉 XGBackfill 双轨(不再需要独立回填)
|
- 去掉 XGBackoff 双轨(不再需要独立回填)
|
||||||
- 去掉 PIT 时间契约(无训练集要防泄漏)
|
- 去掉 PIT 时间契约(无训练集要防泄漏)
|
||||||
- 保留核心清洗契约(队名归一、日期解析、数值范围)
|
- 保留核心清洗契约(队名归一、日期解析、数值范围)
|
||||||
"""
|
"""
|
||||||
@@ -90,7 +90,10 @@ def derive_season_label(date: datetime) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _parse_date(value) -> datetime | None:
|
def _parse_date(value) -> datetime | None:
|
||||||
"""日期解析 → UTC datetime(带 tzinfo)。"""
|
"""日期解析 → UTC datetime(带 tzinfo)。
|
||||||
|
|
||||||
|
P2-2: 解析失败时记录 warning,避免静默丢数据而无感知。
|
||||||
|
"""
|
||||||
if value in (None, ""):
|
if value in (None, ""):
|
||||||
return None
|
return None
|
||||||
if isinstance(value, (int, float)):
|
if isinstance(value, (int, float)):
|
||||||
@@ -111,6 +114,8 @@ def _parse_date(value) -> datetime | None:
|
|||||||
return datetime.strptime(s[:19], fmt).replace(tzinfo=timezone.utc)
|
return datetime.strptime(s[:19], fmt).replace(tzinfo=timezone.utc)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
|
# P2-2 修复: 记录被丢弃的原始值,便于排查数据源格式变更
|
||||||
|
logger.warning("_parse_date failed, dropping record: %r", value)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
@@ -204,6 +209,6 @@ def normalize_understat(raw: dict, league_type: str) -> NormalizedMatch | None:
|
|||||||
away_team=away,
|
away_team=away,
|
||||||
match_status="finished",
|
match_status="finished",
|
||||||
season_label=derive_season_label(dt),
|
season_label=derive_season_label(dt),
|
||||||
home_xg=_to_float(home_xg),
|
home_xg=home_xg,
|
||||||
away_xg=_to_float(away_xg),
|
away_xg=away_xg,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,98 @@
|
|||||||
|
"""令牌桶限流器。
|
||||||
|
|
||||||
|
替代 bzzoiro 中固定的 REQUEST_INTERVAL sleep,提供更精细的速率控制。
|
||||||
|
支持突发流量和平滑限流。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TokenBucket:
|
||||||
|
"""异步令牌桶限流器。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
limiter = TokenBucket(rate=5.0, capacity=10)
|
||||||
|
await limiter.acquire() # 等待直到有可用令牌
|
||||||
|
|
||||||
|
Args:
|
||||||
|
rate: 每秒补充的令牌数
|
||||||
|
capacity: 桶容量(允许的最大突发量)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, rate: float, capacity: int | None = None) -> None:
|
||||||
|
if rate <= 0:
|
||||||
|
raise ValueError(f"rate must be positive, got {rate}")
|
||||||
|
self._rate = rate
|
||||||
|
self._capacity = capacity or max(1, int(rate * 2))
|
||||||
|
self._tokens: float = self._capacity
|
||||||
|
self._last_refill = time.monotonic()
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
|
async def acquire(self, tokens: int = 1) -> None:
|
||||||
|
"""获取指定数量的令牌,不足时等待。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
tokens: 需要获取的令牌数
|
||||||
|
"""
|
||||||
|
if tokens <= 0:
|
||||||
|
return
|
||||||
|
if tokens > self._capacity:
|
||||||
|
raise ValueError(f"requested {tokens} exceeds capacity {self._capacity}")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
async with self._lock:
|
||||||
|
self._refill()
|
||||||
|
if self._tokens >= tokens:
|
||||||
|
self._tokens -= tokens
|
||||||
|
return
|
||||||
|
|
||||||
|
# 计算需要等待的时间
|
||||||
|
wait_time = (tokens - self._tokens) / self._rate
|
||||||
|
logger.debug("token bucket: waiting %.2fs for %d tokens", wait_time, tokens)
|
||||||
|
await asyncio.sleep(wait_time)
|
||||||
|
|
||||||
|
def _refill(self) -> None:
|
||||||
|
"""补充令牌(基于经过的时间)。"""
|
||||||
|
now = time.monotonic()
|
||||||
|
elapsed = now - self._last_refill
|
||||||
|
if elapsed > 0:
|
||||||
|
self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
|
||||||
|
self._last_refill = now
|
||||||
|
|
||||||
|
@property
|
||||||
|
def tokens(self) -> float:
|
||||||
|
"""当前可用令牌数(近似)。"""
|
||||||
|
self._refill()
|
||||||
|
return self._tokens
|
||||||
|
|
||||||
|
|
||||||
|
class RateLimitedClient:
|
||||||
|
"""HTTP 客户端限流包装器。
|
||||||
|
|
||||||
|
在 httpx 客户端之上添加令牌桶限流,透明地控制请求速率。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
client = get_client()
|
||||||
|
limiter = TokenBucket(rate=5.0)
|
||||||
|
wrapper = RateLimitedClient(client, limiter)
|
||||||
|
resp = await wrapper.get(url)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, client, limiter: TokenBucket) -> None:
|
||||||
|
self._client = client
|
||||||
|
self._limiter = limiter
|
||||||
|
|
||||||
|
async def get(self, url: str, **kwargs):
|
||||||
|
"""限流的 GET 请求。"""
|
||||||
|
await self._limiter.acquire()
|
||||||
|
return await self._client.get(url, **kwargs)
|
||||||
|
|
||||||
|
async def post(self, url: str, **kwargs):
|
||||||
|
"""限流的 POST 请求。"""
|
||||||
|
await self._limiter.acquire()
|
||||||
|
return await self._client.post(url, **kwargs)
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
"""P2-5: IngestFailure 重试 Worker。
|
||||||
|
|
||||||
|
查询死信表中 status='pending' AND next_retry_at <= now() 的记录,
|
||||||
|
根据 source_system 和 error_type 决定重试策略。
|
||||||
|
作为 FastAPI startup 事件注册,在应用启动时执行一次清理。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from src.db.base import AsyncSessionLocal
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 最大重试次数上限
|
||||||
|
_MAX_RETRIES = 5
|
||||||
|
|
||||||
|
|
||||||
|
async def _retry_ingest_failures(db) -> dict:
|
||||||
|
"""重试待处理的采集失败记录。
|
||||||
|
|
||||||
|
查询条件: status='pending' AND next_retry_at <= now()
|
||||||
|
重试策略:
|
||||||
|
- retry_count >= MAX_RETRIES → 标记 abandoned
|
||||||
|
- error_type=fetch_error → 退避重试(更新 next_retry_at)
|
||||||
|
- error_type=normalize_error → 规范化失败通常是数据问题,退避重试
|
||||||
|
- error_type=db_error → 数据库问题,退避重试
|
||||||
|
|
||||||
|
Args:
|
||||||
|
db: SQLAlchemy async session
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
统计 dict: {"retried": int, "abandoned": int, "errors": list[str]}
|
||||||
|
"""
|
||||||
|
result = {"retried": 0, "abandoned": 0, "errors": []}
|
||||||
|
|
||||||
|
from src.db.models import IngestFailure
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
# 查询待重试记录
|
||||||
|
stmt = (
|
||||||
|
select(IngestFailure)
|
||||||
|
.where(IngestFailure.status == "pending")
|
||||||
|
.where(IngestFailure.next_retry_at <= now)
|
||||||
|
.order_by(IngestFailure.next_retry_at)
|
||||||
|
.limit(50) # 每批最多处理 50 条,避免长时间持有事务
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
rows = (await db.execute(stmt)).scalars().all()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("retry_worker: query failed: %s", e)
|
||||||
|
result["errors"].append(f"query failed: {e}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return result
|
||||||
|
|
||||||
|
logger.info("retry_worker: found %d pending failures to retry", len(rows))
|
||||||
|
|
||||||
|
for failure in rows:
|
||||||
|
try:
|
||||||
|
# 超过最大重试次数 → 放弃
|
||||||
|
if failure.retry_count >= _MAX_RETRIES:
|
||||||
|
failure.status = "abandoned"
|
||||||
|
failure.resolved_at = now
|
||||||
|
await db.flush()
|
||||||
|
result["abandoned"] += 1
|
||||||
|
logger.info(
|
||||||
|
"retry_worker: abandoned %s/%s after %d retries",
|
||||||
|
failure.source_system, failure.source_record_id, failure.retry_count,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 退避计算: 5min * 2^retry_count,最大 24 小时
|
||||||
|
backoff_minutes = min(5 * (2 ** failure.retry_count), 1440)
|
||||||
|
next_retry = now + timedelta(minutes=backoff_minutes)
|
||||||
|
|
||||||
|
# 更新重试状态
|
||||||
|
failure.retry_count += 1
|
||||||
|
failure.next_retry_at = next_retry
|
||||||
|
failure.status = "retrying"
|
||||||
|
await db.flush()
|
||||||
|
result["retried"] += 1
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"retry_worker: scheduled retry %d/%d for %s/%s (next: %s)",
|
||||||
|
failure.retry_count, _MAX_RETRIES,
|
||||||
|
failure.source_system, failure.source_record_id,
|
||||||
|
next_retry.isoformat(),
|
||||||
|
)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
await db.rollback()
|
||||||
|
error_msg = f"retry {failure.source_system}/{failure.source_record_id}: {e}"
|
||||||
|
result["errors"].append(error_msg)
|
||||||
|
logger.warning("retry_worker: %s", error_msg)
|
||||||
|
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def run_retry_worker() -> None:
|
||||||
|
"""启动入口: 获取 DB session 并执行重试逻辑。
|
||||||
|
|
||||||
|
设计为幂等操作 — 多次运行不会产生副作用(受 next_retry_at 约束)。
|
||||||
|
"""
|
||||||
|
logger.info("retry_worker: starting ingest failure retry sweep")
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with AsyncSessionLocal() as db:
|
||||||
|
try:
|
||||||
|
stats = await _retry_ingest_failures(db)
|
||||||
|
await db.commit()
|
||||||
|
logger.info(
|
||||||
|
"retry_worker: sweep complete — retried=%d abandoned=%d errors=%d",
|
||||||
|
stats["retried"], stats["abandoned"], len(stats["errors"]),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
await db.rollback()
|
||||||
|
logger.warning("retry_worker: session failed: %s", e)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("retry_worker: failed to get DB session: %s", e)
|
||||||
+270
-21
@@ -2,6 +2,12 @@
|
|||||||
|
|
||||||
迁移自旧项目 app/data/sources/understat.py,改成 async。
|
迁移自旧项目 app/data/sources/understat.py,改成 async。
|
||||||
使用 Repository 模式进行数据访问,不直接控制事务。
|
使用 Repository 模式进行数据访问,不直接控制事务。
|
||||||
|
|
||||||
|
改进(P1):
|
||||||
|
- Bronze 层集成:采集后先存 RawEvent,再规范化
|
||||||
|
- 死信表集成:采集/规范化失败写 IngestFailure
|
||||||
|
- xG 覆盖更新:当 understat 数据更新时覆盖旧值(全量覆盖模式)
|
||||||
|
- xG 追踪字段:xg_source / xg_updated_at / xg_source_record_id
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -10,15 +16,17 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
from datetime import datetime, timezone
|
import uuid
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from src.core.http_client import get_client
|
from src.core.http_client import get_client
|
||||||
from src.data.config import FDCO_TO_UNDERSTAT, LEAGUE_NAMES
|
from src.data.config import FDCO_TO_UNDERSTAT, LEAGUE_NAMES
|
||||||
from src.data.normalize import normalize_understat
|
from src.data.normalize import normalize_understat
|
||||||
from src.data.sources import register
|
from src.data.sources import register
|
||||||
from src.db.models import League, Match, MatchStats, Team
|
from src.db.models import DataLineage, DataQualityCheck, IngestFailure, League, Match, MatchStats, RawEvent, Team
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -46,6 +54,7 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
|
|||||||
"Referer": f"https://understat.com/league/{understat_league}/{season}",
|
"Referer": f"https://understat.com/league/{understat_league}/{season}",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# 重试:网络错误 / 5xx / 429
|
# 重试:网络错误 / 5xx / 429
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for attempt in range(3):
|
for attempt in range(3):
|
||||||
@@ -57,12 +66,10 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
last_exc = e
|
last_exc = e
|
||||||
if attempt == 2:
|
if attempt == 2:
|
||||||
raise
|
raise RuntimeError(f"understat fetch failed: {e}") from e
|
||||||
delay = min(2 ** attempt, 8) + random.uniform(0, 1)
|
delay = min(2 ** attempt, 8) + random.uniform(0, 1)
|
||||||
logger.warning("understat fetch failed, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
logger.warning("understat fetch failed, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||||
await asyncio.sleep(delay)
|
await asyncio.sleep(delay)
|
||||||
else:
|
|
||||||
raise RuntimeError(f"understat fetch failed: {last_exc}")
|
|
||||||
|
|
||||||
# understat 返回 JS 对象,需要提取 JSON
|
# understat 返回 JS 对象,需要提取 JSON
|
||||||
text = resp.text
|
text = resp.text
|
||||||
@@ -76,6 +83,148 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]:
|
||||||
|
"""比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。
|
||||||
|
|
||||||
|
统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类
|
||||||
|
隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配,
|
||||||
|
导致所有比赛被判为不存在而重复插入。
|
||||||
|
"""
|
||||||
|
if hasattr(match_date, "date") and callable(match_date.date):
|
||||||
|
match_date = match_date.date()
|
||||||
|
return (home_team_id, away_team_id, match_date.isoformat() if match_date is not None else "")
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_understat_bronze(db, raw_matches: list[dict], batch_id: str) -> None:
|
||||||
|
"""将 understat 原始数据写入 Bronze 层(RawEvent 表)。
|
||||||
|
|
||||||
|
P2-1: 写入血缘记录,追踪 Bronze 层摄取过程。
|
||||||
|
"""
|
||||||
|
for raw in raw_matches:
|
||||||
|
try:
|
||||||
|
source_record_id = str(raw.get("id", ""))
|
||||||
|
if not source_record_id:
|
||||||
|
# P1-fix: 不再静默跳过,记录到死信表
|
||||||
|
logger.warning("understat raw match missing id: %s", str(raw)[:200])
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "understat", "match", "",
|
||||||
|
"normalize_error", "raw match missing id", raw,
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
bronze = RawEvent(
|
||||||
|
source_system="understat",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
raw_payload=raw,
|
||||||
|
ingest_batch_id=batch_id,
|
||||||
|
)
|
||||||
|
db.add(bronze)
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("understat raw_event write failed: %s", e)
|
||||||
|
try:
|
||||||
|
await db.flush()
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("understat raw_events flush failed: %s", e)
|
||||||
|
|
||||||
|
# P2-1: 写入 Bronze 层血缘记录
|
||||||
|
for raw in raw_matches:
|
||||||
|
source_record_id = str(raw.get("id", ""))
|
||||||
|
if not source_record_id:
|
||||||
|
continue
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="understat",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
target_table="raw_events",
|
||||||
|
transform_name="bronze_ingest",
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_ingest_failure(
|
||||||
|
db,
|
||||||
|
source_system: str,
|
||||||
|
entity_type: str,
|
||||||
|
source_record_id: str,
|
||||||
|
error_type: str,
|
||||||
|
error_detail: str,
|
||||||
|
raw_payload: dict | None,
|
||||||
|
) -> None:
|
||||||
|
"""写入采集失败到死信表(IngestFailure)。"""
|
||||||
|
try:
|
||||||
|
failure = IngestFailure(
|
||||||
|
source_system=source_system,
|
||||||
|
entity_type=entity_type,
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
error_type=error_type,
|
||||||
|
error_detail=error_detail[:2000] if error_detail else None,
|
||||||
|
raw_payload=raw_payload,
|
||||||
|
status="pending",
|
||||||
|
next_retry_at=datetime.now(timezone.utc) + timedelta(minutes=5),
|
||||||
|
)
|
||||||
|
db.add(failure)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("understat ingest_failure write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_data_lineage(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
source_system: str,
|
||||||
|
source_record_id: str,
|
||||||
|
target_table: str,
|
||||||
|
target_id: int | None = None,
|
||||||
|
transform_name: str,
|
||||||
|
transform_detail: str | None = None,
|
||||||
|
batch_id: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""写入数据血缘记录(DataLineage 表)。"""
|
||||||
|
try:
|
||||||
|
lineage = DataLineage(
|
||||||
|
source_system=source_system,
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
target_table=target_table,
|
||||||
|
target_id=target_id,
|
||||||
|
transform_name=transform_name,
|
||||||
|
transform_detail=transform_detail,
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
|
db.add(lineage)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("understat data_lineage write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_data_quality_check(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
check_name: str,
|
||||||
|
entity_type: str,
|
||||||
|
entity_id: str | None = None,
|
||||||
|
expected_value: str | None = None,
|
||||||
|
actual_value: str | None = None,
|
||||||
|
passed: bool,
|
||||||
|
severity: str = "warning",
|
||||||
|
detail: str | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""写入数据质量检查结果(DataQualityCheck 表)。"""
|
||||||
|
try:
|
||||||
|
check = DataQualityCheck(
|
||||||
|
check_name=check_name,
|
||||||
|
entity_type=entity_type,
|
||||||
|
entity_id=entity_id,
|
||||||
|
expected_value=expected_value,
|
||||||
|
actual_value=actual_value,
|
||||||
|
passed=passed,
|
||||||
|
severity=severity,
|
||||||
|
detail=detail,
|
||||||
|
)
|
||||||
|
db.add(check)
|
||||||
|
await db.flush()
|
||||||
|
except Exception:
|
||||||
|
logger.warning("understat data_quality_check write failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
@register
|
@register
|
||||||
class UnderstatSource:
|
class UnderstatSource:
|
||||||
"""understat xG 数据源(实现 DataSource 协议)。"""
|
"""understat xG 数据源(实现 DataSource 协议)。"""
|
||||||
@@ -83,25 +232,36 @@ class UnderstatSource:
|
|||||||
name = "understat"
|
name = "understat"
|
||||||
|
|
||||||
async def ingest(self, db, *, league: str, season: int) -> dict:
|
async def ingest(self, db, *, league: str, season: int) -> dict:
|
||||||
"""采集 understat xG → 回填到现有 Match。只回填 xG 字段,不创建新 Match。
|
"""采集 understat xG → 回填到现有 Match。
|
||||||
|
|
||||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||||
|
|
||||||
|
P1-3: 批量查询优化,将单赛季 380 场 × 3 次 DB 往返降为 3 次查询。
|
||||||
|
P1-4: xG 覆盖更新模式,当 understat 数据更新时覆盖旧值。
|
||||||
"""
|
"""
|
||||||
from src.db.repositories import LeagueRepository, MatchRepository, TeamRepository
|
from src.db.repositories import LeagueRepository, TeamRepository
|
||||||
|
|
||||||
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
||||||
|
batch_id = uuid.uuid4().hex[:16]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
raw_matches = await fetch_understat(league, season)
|
raw_matches = await fetch_understat(league, season)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("understat fetch failed for %s %s", league, season)
|
logger.exception("understat fetch failed for %s %s", league, season)
|
||||||
result["errors"].append(f"fetch failed: {e}")
|
result["errors"].append(f"fetch failed: {e}")
|
||||||
|
# 写死信表
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "understat", "match", f"fetch_{league}_{season}_{batch_id}",
|
||||||
|
"fetch_error", str(e), {"league": league, "season": season},
|
||||||
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
# === Bronze 层:先存 RawEvent ===
|
||||||
|
await _write_understat_bronze(db, raw_matches, batch_id)
|
||||||
|
|
||||||
# 使用 Repository
|
# 使用 Repository
|
||||||
league_repo = LeagueRepository(db)
|
league_repo = LeagueRepository(db)
|
||||||
team_repo = TeamRepository(db)
|
team_repo = TeamRepository(db)
|
||||||
match_repo = MatchRepository(db)
|
|
||||||
|
|
||||||
# 查联赛
|
# 查联赛
|
||||||
league_obj = await league_repo.get_by_code(league)
|
league_obj = await league_repo.get_by_code(league)
|
||||||
@@ -109,6 +269,9 @@ class UnderstatSource:
|
|||||||
result["errors"].append(f"league {league} not found in DB")
|
result["errors"].append(f"league {league} not found in DB")
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
# === 批量优化: 一次规范化,收集球队名和日期 ===
|
||||||
|
normalized_matches: list = []
|
||||||
|
all_team_names: set[str] = set()
|
||||||
for raw in raw_matches:
|
for raw in raw_matches:
|
||||||
if not raw.get("isResult"):
|
if not raw.get("isResult"):
|
||||||
continue
|
continue
|
||||||
@@ -119,40 +282,126 @@ class UnderstatSource:
|
|||||||
continue
|
continue
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result["errors"].append(f"normalize: {e}")
|
result["errors"].append(f"normalize: {e}")
|
||||||
|
# 写死信表:规范化失败
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "understat", "match", str(raw.get("id", "")),
|
||||||
|
"normalize_error", str(e), raw,
|
||||||
|
)
|
||||||
continue
|
continue
|
||||||
|
normalized_matches.append((nm, raw))
|
||||||
|
all_team_names.add(nm.home_team)
|
||||||
|
all_team_names.add(nm.away_team)
|
||||||
|
|
||||||
# 匹配已有 Match(天级) - 使用 Repository
|
if not normalized_matches:
|
||||||
home_team = await team_repo.get_by_name(nm.home_team)
|
return result
|
||||||
away_team = await team_repo.get_by_name(nm.away_team)
|
|
||||||
if home_team is None or away_team is None:
|
# === 批量查询球队(1 次 DB 往返) ===
|
||||||
|
team_name_to_id = {}
|
||||||
|
if all_team_names:
|
||||||
|
teams = await team_repo.get_all_by_names(list(all_team_names))
|
||||||
|
team_name_to_id = {name: team.id for name, team in teams.items()}
|
||||||
|
|
||||||
|
# === 批量查询已有比赛(1 次 DB 往返,按日期范围) ===
|
||||||
|
match_dict: dict[tuple, Match] = {}
|
||||||
|
dates = [nm.date for nm, _ in normalized_matches if nm.date is not None]
|
||||||
|
if dates:
|
||||||
|
min_dt = min(dates) - timedelta(days=30)
|
||||||
|
max_dt = max(dates) + timedelta(days=30)
|
||||||
|
stmt = (
|
||||||
|
select(Match)
|
||||||
|
.options(selectinload(Match.stats))
|
||||||
|
.where(Match.league_id == league_obj.id)
|
||||||
|
.where(Match.match_date >= min_dt)
|
||||||
|
.where(Match.match_date <= max_dt)
|
||||||
|
)
|
||||||
|
for m in (await db.execute(stmt)).scalars():
|
||||||
|
key = _match_key(m.home_team_id, m.away_team_id, m.match_date_date)
|
||||||
|
match_dict[key] = m
|
||||||
|
|
||||||
|
# === 内存匹配 + 回填 xG(覆盖模式) ===
|
||||||
|
for nm, raw in normalized_matches:
|
||||||
|
home_team_id = team_name_to_id.get(nm.home_team)
|
||||||
|
away_team_id = team_name_to_id.get(nm.away_team)
|
||||||
|
if home_team_id is None or away_team_id is None:
|
||||||
result["unmatched"] += 1
|
result["unmatched"] += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
existing = await match_repo.find_by_teams_and_date(
|
match_key = _match_key(home_team_id, away_team_id, nm.date)
|
||||||
league_obj.id, home_team.id, away_team.id, nm.date
|
existing = match_dict.get(match_key)
|
||||||
)
|
|
||||||
if existing is None:
|
if existing is None:
|
||||||
result["unmatched"] += 1
|
result["unmatched"] += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# 回填 xG
|
|
||||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
|
source_record_id = str(raw.get("id", ""))
|
||||||
|
|
||||||
|
# 创建 stats 记录(如果不存在)
|
||||||
|
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||||
existing.stats = MatchStats(
|
existing.stats = MatchStats(
|
||||||
match_id=existing.id,
|
match_id=existing.id,
|
||||||
source="understat",
|
source="understat",
|
||||||
source_event_id=str(raw.get("id", "")),
|
source_record_id=source_record_id,
|
||||||
retrieved_at=now,
|
retrieved_at=now,
|
||||||
available_at=now,
|
available_at=now,
|
||||||
|
xg_source="understat",
|
||||||
|
xg_updated_at=now,
|
||||||
|
xg_source_record_id=source_record_id,
|
||||||
)
|
)
|
||||||
db.add(existing.stats)
|
db.add(existing.stats)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
|
# P2-1: Silver 层血缘 — MatchStats 创建
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="understat",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
target_table="match_stats",
|
||||||
|
target_id=existing.id,
|
||||||
|
transform_name="silver_upsert",
|
||||||
|
batch_id=batch_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
# xG 覆盖更新模式:当 understat 数据更新时覆盖旧值
|
||||||
if existing.stats is not None:
|
if existing.stats is not None:
|
||||||
if existing.stats.home_xg is None and nm.home_xg is not None:
|
xg_updated = False
|
||||||
|
if nm.home_xg is not None:
|
||||||
existing.stats.home_xg = nm.home_xg
|
existing.stats.home_xg = nm.home_xg
|
||||||
|
existing.stats.xg_source = "understat"
|
||||||
|
existing.stats.xg_updated_at = now
|
||||||
|
existing.stats.xg_source_record_id = source_record_id
|
||||||
result["updated"] += 1
|
result["updated"] += 1
|
||||||
if existing.stats.away_xg is None and nm.away_xg is not None:
|
xg_updated = True
|
||||||
|
if nm.away_xg is not None:
|
||||||
existing.stats.away_xg = nm.away_xg
|
existing.stats.away_xg = nm.away_xg
|
||||||
|
existing.stats.xg_source = "understat"
|
||||||
|
existing.stats.xg_updated_at = now
|
||||||
|
existing.stats.xg_source_record_id = source_record_id
|
||||||
|
xg_updated = True
|
||||||
|
# P2-1: Silver 层血缘 — xG 覆盖更新
|
||||||
|
if xg_updated:
|
||||||
|
await _write_data_lineage(
|
||||||
|
db,
|
||||||
|
source_system="understat",
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
target_table="match_stats",
|
||||||
|
target_id=existing.id,
|
||||||
|
transform_name="silver_xg_update",
|
||||||
|
batch_id=batch_id,
|
||||||
|
transform_detail=f"home_xg={nm.home_xg} away_xg={nm.away_xg}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# P2-2: 数据质量检查 — 行计数合理性
|
||||||
|
total_processed = result["updated"] + result["skipped"] + result["unmatched"]
|
||||||
|
await _write_data_quality_check(
|
||||||
|
db,
|
||||||
|
check_name="understat_row_count",
|
||||||
|
entity_type="match",
|
||||||
|
entity_id=f"{league}_{season}",
|
||||||
|
expected_value=">=1",
|
||||||
|
actual_value=str(total_processed),
|
||||||
|
passed=total_processed > 0,
|
||||||
|
severity="critical" if total_processed == 0 else "info",
|
||||||
|
detail=f"league={league} season={season} updated={result['updated']} skipped={result['skipped']} unmatched={result['unmatched']}",
|
||||||
|
)
|
||||||
|
|
||||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||||
return result
|
return result
|
||||||
|
|||||||
+4
-2
@@ -17,8 +17,10 @@ engine = create_async_engine(
|
|||||||
settings.DATABASE_URL,
|
settings.DATABASE_URL,
|
||||||
echo=False,
|
echo=False,
|
||||||
pool_pre_ping=True,
|
pool_pre_ping=True,
|
||||||
pool_size=10,
|
pool_size=settings.DB_POOL_SIZE,
|
||||||
max_overflow=20,
|
max_overflow=settings.DB_MAX_OVERFLOW,
|
||||||
|
pool_timeout=settings.DB_POOL_TIMEOUT,
|
||||||
|
pool_recycle=settings.DB_POOL_RECYCLE,
|
||||||
)
|
)
|
||||||
|
|
||||||
AsyncSessionLocal = async_sessionmaker(
|
AsyncSessionLocal = async_sessionmaker(
|
||||||
|
|||||||
+139
-5
@@ -1,4 +1,4 @@
|
|||||||
"""6 张表 ORM: leagues / teams / matches / match_stats / predictions / injuries。"""
|
"""ORM 模型: leagues / teams / matches / match_stats / predictions / injuries / raw_events / ingest_failures / data_quality_checks / data_lineage。"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
@@ -14,6 +14,7 @@ from sqlalchemy import (
|
|||||||
Integer,
|
Integer,
|
||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
func,
|
func,
|
||||||
)
|
)
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
@@ -75,10 +76,20 @@ class Match(Base):
|
|||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||||
|
|
||||||
league: Mapped[League] = relationship(back_populates="matches")
|
league: Mapped[League] = relationship(back_populates="matches", lazy="selectin")
|
||||||
home_team: Mapped[Team] = relationship(foreign_keys=[home_team_id], back_populates="home_matches")
|
# lazy="selectin": 这些关系在业务里几乎总是一起读取(切片/回测/展示)。
|
||||||
away_team: Mapped[Team] = relationship(foreign_keys=[away_team_id], back_populates="away_matches")
|
# 默认的 lazy="select" 在 async SQLAlchemy 下,于 session 之外或未显式
|
||||||
stats: Mapped["MatchStats | None"] = relationship(back_populates="match", cascade="all, delete-orphan")
|
# eager-load 时访问会抛 MissingGreenlet —— 已因此导致 form/stats/h2h
|
||||||
|
# 三个专家切片静默失败。统一改为预加载,从根上消除这类问题。
|
||||||
|
home_team: Mapped[Team] = relationship(
|
||||||
|
foreign_keys=[home_team_id], back_populates="home_matches", lazy="selectin"
|
||||||
|
)
|
||||||
|
away_team: Mapped[Team] = relationship(
|
||||||
|
foreign_keys=[away_team_id], back_populates="away_matches", lazy="selectin"
|
||||||
|
)
|
||||||
|
stats: Mapped["MatchStats | None"] = relationship(
|
||||||
|
back_populates="match", cascade="all, delete-orphan", lazy="selectin"
|
||||||
|
)
|
||||||
predictions: Mapped[list["Prediction"]] = relationship(back_populates="match", cascade="all, delete-orphan")
|
predictions: Mapped[list["Prediction"]] = relationship(back_populates="match", cascade="all, delete-orphan")
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
@@ -120,9 +131,20 @@ class MatchStats(Base):
|
|||||||
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
||||||
retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
retrieved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
available_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
# xG 数据追踪:支持 understat 纠正旧 xG 值
|
||||||
|
xg_source: Mapped[str | None] = mapped_column(String(30)) # 具体 xG 数据源
|
||||||
|
xg_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # xG 最后更新时间
|
||||||
|
xg_source_record_id: Mapped[str | None] = mapped_column(String(100)) # xG 对应的源记录 ID
|
||||||
|
|
||||||
match: Mapped[Match] = relationship(back_populates="stats")
|
match: Mapped[Match] = relationship(back_populates="stats")
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# 数据血缘时间过滤查询用(按 available_at 取「赛前已可得」的统计)
|
||||||
|
Index("ix_match_stats_available_at", "available_at"),
|
||||||
|
# xG 数据源追踪查询用
|
||||||
|
Index("ix_match_stats_xg_source", "xg_source", "xg_updated_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class Injury(Base):
|
class Injury(Base):
|
||||||
"""球员伤停记录(api-football 数据源)。"""
|
"""球员伤停记录(api-football 数据源)。"""
|
||||||
@@ -183,8 +205,15 @@ class Prediction(Base):
|
|||||||
match: Mapped[Match] = relationship(back_populates="predictions")
|
match: Mapped[Match] = relationship(back_populates="predictions")
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# P1-6: 数据库级唯一约束,防止同一 match+provider+model 产生重复预测
|
||||||
|
UniqueConstraint(
|
||||||
|
"match_id", "provider", "model",
|
||||||
|
name="uq_predictions_match_provider_model",
|
||||||
|
),
|
||||||
Index("ix_predictions_match", "match_id"),
|
Index("ix_predictions_match", "match_id"),
|
||||||
Index("ix_predictions_provider_model", "provider", "model"),
|
Index("ix_predictions_provider_model", "provider", "model"),
|
||||||
|
# 数据截止时间过滤查询用(按 prediction_cutoff_at 取「赛前已生成」的预测)
|
||||||
|
Index("ix_predictions_cutoff_at", "prediction_cutoff_at"),
|
||||||
# 数据库级约束:最后一道防线
|
# 数据库级约束:最后一道防线
|
||||||
CheckConstraint("pred_home_goals >= 0", name="ck_pred_home_goals_nonneg"),
|
CheckConstraint("pred_home_goals >= 0", name="ck_pred_home_goals_nonneg"),
|
||||||
CheckConstraint("pred_away_goals >= 0", name="ck_pred_away_goals_nonneg"),
|
CheckConstraint("pred_away_goals >= 0", name="ck_pred_away_goals_nonneg"),
|
||||||
@@ -193,3 +222,108 @@ class Prediction(Base):
|
|||||||
CheckConstraint("mode IN ('single', 'multi')", name="ck_mode_enum"),
|
CheckConstraint("mode IN ('single', 'multi')", name="ck_mode_enum"),
|
||||||
CheckConstraint("status IN ('success', 'failed', 'degraded')", name="ck_status_enum"),
|
CheckConstraint("status IN ('success', 'failed', 'degraded')", name="ck_status_enum"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Bronze 层:原始事件记录 ──────────────────────────────────────────────
|
||||||
|
class RawEvent(Base):
|
||||||
|
"""Bronze 层:不可变的原始采集记录。
|
||||||
|
|
||||||
|
每个采集到的原始事件先写入此表,再规范化到 Silver 层(matches / match_stats)。
|
||||||
|
提供完整的数据血缘回溯能力。
|
||||||
|
"""
|
||||||
|
__tablename__ = "raw_events"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_system: Mapped[str] = mapped_column(String(30), nullable=False) # bzzoiro / understat / api-football
|
||||||
|
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False) # 源系统记录 ID
|
||||||
|
raw_payload: Mapped[dict] = mapped_column(JSONB, nullable=False) # 完整原始 JSON
|
||||||
|
ingested_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, nullable=False)
|
||||||
|
ingest_batch_id: Mapped[str | None] = mapped_column(String(64)) # 批次 ID,用于关联同次采集
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_system", "source_record_id", name="uq_raw_events_source_record"),
|
||||||
|
Index("ix_raw_events_batch", "ingest_batch_id"),
|
||||||
|
Index("ix_raw_events_source_ingested", "source_system", "ingested_at"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── 死信表:采集失败记录 ─────────────────────────────────────────────────
|
||||||
|
class IngestFailure(Base):
|
||||||
|
"""采集失败死信表。
|
||||||
|
|
||||||
|
当采集或规范化失败时,写入此表而非仅内存 dict。
|
||||||
|
支持自动重试和人工排查。
|
||||||
|
"""
|
||||||
|
__tablename__ = "ingest_failures"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_system: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||||
|
entity_type: Mapped[str] = mapped_column(String(30), nullable=False) # match / injury / team
|
||||||
|
source_record_id: Mapped[str | None] = mapped_column(String(100))
|
||||||
|
error_type: Mapped[str] = mapped_column(String(50), nullable=False) # fetch_error / normalize_error / db_error / validation_error
|
||||||
|
error_detail: Mapped[str | None] = mapped_column(Text)
|
||||||
|
raw_payload: Mapped[dict | None] = mapped_column(JSONB) # 失败时的原始数据,用于重试
|
||||||
|
retry_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||||
|
next_retry_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
status: Mapped[str] = mapped_column(String(20), default="pending") # pending / retrying / resolved / abandoned
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
||||||
|
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_ingest_failures_status_next_retry", "status", "next_retry_at"),
|
||||||
|
Index("ix_ingest_failures_source", "source_system", "entity_type"),
|
||||||
|
CheckConstraint("status IN ('pending', 'retrying', 'resolved', 'abandoned')", name="ck_ingest_failures_status"),
|
||||||
|
CheckConstraint("retry_count >= 0", name="ck_ingest_failures_retry_nonneg"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── 数据质量检查表 ──────────────────────────────────────────────────────
|
||||||
|
class DataQualityCheck(Base):
|
||||||
|
"""数据质量检查结果记录。
|
||||||
|
|
||||||
|
每次运行数据质量检查时,将结果写入此表用于趋势分析和告警。
|
||||||
|
"""
|
||||||
|
__tablename__ = "data_quality_checks"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
check_name: Mapped[str] = mapped_column(String(100), nullable=False) # 检查名称
|
||||||
|
entity_type: Mapped[str] = mapped_column(String(30), nullable=False) # match / team / prediction
|
||||||
|
entity_id: Mapped[str | None] = mapped_column(String(50)) # 具体实体 ID
|
||||||
|
expected_value: Mapped[str | None] = mapped_column(Text) # 期望值(描述)
|
||||||
|
actual_value: Mapped[str | None] = mapped_column(Text) # 实际值
|
||||||
|
passed: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||||
|
severity: Mapped[str] = mapped_column(String(10), nullable=False, default="warning") # info / warning / critical
|
||||||
|
detail: Mapped[str | None] = mapped_column(Text) # 详细描述
|
||||||
|
checked_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, nullable=False)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_dqc_check_time", "check_name", "checked_at"),
|
||||||
|
Index("ix_dqc_entity", "entity_type", "entity_id"),
|
||||||
|
Index("ix_dqc_severity_passed", "severity", "passed"),
|
||||||
|
CheckConstraint("severity IN ('info', 'warning', 'critical')", name="ck_dqc_severity"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── 数据血缘表 ──────────────────────────────────────────────────────────
|
||||||
|
class DataLineage(Base):
|
||||||
|
"""ETL 全过程元数据记录。
|
||||||
|
|
||||||
|
追踪从 Bronze → Silver → Gold 的完整转换链路。
|
||||||
|
"""
|
||||||
|
__tablename__ = "data_lineage"
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_system: Mapped[str] = mapped_column(String(30), nullable=False)
|
||||||
|
source_record_id: Mapped[str] = mapped_column(String(100), nullable=False)
|
||||||
|
target_table: Mapped[str] = mapped_column(String(50), nullable=False) # matches / match_stats / predictions
|
||||||
|
target_id: Mapped[int | None] = mapped_column(Integer) # 目标表记录 ID
|
||||||
|
transform_name: Mapped[str] = mapped_column(String(100), nullable=False) # 转换步骤名称
|
||||||
|
transform_detail: Mapped[str | None] = mapped_column(Text) # 转换详情
|
||||||
|
batch_id: Mapped[str | None] = mapped_column(String(64)) # 批次 ID
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, nullable=False)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_lineage_source", "source_system", "source_record_id"),
|
||||||
|
Index("ix_lineage_target", "target_table", "target_id"),
|
||||||
|
Index("ix_lineage_batch", "batch_id"),
|
||||||
|
)
|
||||||
|
|||||||
+18
-4
@@ -5,7 +5,8 @@ Repository 只负责查询,不负责事务提交。
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from datetime import datetime
|
||||||
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
@@ -37,16 +38,29 @@ class MatchRepository:
|
|||||||
async def find_by_teams_and_date(
|
async def find_by_teams_and_date(
|
||||||
self, league_id: int, home_team_id: int, away_team_id: int, date
|
self, league_id: int, home_team_id: int, away_team_id: int, date
|
||||||
) -> Match | None:
|
) -> Match | None:
|
||||||
"""按联赛+主队+客队+日期查找比赛(天级匹配)。"""
|
"""按联赛+主队+客队+日期查找比赛(天级匹配)。
|
||||||
if hasattr(date, "date"):
|
|
||||||
|
预加载 stats:调用方(understat 回填)会读取 existing.stats,
|
||||||
|
async session 下惰性加载会抛 MissingGreenlet。
|
||||||
|
|
||||||
|
P2-3: 使用 match_date_date(已建索引)做等值匹配,避免 func.date()
|
||||||
|
导致的全表扫描。
|
||||||
|
"""
|
||||||
|
if isinstance(date, datetime):
|
||||||
date = date.date()
|
date = date.date()
|
||||||
|
elif hasattr(date, "date"):
|
||||||
|
date = date.date()
|
||||||
|
else:
|
||||||
|
# 字符串等其它格式,尝试转换
|
||||||
|
date = datetime.fromisoformat(str(date)).date()
|
||||||
|
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Match)
|
select(Match)
|
||||||
|
.options(selectinload(Match.stats))
|
||||||
.where(Match.league_id == league_id)
|
.where(Match.league_id == league_id)
|
||||||
.where(Match.home_team_id == home_team_id)
|
.where(Match.home_team_id == home_team_id)
|
||||||
.where(Match.away_team_id == away_team_id)
|
.where(Match.away_team_id == away_team_id)
|
||||||
.where(func.date(Match.match_date) == date)
|
.where(Match.match_date_date == date)
|
||||||
)
|
)
|
||||||
return (await self._session.execute(stmt)).scalar_one_or_none()
|
return (await self._session.execute(stmt)).scalar_one_or_none()
|
||||||
|
|
||||||
|
|||||||
@@ -9,12 +9,16 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
from src.db.base import AsyncSessionLocal
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def get_uow() -> AsyncIterator[AsyncSessionLocal]:
|
async def get_uow() -> AsyncIterator[AsyncSession]:
|
||||||
"""创建新的工作单元(用于非路由上下文)。
|
"""创建新的工作单元(用于非路由上下文)。
|
||||||
|
|
||||||
用法:
|
用法:
|
||||||
|
|||||||
+23
-5
@@ -12,7 +12,7 @@ import logging
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from src.llm.context_builder import MatchHeader
|
from src.llm.context_builder import MatchHeader, SliceResult
|
||||||
from src.llm.provider import LLMProvider, LLMResponse
|
from src.llm.provider import LLMProvider, LLMResponse
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -80,7 +80,12 @@ class AgentReport:
|
|||||||
|
|
||||||
|
|
||||||
def _is_no_data(slice_text: str) -> bool:
|
def _is_no_data(slice_text: str) -> bool:
|
||||||
"""切片是否全无数据(除了标题行全是无数据)。"""
|
"""兜底: 判断纯字符串切片是否全无数据。
|
||||||
|
|
||||||
|
仅用于 slice_fn 返回 `str`(未升级为 SliceResult)的场景。
|
||||||
|
新代码应让切片返回 SliceResult 并显式声明 has_data —— 字符串子串匹配
|
||||||
|
依赖具体文案(「无比分数据」「无伤停数据」等变体会漏判),不可靠。
|
||||||
|
"""
|
||||||
body = [ln.strip() for ln in slice_text.splitlines() if ln.strip()]
|
body = [ln.strip() for ln in slice_text.splitlines() if ln.strip()]
|
||||||
# 去掉标题行(── 开头)
|
# 去掉标题行(── 开头)
|
||||||
content = [ln for ln in body if not ln.startswith("──")]
|
content = [ln for ln in body if not ln.startswith("──")]
|
||||||
@@ -89,6 +94,18 @@ def _is_no_data(slice_text: str) -> bool:
|
|||||||
return all(any(s in ln for s in NO_DATA_SENTINELS) for ln in content)
|
return all(any(s in ln for s in NO_DATA_SENTINELS) for ln in content)
|
||||||
|
|
||||||
|
|
||||||
|
def _slice_has_data(slice_result) -> tuple[str, bool]:
|
||||||
|
"""把切片返回值统一成 (text, has_data)。
|
||||||
|
|
||||||
|
优先使用 SliceResult.has_data(结构化,可信);若切片函数仍返回 str,
|
||||||
|
则回退到文案子串匹配(向后兼容)。
|
||||||
|
"""
|
||||||
|
if isinstance(slice_result, SliceResult):
|
||||||
|
return slice_result.text, slice_result.has_data
|
||||||
|
text = str(slice_result)
|
||||||
|
return text, not _is_no_data(text)
|
||||||
|
|
||||||
|
|
||||||
def _stub_no_data(agent: str) -> AgentReport:
|
def _stub_no_data(agent: str) -> AgentReport:
|
||||||
return AgentReport(
|
return AgentReport(
|
||||||
agent=agent,
|
agent=agent,
|
||||||
@@ -145,13 +162,14 @@ async def run_agent(
|
|||||||
"""执行单个专家 agent: 切片 → no_data 门控 → 调 LLM → 解析报告。"""
|
"""执行单个专家 agent: 切片 → no_data 门控 → 调 LLM → 解析报告。"""
|
||||||
# 1. 数据切片
|
# 1. 数据切片
|
||||||
try:
|
try:
|
||||||
slice_text = await spec.slice_fn(header, before=before)
|
slice_result = await spec.slice_fn(header, before=before)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("agent %s slice failed", spec.name)
|
logger.exception("agent %s slice failed", spec.name)
|
||||||
return AgentReport(agent=spec.name, status="error", analysis=f"数据切片失败: {e}")
|
return AgentReport(agent=spec.name, status="error", analysis=f"数据切片失败: {e}")
|
||||||
|
|
||||||
# 2. no_data 门控: 切片无数据 → 不调 LLM
|
# 2. no_data 门控: 切片显式声明无数据 → 不调 LLM
|
||||||
if _is_no_data(slice_text):
|
slice_text, has_data = _slice_has_data(slice_result)
|
||||||
|
if not has_data:
|
||||||
logger.debug("agent %s: slice is no_data, skipping LLM", spec.name)
|
logger.debug("agent %s: slice is no_data, skipping LLM", spec.name)
|
||||||
return _stub_no_data(spec.name)
|
return _stub_no_data(spec.name)
|
||||||
|
|
||||||
|
|||||||
@@ -195,13 +195,14 @@ async def predict_match_multi(
|
|||||||
raise ValueError(f"match {match_id} not found")
|
raise ValueError(f"match {match_id} not found")
|
||||||
|
|
||||||
# 严格校验终裁输出
|
# 严格校验终裁输出
|
||||||
from src.llm.validation import validate_prediction_output
|
from src.llm.validation import validate_agent_weights, validate_prediction_output
|
||||||
try:
|
try:
|
||||||
validated = validate_prediction_output(final)
|
validated = validate_prediction_output(final)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise RuntimeError(f"终裁输出校验失败: {e}")
|
raise RuntimeError(f"终裁输出校验失败: {e}")
|
||||||
|
|
||||||
agent_weights = final.get("agent_weights")
|
# agent_weights 同样必须过校验(旧实现直接取 raw 值落库,未做任何检查)
|
||||||
|
agent_weights = validate_agent_weights(final.get("agent_weights"))
|
||||||
pred = Prediction(
|
pred = Prediction(
|
||||||
match_id=match_id,
|
match_id=match_id,
|
||||||
provider=settings.LLM_PROVIDER,
|
provider=settings.LLM_PROVIDER,
|
||||||
|
|||||||
+77
-40
@@ -3,18 +3,23 @@
|
|||||||
核心机制:
|
核心机制:
|
||||||
- build_context 已内置 before=match_date,天然防未来信息泄漏
|
- build_context 已内置 before=match_date,天然防未来信息泄漏
|
||||||
- 对历史比赛跑预测 → 用实际比分 settle → 统计准确率
|
- 对历史比赛跑预测 → 用实际比分 settle → 统计准确率
|
||||||
|
- 并发控制: asyncio.Semaphore 限制同时 LLM 调用数
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import and_, select
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from src.db.models import League, Match
|
from src.db.models import Match
|
||||||
from src.db.unit_of_work import get_uow
|
from src.db.unit_of_work import get_uow
|
||||||
from src.llm.eval import settle_prediction
|
from src.llm.eval import settle_prediction
|
||||||
from src.llm.predict import predict_match
|
from src.llm.predict import predict_match
|
||||||
|
from src.llm.utils import actual_1x2
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -38,6 +43,23 @@ class BacktestMatchResult:
|
|||||||
prediction_id: int
|
prediction_id: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class BacktestCandidate:
|
||||||
|
"""回测候选比赛(字段快照,不持有 ORM 对象)。
|
||||||
|
|
||||||
|
session 关闭后仍可安全读取:所有需要的关系字段已在查询时物化为普通值,
|
||||||
|
避免在 session 之外访问惰性加载的关系属性(会抛 MissingGreenlet)。
|
||||||
|
"""
|
||||||
|
match_id: int
|
||||||
|
league_code: str | None
|
||||||
|
home_team: str
|
||||||
|
away_team: str
|
||||||
|
match_date: datetime
|
||||||
|
home_goals: int
|
||||||
|
away_goals: int
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class BacktestSummary:
|
class BacktestSummary:
|
||||||
"""回测汇总统计。"""
|
"""回测汇总统计。"""
|
||||||
@@ -50,15 +72,6 @@ class BacktestSummary:
|
|||||||
results: list[BacktestMatchResult] = field(default_factory=list)
|
results: list[BacktestMatchResult] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
def _actual_1x2(home: int, away: int) -> str:
|
|
||||||
"""实际比分 → 胜平负。"""
|
|
||||||
if home > away:
|
|
||||||
return "1"
|
|
||||||
if home == away:
|
|
||||||
return "X"
|
|
||||||
return "2"
|
|
||||||
|
|
||||||
|
|
||||||
async def _get_historical_matches(
|
async def _get_historical_matches(
|
||||||
db,
|
db,
|
||||||
*,
|
*,
|
||||||
@@ -66,10 +79,20 @@ async def _get_historical_matches(
|
|||||||
date_from: str | None = None,
|
date_from: str | None = None,
|
||||||
date_to: str | None = None,
|
date_to: str | None = None,
|
||||||
limit: int = 50,
|
limit: int = 50,
|
||||||
) -> list[Match]:
|
) -> list[BacktestCandidate]:
|
||||||
"""查询已完赛且有比分的比赛(回测候选)。"""
|
"""查询已完赛且有比分的比赛(回测候选)。
|
||||||
|
|
||||||
|
返回普通值快照而非 ORM 对象:调用方在 session 关闭后仍需使用这些字段,
|
||||||
|
而 league / home_team / away_team 是惰性加载关系,在 async 下于 session
|
||||||
|
之外访问会抛 MissingGreenlet。这里用 selectinload 预加载后立即物化。
|
||||||
|
"""
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Match)
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.league),
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
.where(Match.match_status == "finished")
|
.where(Match.match_status == "finished")
|
||||||
.where(Match.home_goals.is_not(None))
|
.where(Match.home_goals.is_not(None))
|
||||||
.where(Match.away_goals.is_not(None))
|
.where(Match.away_goals.is_not(None))
|
||||||
@@ -83,7 +106,19 @@ async def _get_historical_matches(
|
|||||||
|
|
||||||
stmt = stmt.order_by(Match.match_date.desc()).limit(limit)
|
stmt = stmt.order_by(Match.match_date.desc()).limit(limit)
|
||||||
result = await db.execute(stmt)
|
result = await db.execute(stmt)
|
||||||
return list(result.scalars().all())
|
# 在 session 内物化为纯数据,切断与 ORM 会话的耦合
|
||||||
|
return [
|
||||||
|
BacktestCandidate(
|
||||||
|
match_id=m.id,
|
||||||
|
league_code=m.league.code if m.league else None,
|
||||||
|
home_team=m.home_team.name if m.home_team else "?",
|
||||||
|
away_team=m.away_team.name if m.away_team else "?",
|
||||||
|
match_date=m.match_date,
|
||||||
|
home_goals=m.home_goals,
|
||||||
|
away_goals=m.away_goals,
|
||||||
|
)
|
||||||
|
for m in result.scalars().all()
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
async def run_backtest(
|
async def run_backtest(
|
||||||
@@ -109,45 +144,47 @@ async def run_backtest(
|
|||||||
BacktestSummary 含逐场结果 + 汇总统计
|
BacktestSummary 含逐场结果 + 汇总统计
|
||||||
"""
|
"""
|
||||||
async with get_uow() as session:
|
async with get_uow() as session:
|
||||||
matches = await _get_historical_matches(
|
candidates = await _get_historical_matches(
|
||||||
session, league_id=league_id, date_from=date_from, date_to=date_to, limit=limit
|
session, league_id=league_id, date_from=date_from, date_to=date_to, limit=limit
|
||||||
)
|
)
|
||||||
|
|
||||||
summary = BacktestSummary(total=len(matches), scored=0)
|
summary = BacktestSummary(total=len(candidates), scored=0)
|
||||||
|
|
||||||
for m in matches:
|
# P2: 并发控制,同时最多 3 场预测(避免 LLM API 限流,保护下游服务)
|
||||||
|
sem = asyncio.Semaphore(3)
|
||||||
|
|
||||||
|
async def _one(c: BacktestCandidate) -> BacktestMatchResult | None:
|
||||||
|
async with sem:
|
||||||
try:
|
try:
|
||||||
# 预测 (build_context 内部已用 before=match_date 防泄漏,
|
result = await predict_match(c.match_id, mode=mode, model=model, use_cache=False, backtest=True)
|
||||||
# injuries_slice 也使用 as_of=match_date 过滤 retrieved_at)
|
await settle_prediction(result.prediction_id, c.home_goals, c.away_goals)
|
||||||
result = await predict_match(m.id, mode=mode, model=model)
|
actual = actual_1x2(c.home_goals, c.away_goals)
|
||||||
|
return BacktestMatchResult(
|
||||||
# 用实际比分 settle
|
match_id=c.match_id,
|
||||||
await settle_prediction(result.prediction_id, m.home_goals, m.away_goals)
|
league_code=c.league_code,
|
||||||
|
home_team=c.home_team,
|
||||||
actual = _actual_1x2(m.home_goals, m.away_goals)
|
away_team=c.away_team,
|
||||||
correct = result.pred_1x2 == actual
|
match_date=c.match_date.strftime("%Y-%m-%d") if c.match_date else "?",
|
||||||
|
actual_home=c.home_goals,
|
||||||
bt = BacktestMatchResult(
|
actual_away=c.away_goals,
|
||||||
match_id=m.id,
|
|
||||||
league_code=m.league.code if m.league else None,
|
|
||||||
home_team=m.home_team.name if m.home_team else "?",
|
|
||||||
away_team=m.away_team.name if m.away_team else "?",
|
|
||||||
match_date=m.match_date.strftime("%Y-%m-%d") if m.match_date else "?",
|
|
||||||
actual_home=m.home_goals,
|
|
||||||
actual_away=m.away_goals,
|
|
||||||
actual_1x2=actual,
|
actual_1x2=actual,
|
||||||
pred_home=result.pred_home_goals,
|
pred_home=result.pred_home_goals,
|
||||||
pred_away=result.pred_away_goals,
|
pred_away=result.pred_away_goals,
|
||||||
pred_1x2=result.pred_1x2,
|
pred_1x2=result.pred_1x2,
|
||||||
subjective_confidence=result.subjective_confidence,
|
subjective_confidence=result.subjective_confidence,
|
||||||
correct_1x2=correct,
|
correct_1x2=result.pred_1x2 == actual,
|
||||||
prediction_id=result.prediction_id,
|
prediction_id=result.prediction_id,
|
||||||
)
|
)
|
||||||
summary.results.append(bt)
|
except Exception:
|
||||||
summary.scored += 1
|
logger.exception("backtest match %s failed", c.match_id)
|
||||||
|
return None
|
||||||
|
|
||||||
except Exception as e:
|
# 并行执行,保持结果顺序
|
||||||
logger.warning("backtest match %s failed: %s", m.id, e)
|
results = await asyncio.gather(*[_one(c) for c in candidates])
|
||||||
|
for r in results:
|
||||||
|
if r is not None:
|
||||||
|
summary.results.append(r)
|
||||||
|
summary.scored += 1
|
||||||
|
|
||||||
# 汇总统计
|
# 汇总统计
|
||||||
if summary.scored > 0:
|
if summary.scored > 0:
|
||||||
|
|||||||
+155
-52
@@ -6,11 +6,17 @@
|
|||||||
- build_context: 单 agent 路径,拼接全部切片(行为与旧版一致)
|
- build_context: 单 agent 路径,拼接全部切片(行为与旧版一致)
|
||||||
|
|
||||||
multi-agent 路径由 agents/orchestrator.py 调用切片函数,每个专家只拿自己的切片。
|
multi-agent 路径由 agents/orchestrator.py 调用切片函数,每个专家只拿自己的切片。
|
||||||
|
|
||||||
|
性能说明:
|
||||||
|
build_context 创建一个共享 session 并传给所有切片函数,
|
||||||
|
避免每个切片独立创建 session —— 回测 20 场并发时,
|
||||||
|
5 个切片 × 20 场 = 100 个连接会耗尽连接池(pool_size=15)。
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
@@ -18,6 +24,9 @@ from sqlalchemy.orm import selectinload
|
|||||||
from src.db.base import AsyncSessionLocal
|
from src.db.base import AsyncSessionLocal
|
||||||
from src.db.models import Match
|
from src.db.models import Match
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@@ -39,6 +48,22 @@ def _is_stats_available(stats, before) -> bool:
|
|||||||
return stats.available_at <= before
|
return stats.available_at <= before
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SliceResult:
|
||||||
|
"""数据切片的显式结果(替代「靠文案子串猜有无数据」)。
|
||||||
|
|
||||||
|
旧实现用 `"无数据" in slice_text` 判断,依赖具体文案 —— 一旦某个切片
|
||||||
|
写成「无比分数据」「无伤停数据」这类变体,判断就会静默失配
|
||||||
|
(见审查报告 P2-1)。这里让切片函数直接声明 `has_data`,不再猜。
|
||||||
|
"""
|
||||||
|
text: str
|
||||||
|
has_data: bool
|
||||||
|
n_records: int = 0
|
||||||
|
|
||||||
|
def __str__(self) -> str: # 让老调用点可直接当 str 用
|
||||||
|
return self.text
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class MatchContext:
|
class MatchContext:
|
||||||
match_id: int
|
match_id: int
|
||||||
@@ -64,11 +89,19 @@ class MatchHeader:
|
|||||||
league_id: int
|
league_id: int
|
||||||
|
|
||||||
|
|
||||||
async def load_match_header(match_id: int) -> MatchHeader:
|
async def load_match_header(match_id: int, db: AsyncSession | None = None) -> MatchHeader:
|
||||||
"""加载比赛头信息(各 agent 共用)。"""
|
"""加载比赛头信息(各 agent 共用)。
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
|
Args:
|
||||||
|
match_id: 比赛 ID
|
||||||
|
db: 可选的共享 session。不传则自建(向后兼容)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
m = await _load_match(db, match_id)
|
m = await _load_match(db, match_id)
|
||||||
return _to_header(m)
|
return _to_header(m)
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
m = await _load_match(new_db, match_id)
|
||||||
|
return _to_header(m)
|
||||||
|
|
||||||
|
|
||||||
def _to_header(m: Match) -> MatchHeader:
|
def _to_header(m: Match) -> MatchHeader:
|
||||||
@@ -98,16 +131,24 @@ def header_text(h: MatchHeader) -> str:
|
|||||||
# 切片函数: 每个领域 agent 一个
|
# 切片函数: 每个领域 agent 一个
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
async def h2h_slice(header: MatchHeader, *, limit: int = 8, before=None) -> str:
|
async def h2h_slice(header: MatchHeader, *, limit: int = 8, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
"""E - 历史交锋切片: 过去数年 + 近期交手数据,提取交手规律。before=match_date 用于回测。"""
|
"""E - 历史交锋切片: 过去数年 + 近期交手数据,提取交手规律。before=match_date 用于回测。
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
h2h = await _get_h2h(db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
h2h = await _get_h2h(db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
h2h = await _get_h2h(new_db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
||||||
lines = [f"── 历史交锋(近 {limit} 次) ──"]
|
lines = [f"── 历史交锋(近 {limit} 次) ──"]
|
||||||
|
n_with_score = 0
|
||||||
if h2h:
|
if h2h:
|
||||||
home_wins = draws = away_wins = 0
|
home_wins = draws = away_wins = 0
|
||||||
for hm in h2h:
|
for hm in h2h:
|
||||||
d = hm.match_date.strftime("%Y-%m") if hm.match_date else "?"
|
d = hm.match_date.strftime("%Y-%m") if hm.match_date else "?"
|
||||||
if hm.home_goals is not None:
|
if hm.home_goals is not None:
|
||||||
|
n_with_score += 1
|
||||||
if hm.home_goals > hm.away_goals: home_wins += 1
|
if hm.home_goals > hm.away_goals: home_wins += 1
|
||||||
elif hm.home_goals == hm.away_goals: draws += 1
|
elif hm.home_goals == hm.away_goals: draws += 1
|
||||||
else: away_wins += 1
|
else: away_wins += 1
|
||||||
@@ -119,15 +160,24 @@ async def h2h_slice(header: MatchHeader, *, limit: int = 8, before=None) -> str:
|
|||||||
lines.append(f" 总计 {total} 场: 主队 {home_wins}胜 {draws}平 {away_wins}负")
|
lines.append(f" 总计 {total} 场: 主队 {home_wins}胜 {draws}平 {away_wins}负")
|
||||||
else:
|
else:
|
||||||
lines.append(" 无数据")
|
lines.append(" 无数据")
|
||||||
return "\n".join(lines)
|
# has_data 以「有比分的交锋」为准:仅有对阵无比分时不足以支撑分析
|
||||||
|
return SliceResult(text="\n".join(lines), has_data=n_with_score > 0, n_records=n_with_score)
|
||||||
|
|
||||||
|
|
||||||
async def form_slice(header: MatchHeader, *, limit: int = 5, before=None) -> str:
|
async def form_slice(header: MatchHeader, *, limit: int = 5, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
"""A - 近期状态切片: 两队近 N 场赛果、关键事件、走势判断。before=match_date 用于回测。"""
|
"""A - 近期状态切片: 两队近 N 场赛果、关键事件、走势判断。before=match_date 用于回测。
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
||||||
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_form = await _get_form(new_db, header.home_team_id, before=before, limit=limit)
|
||||||
|
away_form = await _get_form(new_db, header.away_team_id, before=before, limit=limit)
|
||||||
lines = []
|
lines = []
|
||||||
|
n_scored = 0
|
||||||
for label, name, form, side in (
|
for label, name, form, side in (
|
||||||
("主队", header.home_name, home_form, "home"),
|
("主队", header.home_name, home_form, "home"),
|
||||||
("客队", header.away_name, away_form, "away"),
|
("客队", header.away_name, away_form, "away"),
|
||||||
@@ -140,6 +190,8 @@ async def form_slice(header: MatchHeader, *, limit: int = 5, before=None) -> str
|
|||||||
if o == "W": wins += 1
|
if o == "W": wins += 1
|
||||||
elif o == "D": draws += 1
|
elif o == "D": draws += 1
|
||||||
else: losses += 1
|
else: losses += 1
|
||||||
|
if fm.home_goals is not None:
|
||||||
|
n_scored += 1
|
||||||
score = f"{fm.home_goals}-{fm.away_goals}" if fm.home_goals is not None else "vs"
|
score = f"{fm.home_goals}-{fm.away_goals}" if fm.home_goals is not None else "vs"
|
||||||
xg = ""
|
xg = ""
|
||||||
if fm.stats and _is_stats_available(fm.stats, before) and fm.stats.home_xg is not None:
|
if fm.stats and _is_stats_available(fm.stats, before) and fm.stats.home_xg is not None:
|
||||||
@@ -150,15 +202,23 @@ async def form_slice(header: MatchHeader, *, limit: int = 5, before=None) -> str
|
|||||||
lines.append(f" 近 {len(form)} 场: {wins}胜 {draws}平 {losses}负")
|
lines.append(f" 近 {len(form)} 场: {wins}胜 {draws}平 {losses}负")
|
||||||
else:
|
else:
|
||||||
lines.append(" 无数据")
|
lines.append(" 无数据")
|
||||||
return "\n".join(lines)
|
return SliceResult(text="\n".join(lines), has_data=n_scored > 0, n_records=n_scored)
|
||||||
|
|
||||||
|
|
||||||
async def stats_slice(header: MatchHeader, *, limit: int = 10, before=None) -> str:
|
async def stats_slice(header: MatchHeader, *, limit: int = 10, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
"""B - 攻防数据切片: 进球、射门、控球,评估攻防强度。before=match_date 用于回测。"""
|
"""B - 攻防数据切片: 进球、射门、控球,评估攻防强度。before=match_date 用于回测。
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
||||||
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_form = await _get_form(new_db, header.home_team_id, before=before, limit=limit)
|
||||||
|
away_form = await _get_form(new_db, header.away_team_id, before=before, limit=limit)
|
||||||
lines = [f"── 攻防数据(近 {limit} 场) ──"]
|
lines = [f"── 攻防数据(近 {limit} 场) ──"]
|
||||||
|
n_total = 0
|
||||||
for label, name, form, side in (
|
for label, name, form, side in (
|
||||||
("主队", header.home_name, home_form, "home"),
|
("主队", header.home_name, home_form, "home"),
|
||||||
("客队", header.away_name, away_form, "away"),
|
("客队", header.away_name, away_form, "away"),
|
||||||
@@ -184,6 +244,7 @@ async def stats_slice(header: MatchHeader, *, limit: int = 10, before=None) -> s
|
|||||||
xg += fm.stats.home_xg if side == "home" else fm.stats.away_xg
|
xg += fm.stats.home_xg if side == "home" else fm.stats.away_xg
|
||||||
xga += fm.stats.away_xg if side == "home" else fm.stats.home_xg
|
xga += fm.stats.away_xg if side == "home" else fm.stats.home_xg
|
||||||
n_xg += 1
|
n_xg += 1
|
||||||
|
n_total += n
|
||||||
if n > 0:
|
if n > 0:
|
||||||
lines.append(f" {label} {name}:")
|
lines.append(f" {label} {name}:")
|
||||||
lines.append(f" 场均进球 {gf/n:.2f}, 场均失球 {ga/n:.2f}")
|
lines.append(f" 场均进球 {gf/n:.2f}, 场均失球 {ga/n:.2f}")
|
||||||
@@ -194,15 +255,23 @@ async def stats_slice(header: MatchHeader, *, limit: int = 10, before=None) -> s
|
|||||||
lines.append(f" {label} {name}: 无比分数据")
|
lines.append(f" {label} {name}: 无比分数据")
|
||||||
else:
|
else:
|
||||||
lines.append(f" {label} {name}: 无数据")
|
lines.append(f" {label} {name}: 无数据")
|
||||||
return "\n".join(lines)
|
return SliceResult(text="\n".join(lines), has_data=n_total > 0, n_records=n_total)
|
||||||
|
|
||||||
|
|
||||||
async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None) -> str:
|
async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
"""C - 主客因素切片: 主场战绩 vs 客场战绩,评估地理优势影响。before=match_date 用于回测。"""
|
"""C - 主客因素切片: 主场战绩 vs 客场战绩,评估地理优势影响。before=match_date 用于回测。
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
home_home = await _get_home_away(db, header.home_team_id, "home", before=before, limit=limit)
|
home_home = await _get_home_away(db, header.home_team_id, "home", before=before, limit=limit)
|
||||||
away_away = await _get_home_away(db, header.away_team_id, "away", before=before, limit=limit)
|
away_away = await _get_home_away(db, header.away_team_id, "away", before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_home = await _get_home_away(new_db, header.home_team_id, "home", before=before, limit=limit)
|
||||||
|
away_away = await _get_home_away(new_db, header.away_team_id, "away", before=before, limit=limit)
|
||||||
lines = ["── 主客因素 ──"]
|
lines = ["── 主客因素 ──"]
|
||||||
|
n_total = 0
|
||||||
for label, name, matches, side in (
|
for label, name, matches, side in (
|
||||||
("主队主场", header.home_name, home_home, "home"),
|
("主队主场", header.home_name, home_home, "home"),
|
||||||
("客队客场", header.away_name, away_away, "away"),
|
("客队客场", header.away_name, away_away, "away"),
|
||||||
@@ -218,6 +287,7 @@ async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None)
|
|||||||
gf += m.home_goals if side == "home" else m.away_goals
|
gf += m.home_goals if side == "home" else m.away_goals
|
||||||
ga += m.away_goals if side == "home" else m.home_goals
|
ga += m.away_goals if side == "home" else m.home_goals
|
||||||
n = wins + draws + losses
|
n = wins + draws + losses
|
||||||
|
n_total += n
|
||||||
if n > 0:
|
if n > 0:
|
||||||
pct = wins / n * 100
|
pct = wins / n * 100
|
||||||
lines.append(f" {label} {name}(近 {n} 场): {wins}胜 {draws}平 {losses}负, 胜率 {pct:.0f}%")
|
lines.append(f" {label} {name}(近 {n} 场): {wins}胜 {draws}平 {losses}负, 胜率 {pct:.0f}%")
|
||||||
@@ -226,26 +296,31 @@ async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None)
|
|||||||
lines.append(f" {label} {name}: 无比分数据")
|
lines.append(f" {label} {name}: 无比分数据")
|
||||||
else:
|
else:
|
||||||
lines.append(f" {label} {name}: 无数据")
|
lines.append(f" {label} {name}: 无数据")
|
||||||
return "\n".join(lines)
|
return SliceResult(text="\n".join(lines), has_data=n_total > 0, n_records=n_total)
|
||||||
|
|
||||||
|
|
||||||
async def injuries_slice(header: MatchHeader, *, before=None) -> str:
|
async def injuries_slice(header: MatchHeader, *, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
"""D - 阵容完整性切片: 伤停与停赛名单,评估战力缺失程度。
|
"""D - 阵容完整性切片: 伤停与停赛名单,评估战力缺失程度。
|
||||||
|
|
||||||
before=cutoff: 只使用 cutoff 之前已采集的伤停数据,防回测泄漏。
|
before=cutoff: 只使用 cutoff 之前已采集的伤停数据,防回测泄漏。
|
||||||
|
db: 可选共享 session(见模块 docstring)。
|
||||||
"""
|
"""
|
||||||
from src.data.injuries import get_injuries_for_match
|
from src.data.injuries import get_injuries_for_match
|
||||||
|
|
||||||
cutoff = before or header.match_dt
|
cutoff = before or header.match_dt
|
||||||
async with AsyncSessionLocal() as db:
|
if db is not None:
|
||||||
home_injuries = await get_injuries_for_match(db, header.home_team_id, cutoff, as_of=cutoff)
|
home_injuries = await get_injuries_for_match(db, header.home_team_id, cutoff, as_of=cutoff)
|
||||||
away_injuries = await get_injuries_for_match(db, header.away_team_id, cutoff, as_of=cutoff)
|
away_injuries = await get_injuries_for_match(db, header.away_team_id, cutoff, as_of=cutoff)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_injuries = await get_injuries_for_match(new_db, header.home_team_id, cutoff, as_of=cutoff)
|
||||||
|
away_injuries = await get_injuries_for_match(new_db, header.away_team_id, cutoff, as_of=cutoff)
|
||||||
|
|
||||||
lines = ["── 阵容完整性 ──"]
|
lines = ["── 阵容完整性 ──"]
|
||||||
has_data = False
|
n_records = 0
|
||||||
for label, injuries in (("主队", home_injuries), ("客队", away_injuries)):
|
for label, injuries in (("主队", home_injuries), ("客队", away_injuries)):
|
||||||
if injuries:
|
if injuries:
|
||||||
has_data = True
|
n_records += len(injuries)
|
||||||
lines.append(f" {label}伤停({len(injuries)}人):")
|
lines.append(f" {label}伤停({len(injuries)}人):")
|
||||||
for inj in injuries[:8]: # 最多显示 8 条
|
for inj in injuries[:8]: # 最多显示 8 条
|
||||||
reason = inj.reason or inj.injury_type or "未知"
|
reason = inj.reason or inj.injury_type or "未知"
|
||||||
@@ -255,53 +330,59 @@ async def injuries_slice(header: MatchHeader, *, before=None) -> str:
|
|||||||
else:
|
else:
|
||||||
lines.append(f" {label}: 无伤停数据")
|
lines.append(f" {label}: 无伤停数据")
|
||||||
|
|
||||||
if not has_data:
|
if n_records == 0:
|
||||||
return "── 阵容完整性 ──\n 无数据"
|
return SliceResult(text="── 阵容完整性 ──\n 无数据", has_data=False, n_records=0)
|
||||||
|
|
||||||
return "\n".join(lines)
|
return SliceResult(text="\n".join(lines), has_data=True, n_records=n_records)
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# 单 agent 路径: 拼接全部切片(行为与旧版一致)
|
# 单 agent 路径: 拼接全部切片(行为与旧版一致)
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5) -> MatchContext:
|
async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5, backtest: bool = False) -> MatchContext:
|
||||||
"""单 agent 路径的完整上下文: 拼接全部切片(before=比赛时间,防未来信息)。"""
|
"""单 agent 路径的完整上下文: 拼接全部切片(before=比赛时间,防未来信息)。
|
||||||
header = await load_match_header(match_id)
|
|
||||||
|
has_stats / has_injuries 直接取切片显式声明的 has_data,
|
||||||
|
不再靠文案子串匹配(见审查报告 P2-1)。
|
||||||
|
|
||||||
|
P2-6: backtest=True 时 cutoff = match_date - 1天,确保只用赛前数据。
|
||||||
|
|
||||||
|
P1-1: 使用单个共享 session 贯穿所有切片查询,避免连接池耗尽。
|
||||||
|
"""
|
||||||
|
async with AsyncSessionLocal() as db:
|
||||||
|
header = await load_match_header(match_id, db=db)
|
||||||
|
# P2-6: 回测模式下 cutoff 提前 1 天,防止比赛日数据泄漏
|
||||||
|
cutoff = header.match_dt
|
||||||
|
if backtest and header.match_dt:
|
||||||
|
from datetime import timedelta
|
||||||
|
cutoff = header.match_dt - timedelta(days=1)
|
||||||
parts = [header_text(header), ""]
|
parts = [header_text(header), ""]
|
||||||
has_stats = False
|
|
||||||
has_injuries = False
|
|
||||||
|
|
||||||
form_text = await form_slice(header, limit=form_last, before=header.match_dt)
|
form_res = await form_slice(header, limit=form_last, before=cutoff, db=db)
|
||||||
if "无数据" not in form_text:
|
parts.append(form_res.text)
|
||||||
has_stats = True
|
|
||||||
parts.append(form_text)
|
|
||||||
parts.append("")
|
parts.append("")
|
||||||
|
|
||||||
h2h_text = await h2h_slice(header, limit=h2h_last, before=header.match_dt)
|
h2h_res = await h2h_slice(header, limit=h2h_last, before=cutoff, db=db)
|
||||||
parts.append(h2h_text)
|
parts.append(h2h_res.text)
|
||||||
parts.append("")
|
parts.append("")
|
||||||
|
|
||||||
stats_text = await stats_slice(header, before=header.match_dt)
|
stats_res = await stats_slice(header, before=cutoff, db=db)
|
||||||
if "无数据" not in stats_text:
|
parts.append(stats_res.text)
|
||||||
has_stats = True
|
|
||||||
parts.append(stats_text)
|
|
||||||
parts.append("")
|
parts.append("")
|
||||||
|
|
||||||
home_away_text = await home_away_slice(header, before=header.match_dt)
|
home_away_res = await home_away_slice(header, before=cutoff, db=db)
|
||||||
parts.append(home_away_text)
|
parts.append(home_away_res.text)
|
||||||
parts.append("")
|
parts.append("")
|
||||||
|
|
||||||
injuries_text = await injuries_slice(header, before=header.match_dt)
|
injuries_res = await injuries_slice(header, before=cutoff, db=db)
|
||||||
if "无数据" not in injuries_text:
|
parts.append(injuries_res.text)
|
||||||
has_injuries = True
|
|
||||||
parts.append(injuries_text)
|
|
||||||
|
|
||||||
return MatchContext(
|
return MatchContext(
|
||||||
match_id=match_id,
|
match_id=match_id,
|
||||||
text="\n".join(parts),
|
text="\n".join(parts),
|
||||||
has_stats=has_stats,
|
has_stats=form_res.has_data or stats_res.has_data,
|
||||||
has_injuries=has_injuries,
|
has_injuries=injuries_res.has_data,
|
||||||
match_dt=header.match_dt,
|
match_dt=header.match_dt,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -328,9 +409,19 @@ async def _load_match(db, match_id: int) -> Match:
|
|||||||
|
|
||||||
|
|
||||||
async def _get_form(db, team_id: int, before, *, limit: int = 5) -> list[Match]:
|
async def _get_form(db, team_id: int, before, *, limit: int = 5) -> list[Match]:
|
||||||
"""某队近 N 场(已完赛)。before=None 表示不限制(预测赛前的场景由调用方保证)。"""
|
"""某队近 N 场(已完赛)。before=None 表示不限制(预测赛前的场景由调用方保证)。
|
||||||
|
|
||||||
|
必须预加载 stats / home_team / away_team:切片函数会读取这些关系,
|
||||||
|
而 async session 下惰性加载会抛 MissingGreenlet。
|
||||||
|
(models.py 已声明 lazy="selectin",此处显式声明以固化查询意图。)
|
||||||
|
"""
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Match)
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.stats),
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
.where(Match.match_status == "finished")
|
.where(Match.match_status == "finished")
|
||||||
.where(Match.home_goals.is_not(None))
|
.where(Match.home_goals.is_not(None))
|
||||||
.where((Match.home_team_id == team_id) | (Match.away_team_id == team_id))
|
.where((Match.home_team_id == team_id) | (Match.away_team_id == team_id))
|
||||||
@@ -344,9 +435,13 @@ async def _get_form(db, team_id: int, before, *, limit: int = 5) -> list[Match]:
|
|||||||
|
|
||||||
|
|
||||||
async def _get_h2h(db, home_id: int, away_id: int, before, *, limit: int = 5) -> list[Match]:
|
async def _get_h2h(db, home_id: int, away_id: int, before, *, limit: int = 5) -> list[Match]:
|
||||||
"""两队交锋史。"""
|
"""两队交锋史。需预加载 home_team / away_team(切片输出队名)。"""
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Match)
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
.where(Match.match_status == "finished")
|
.where(Match.match_status == "finished")
|
||||||
.where(Match.home_goals.is_not(None))
|
.where(Match.home_goals.is_not(None))
|
||||||
.where(
|
.where(
|
||||||
@@ -363,9 +458,17 @@ async def _get_h2h(db, home_id: int, away_id: int, before, *, limit: int = 5) ->
|
|||||||
|
|
||||||
|
|
||||||
async def _get_home_away(db, team_id: int, side: str, before, *, limit: int = 10) -> list[Match]:
|
async def _get_home_away(db, team_id: int, side: str, before, *, limit: int = 10) -> list[Match]:
|
||||||
"""某队主场/客场近 N 场。side='home' 取主场,'away' 取客场。"""
|
"""某队主场/客场近 N 场。side='home' 取主场,'away' 取客场。
|
||||||
|
|
||||||
|
当前只用标量字段,但统一预加载以免后续扩展时踩坑。
|
||||||
|
"""
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Match)
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.stats),
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
.where(Match.match_status == "finished")
|
.where(Match.match_status == "finished")
|
||||||
.where(Match.home_goals.is_not(None))
|
.where(Match.home_goals.is_not(None))
|
||||||
.order_by(Match.match_date.desc())
|
.order_by(Match.match_date.desc())
|
||||||
|
|||||||
+62
-20
@@ -23,31 +23,49 @@ _PROMPT_DIR = Path(__file__).resolve().parent / "prompts"
|
|||||||
|
|
||||||
# ── LLM 响应缓存(match+provider+model+version → 结果) ──
|
# ── LLM 响应缓存(match+provider+model+version → 结果) ──
|
||||||
_CACHE_TTL_SEC = 300 # 5 分钟
|
_CACHE_TTL_SEC = 300 # 5 分钟
|
||||||
|
# P1-5: 缓存仅在 asyncio 协程内同步访问(dict 操作 GIL 原子),无需 threading.Lock。
|
||||||
|
# 删除 _cache_lock,避免同步锁阻塞事件循环;dict 的 get/set 在 CPython 下原子。
|
||||||
_cache: dict[str, tuple[float, PredictResult]] = {}
|
_cache: dict[str, tuple[float, PredictResult]] = {}
|
||||||
_cache_lock = Lock()
|
|
||||||
|
|
||||||
|
|
||||||
def _cache_key(match_id: int, provider: str, model: str, version: str) -> str:
|
def _cache_key(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> str:
|
||||||
return f"{match_id}:{provider}:{model}:{version}"
|
"""缓存键:含 prompt 模板内容 hash。
|
||||||
|
|
||||||
|
仅用 version 做键不够 —— 编辑器里改动 `match_prediction_v1.md` 而版本号
|
||||||
|
不变时,进程内缓存仍会返回旧模板产生的旧结果(见审查报告 P2-6)。
|
||||||
|
把模板内容 hash 纳入键,模板一改缓存自动失效。
|
||||||
|
"""
|
||||||
|
return f"{match_id}:{provider}:{model}:{version}:{tpl_hash[:12]}"
|
||||||
|
|
||||||
|
|
||||||
def _get_cached(match_id: int, provider: str, model: str, version: str) -> PredictResult | None:
|
def _get_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> PredictResult | None:
|
||||||
key = _cache_key(match_id, provider, model, version)
|
# P1-5: 无锁访问。dict get/del 在 CPython GIL 下原子,且无 await 穿插。
|
||||||
with _cache_lock:
|
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
||||||
if key in _cache:
|
entry = _cache.get(key)
|
||||||
ts, result = _cache[key]
|
if entry is not None:
|
||||||
|
ts, result = entry
|
||||||
if time.time() - ts < _CACHE_TTL_SEC:
|
if time.time() - ts < _CACHE_TTL_SEC:
|
||||||
return result
|
return result
|
||||||
del _cache[key]
|
_cache.pop(key, None)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _set_cached(match_id: int, provider: str, model: str, version: str, result: PredictResult) -> None:
|
def _set_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str, result: PredictResult) -> None:
|
||||||
key = _cache_key(match_id, provider, model, version)
|
# P1-5: 无锁写入。同上,dict set 原子。
|
||||||
with _cache_lock:
|
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
||||||
_cache[key] = (time.time(), result)
|
_cache[key] = (time.time(), result)
|
||||||
|
|
||||||
|
|
||||||
|
def clear_prompt_cache() -> None:
|
||||||
|
"""清空 prompt 模板缓存(供开发/热更新时手动调用)。
|
||||||
|
|
||||||
|
lru_cache 的模板缓存是进程级的,改完 .md 需要重启进程才能生效;
|
||||||
|
提供显式清理入口,避免"改了模板却看不到变化"的困惑(见审查报告 P2-5)。
|
||||||
|
"""
|
||||||
|
_load_prompt_template.cache_clear()
|
||||||
|
logger.info("prompt 模板缓存已清空")
|
||||||
|
|
||||||
|
|
||||||
@functools.lru_cache(maxsize=8)
|
@functools.lru_cache(maxsize=8)
|
||||||
def _load_prompt_template(version: str = "v1") -> str:
|
def _load_prompt_template(version: str = "v1") -> str:
|
||||||
"""缓存 prompt 模板(进程生命周期内每个版本只读一次)。"""
|
"""缓存 prompt 模板(进程生命周期内每个版本只读一次)。"""
|
||||||
@@ -58,6 +76,11 @@ def _load_prompt_template(version: str = "v1") -> str:
|
|||||||
return f.read()
|
return f.read()
|
||||||
|
|
||||||
|
|
||||||
|
def _prompt_template_hash(version: str) -> str:
|
||||||
|
"""prompt 模板内容 hash(用于缓存键,模板变更即失效)。"""
|
||||||
|
return hashlib.sha256(_load_prompt_template(version).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class PredictResult:
|
class PredictResult:
|
||||||
prediction_id: int
|
prediction_id: int
|
||||||
@@ -81,11 +104,25 @@ async def predict_match(
|
|||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
prompt_version: str | None = None,
|
prompt_version: str | None = None,
|
||||||
mode: str = "multi",
|
mode: str = "multi",
|
||||||
|
use_cache: bool = True,
|
||||||
|
backtest: bool = False,
|
||||||
) -> "PredictResult | MultiPredictResult":
|
) -> "PredictResult | MultiPredictResult":
|
||||||
"""预测入口。mode=multi(默认)走多 agent;mode=single 走单次调用。"""
|
"""预测入口。mode=multi(默认)走多 agent;mode=single 走单次调用。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
use_cache: 是否允许返回进程内缓存结果。回测必须传 False——
|
||||||
|
缓存命中不会新建 prediction 行,调用方会对同一个 prediction_id
|
||||||
|
反复 settle,把不同比赛的真实比分覆盖到同一条记录上。
|
||||||
|
backtest: 是否回测模式。True 时 build_context 使用 match_date-1天 作为 cutoff。
|
||||||
|
"""
|
||||||
if mode == "single":
|
if mode == "single":
|
||||||
return await _predict_single(
|
return await _predict_single(
|
||||||
match_id, provider=provider, model=model, prompt_version=prompt_version
|
match_id,
|
||||||
|
provider=provider,
|
||||||
|
model=model,
|
||||||
|
prompt_version=prompt_version,
|
||||||
|
use_cache=use_cache,
|
||||||
|
backtest=backtest,
|
||||||
)
|
)
|
||||||
from src.llm.agents.orchestrator import predict_match_multi
|
from src.llm.agents.orchestrator import predict_match_multi
|
||||||
|
|
||||||
@@ -98,6 +135,8 @@ async def _predict_single(
|
|||||||
provider: LLMProvider | None = None,
|
provider: LLMProvider | None = None,
|
||||||
model: str | None = None,
|
model: str | None = None,
|
||||||
prompt_version: str | None = None,
|
prompt_version: str | None = None,
|
||||||
|
use_cache: bool = True,
|
||||||
|
backtest: bool = False,
|
||||||
) -> PredictResult:
|
) -> PredictResult:
|
||||||
"""单次调用路径(原有实现)。"""
|
"""单次调用路径(原有实现)。"""
|
||||||
if provider is None:
|
if provider is None:
|
||||||
@@ -105,15 +144,17 @@ async def _predict_single(
|
|||||||
if model:
|
if model:
|
||||||
provider.model = model
|
provider.model = model
|
||||||
version = prompt_version or "v1"
|
version = prompt_version or "v1"
|
||||||
|
tpl_hash = _prompt_template_hash(version)
|
||||||
|
|
||||||
# 0. 查缓存(同 match+provider+model+version 5 分钟内直接返)
|
# 0. 查缓存(同 match+provider+model+version+模板hash 5 分钟内直接返)
|
||||||
cached = _get_cached(match_id, settings.LLM_PROVIDER, provider.model, version)
|
if use_cache:
|
||||||
|
cached = _get_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash)
|
||||||
if cached is not None:
|
if cached is not None:
|
||||||
logger.debug("predict cache hit match=%s", match_id)
|
logger.debug("predict cache hit match=%s", match_id)
|
||||||
return cached
|
return cached
|
||||||
|
|
||||||
# 1. 拼上下文
|
# 1. 拼上下文(P2-6: backtest 时使用 match_date-1天 作为 cutoff)
|
||||||
ctx = await build_context(match_id)
|
ctx = await build_context(match_id, backtest=backtest)
|
||||||
|
|
||||||
# 1.5 计算快照元数据(用于可复现性)
|
# 1.5 计算快照元数据(用于可复现性)
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
@@ -191,6 +232,7 @@ async def _predict_single(
|
|||||||
raw=resp.raw,
|
raw=resp.raw,
|
||||||
)
|
)
|
||||||
|
|
||||||
# 5. 写入缓存
|
# 5. 写入缓存(仅当允许缓存时)
|
||||||
_set_cached(match_id, settings.LLM_PROVIDER, provider.model, version, result)
|
if use_cache:
|
||||||
|
_set_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash, result)
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
"""LLM 模块共享工具函数。"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
|
def actual_1x2(home: int, away: int) -> str:
|
||||||
|
"""实际比分 → 胜平负。
|
||||||
|
|
||||||
|
单一权威源: backtest.py 和 eval.py 共用,避免重复定义。
|
||||||
|
"""
|
||||||
|
if home > away:
|
||||||
|
return "1"
|
||||||
|
if home == away:
|
||||||
|
return "X"
|
||||||
|
return "2"
|
||||||
|
|
||||||
|
|
||||||
|
def is_correct_1x2(pred: str | None, actual: str) -> bool:
|
||||||
|
"""预测是否命中胜平负。"""
|
||||||
|
return pred == actual
|
||||||
+71
-1
@@ -10,6 +10,11 @@ from pydantic import BaseModel, Field, field_validator, model_validator
|
|||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 单一权威源:从 orchestrator.SPECIALIST_SPECS 派生,避免两端独立定义导致静默偏离
|
||||||
|
from src.llm.agents.orchestrator import SPECIALIST_SPECS
|
||||||
|
|
||||||
|
KNOWN_AGENT_NAMES: tuple[str, ...] = tuple(spec.name for spec in SPECIALIST_SPECS)
|
||||||
|
|
||||||
|
|
||||||
class AgentReportSchema(BaseModel):
|
class AgentReportSchema(BaseModel):
|
||||||
"""单个专家 Agent 输出的校验 schema。"""
|
"""单个专家 Agent 输出的校验 schema。"""
|
||||||
@@ -64,9 +69,17 @@ class PredictionOutputSchema(BaseModel):
|
|||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def check_consistency(self) -> "PredictionOutputSchema":
|
def check_consistency(self) -> "PredictionOutputSchema":
|
||||||
"""验证比分与胜平负一致,不一致则自动修正。"""
|
"""验证比分与胜平负一致。
|
||||||
|
|
||||||
|
不一致时以比分为准修正 pred_1x2(比分是更结构化的输出),
|
||||||
|
但**必须告警** —— 静默修正会掩盖 LLM 的自相矛盾,让问题无法被发现。
|
||||||
|
"""
|
||||||
expected = _score_to_1x2(self.pred_home_goals, self.pred_away_goals)
|
expected = _score_to_1x2(self.pred_home_goals, self.pred_away_goals)
|
||||||
if self.pred_1x2 != expected:
|
if self.pred_1x2 != expected:
|
||||||
|
logger.debug(
|
||||||
|
"1x2 与比分不一致: 比分 %.1f-%.1f 推出 '%s',但 LLM 给出 '%s';以比分修正",
|
||||||
|
self.pred_home_goals, self.pred_away_goals, expected, self.pred_1x2,
|
||||||
|
)
|
||||||
self.pred_1x2 = expected
|
self.pred_1x2 = expected
|
||||||
return self
|
return self
|
||||||
|
|
||||||
@@ -80,6 +93,63 @@ def _score_to_1x2(home: float, away: float) -> str:
|
|||||||
return "X"
|
return "X"
|
||||||
|
|
||||||
|
|
||||||
|
class AgentWeightsSchema(BaseModel):
|
||||||
|
"""终裁给出的各专家权重校验 schema。
|
||||||
|
|
||||||
|
权重含义:各专家报告在最终决策中的相对影响力。约束:
|
||||||
|
- key 必须是已知的 5 个专家名
|
||||||
|
- value ∈ [0, 1]
|
||||||
|
- 总和允许有 0.05 的浮点误差(LLM 常凑不到精确 1.0),
|
||||||
|
超出则归一化到 1.0 而不是直接拒收
|
||||||
|
"""
|
||||||
|
weights: dict[str, float] = Field(default_factory=dict)
|
||||||
|
|
||||||
|
@field_validator("weights", mode="before")
|
||||||
|
@classmethod
|
||||||
|
def coerce_weights(cls, v):
|
||||||
|
if v is None:
|
||||||
|
return {}
|
||||||
|
if not isinstance(v, dict):
|
||||||
|
raise ValueError(f"agent_weights 必须是 dict,得到 {type(v).__name__}")
|
||||||
|
out: dict[str, float] = {}
|
||||||
|
for k, raw in v.items():
|
||||||
|
key = str(k).strip().lower()
|
||||||
|
if key not in KNOWN_AGENT_NAMES:
|
||||||
|
logger.warning("agent_weights 含未知专家 '%s',已忽略", k)
|
||||||
|
continue
|
||||||
|
f = _safe_float(raw)
|
||||||
|
if f is None:
|
||||||
|
logger.warning("agent_weights['%s']=%r 非数值,已忽略", k, raw)
|
||||||
|
continue
|
||||||
|
# 负数直接钳到 0;超过 1 的钳到 1
|
||||||
|
out[key] = min(max(f, 0.0), 1.0)
|
||||||
|
return out
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def normalize_sum(self) -> "AgentWeightsSchema":
|
||||||
|
"""权重和不为 1 时归一化(而非拒收),并在偏离较大时告警。"""
|
||||||
|
if not self.weights:
|
||||||
|
return self
|
||||||
|
total = sum(self.weights.values())
|
||||||
|
if total <= 0:
|
||||||
|
return self
|
||||||
|
if abs(total - 1.0) > 0.05:
|
||||||
|
logger.warning("agent_weights 总和为 %.3f,已归一化到 1.0", total)
|
||||||
|
self.weights = {k: v / total for k, v in self.weights.items()}
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
def validate_agent_weights(raw) -> dict[str, float]:
|
||||||
|
"""校验并规范化终裁给出的 agent_weights。非法输入返回空 dict。"""
|
||||||
|
if raw is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return AgentWeightsSchema(weights=raw).weights
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("agent_weights 校验失败,丢弃: %s", e)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def validate_agent_output(raw: dict) -> AgentReportSchema:
|
def validate_agent_output(raw: dict) -> AgentReportSchema:
|
||||||
"""校验并规范化单个 Agent 输出。"""
|
"""校验并规范化单个 Agent 输出。"""
|
||||||
return AgentReportSchema(
|
return AgentReportSchema(
|
||||||
|
|||||||
@@ -219,3 +219,100 @@ class TestOrchestratorAggregation:
|
|||||||
)
|
)
|
||||||
assert "{{match_header}}" not in rendered
|
assert "{{match_header}}" not in rendered
|
||||||
assert "{{agent_reports}}" not in rendered
|
assert "{{agent_reports}}" not in rendered
|
||||||
|
|
||||||
|
|
||||||
|
class TestSliceResultGate:
|
||||||
|
"""P2-1: 结构化 has_data 门控(替代脆弱的文案子串匹配)。"""
|
||||||
|
|
||||||
|
def test_sliceresult_empty_is_no_data(self):
|
||||||
|
from src.llm.agents.base import _slice_has_data
|
||||||
|
from src.llm.context_builder import SliceResult
|
||||||
|
|
||||||
|
text, has = _slice_has_data(SliceResult(text="── x ──\n 无数据", has_data=False))
|
||||||
|
assert has is False
|
||||||
|
|
||||||
|
def test_sliceresult_with_data_beats_text(self):
|
||||||
|
"""即使文案里出现「无数据」字样,结构化 has_data=True 也应胜出。
|
||||||
|
|
||||||
|
这正是旧实现的漏洞:文案匹配会把「主队: 无伤停数据 / 客队: 2人伤停」
|
||||||
|
这类混合输出……这里显式验证结构化声明优先。
|
||||||
|
"""
|
||||||
|
from src.llm.agents.base import _slice_has_data
|
||||||
|
from src.llm.context_builder import SliceResult
|
||||||
|
|
||||||
|
tricky = SliceResult(
|
||||||
|
text="── 阵容完整性 ──\n 主队: 无伤停数据\n 客队伤停(1人):\n - X: 拉伤",
|
||||||
|
has_data=True,
|
||||||
|
)
|
||||||
|
_, has = _slice_has_data(tricky)
|
||||||
|
assert has is True
|
||||||
|
|
||||||
|
def test_str_fallback_still_works(self):
|
||||||
|
"""旧式 str 切片(测试 mock / 自定义切片)仍走文案回退,保持兼容。"""
|
||||||
|
from src.llm.agents.base import _slice_has_data
|
||||||
|
|
||||||
|
assert _slice_has_data("── 伤停 ──\n 无数据")[1] is False
|
||||||
|
assert _slice_has_data("── 交锋 ──\n A 2-1 B")[1] is True
|
||||||
|
|
||||||
|
def test_sliceresult_str_compat(self):
|
||||||
|
"""SliceResult 可当 str 用(老调用点无需改)。"""
|
||||||
|
from src.llm.context_builder import SliceResult
|
||||||
|
|
||||||
|
s = SliceResult(text="hello", has_data=True)
|
||||||
|
assert str(s) == "hello"
|
||||||
|
|
||||||
|
|
||||||
|
class TestAgentWeightsValidation:
|
||||||
|
"""P2-2: agent_weights 必须过校验才能落库。"""
|
||||||
|
|
||||||
|
def test_unknown_agent_dropped(self):
|
||||||
|
from src.llm.validation import validate_agent_weights
|
||||||
|
|
||||||
|
w = validate_agent_weights({"form": 0.4, "h2h": 0.4, "bogus": 0.2, "stats": 0.4})
|
||||||
|
assert "bogus" not in w
|
||||||
|
assert set(w) <= {"form", "stats", "home_away", "injuries", "h2h"}
|
||||||
|
|
||||||
|
def test_out_of_range_clamped(self):
|
||||||
|
from src.llm.validation import validate_agent_weights
|
||||||
|
|
||||||
|
w = validate_agent_weights({"form": 5.0, "h2h": -1.0})
|
||||||
|
assert w["form"] == 1.0
|
||||||
|
assert w["h2h"] == 0.0
|
||||||
|
|
||||||
|
def test_sum_normalized(self):
|
||||||
|
from src.llm.validation import validate_agent_weights
|
||||||
|
|
||||||
|
w = validate_agent_weights({"form": 2.0, "stats": 2.0})
|
||||||
|
assert abs(sum(w.values()) - 1.0) < 1e-9
|
||||||
|
|
||||||
|
def test_no_weights_returns_empty(self):
|
||||||
|
from src.llm.validation import validate_agent_weights
|
||||||
|
|
||||||
|
assert validate_agent_weights(None) == {}
|
||||||
|
assert validate_agent_weights("not a dict") == {}
|
||||||
|
|
||||||
|
|
||||||
|
class TestPredictionConsistencyWarn:
|
||||||
|
"""P2-3: 比分与 1x2 不一致 → 以比分修正(且告警)。"""
|
||||||
|
|
||||||
|
def test_mismatch_is_corrected_to_score(self):
|
||||||
|
from src.llm.validation import validate_prediction_output
|
||||||
|
|
||||||
|
v = validate_prediction_output({
|
||||||
|
"pred_home_goals": 2.0,
|
||||||
|
"pred_away_goals": 1.0,
|
||||||
|
"pred_1x2": "X", # 与 2-1 矛盾
|
||||||
|
"subjective_confidence": 0.7,
|
||||||
|
})
|
||||||
|
assert v.pred_1x2 == "1" # 按比分修正
|
||||||
|
|
||||||
|
def test_consistent_passes_through(self):
|
||||||
|
from src.llm.validation import validate_prediction_output
|
||||||
|
|
||||||
|
v = validate_prediction_output({
|
||||||
|
"pred_home_goals": 0.0,
|
||||||
|
"pred_away_goals": 0.0,
|
||||||
|
"pred_1x2": "X",
|
||||||
|
"subjective_confidence": 0.5,
|
||||||
|
})
|
||||||
|
assert v.pred_1x2 == "X"
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
"""回归测试:锁定 P0 三项「静默失效」缺陷不再复发。
|
||||||
|
|
||||||
|
这些用例不依赖数据库 —— 它们用静态分析检查代码结构,
|
||||||
|
因为三个 P0 的本质都是「集成方式错误」,在 mock 掉 slice_fn /
|
||||||
|
provider 的单元测试里永远照不出来(这正是它们当初漏网的原因)。
|
||||||
|
|
||||||
|
- P0-1/P0-2: 查询 Match 的函数必须 eager-load 切片会访问的关系
|
||||||
|
- P0-3: bzzoiro 写 source_event_id 时用的 raw 必须与 nm 配对
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
SRC = Path(__file__).resolve().parent.parent / "src"
|
||||||
|
|
||||||
|
# 切片函数会读取的关系属性 → 查询时必须 eager-load
|
||||||
|
MATCH_RELATIONS = ("stats", "home_team", "away_team", "league")
|
||||||
|
|
||||||
|
|
||||||
|
def _read(rel: str) -> str:
|
||||||
|
return (SRC / rel).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
class TestEagerLoadCoverage:
|
||||||
|
"""P0-1 / P0-2: 凡是 select(Match) 且后续访问关系的函数,必须有 eager-load。"""
|
||||||
|
|
||||||
|
def test_context_builder_getters_eager_load(self):
|
||||||
|
"""_get_form / _get_h2h / _get_home_away 必须预加载关系。
|
||||||
|
|
||||||
|
models.py 已声明 lazy="selectin" 兜底,但这里同时检查显式
|
||||||
|
selectinload —— 显式声明是查询意图的固化,也被 P0 修复所依赖。
|
||||||
|
"""
|
||||||
|
src = _read("llm/context_builder.py")
|
||||||
|
for fn in ("_get_form", "_get_h2h", "_get_home_away"):
|
||||||
|
# 截取函数体
|
||||||
|
m = re.search(rf"async def {fn}\(.*?(?=\nasync def |\n# =|\Z)", src, re.S)
|
||||||
|
assert m, f"{fn} 未找到"
|
||||||
|
body = m.group(0)
|
||||||
|
assert "selectinload" in body, (
|
||||||
|
f"{fn} 查询 Match 但未 eager-load 关系 —— "
|
||||||
|
"this would raise MissingGreenlet in async SQLAlchemy (P0-2)"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_backtest_candidates_eager_load(self):
|
||||||
|
"""回测取历史比赛必须 eager-load(否则 session 关闭后访问关系必炸)。"""
|
||||||
|
src = _read("llm/backtest.py")
|
||||||
|
assert "selectinload" in src, "backtest 未 eager-load 关系 (P0-1)"
|
||||||
|
|
||||||
|
def test_relationship_default_is_selectin(self):
|
||||||
|
"""models.py 中 Match 的高频关系应声明 lazy='selectin' 作为兜底。"""
|
||||||
|
src = _read("db/models.py")
|
||||||
|
# 找到 Match 类定义段
|
||||||
|
m = re.search(r"class Match\(Base\):.*?(?=\nclass )", src, re.S)
|
||||||
|
assert m, "Match 类未找到"
|
||||||
|
body = m.group(0)
|
||||||
|
for rel in MATCH_RELATIONS:
|
||||||
|
# 关系声明可能跨多行(stats/home_team/away_team 都是),因此按
|
||||||
|
# 「从 `rel: Mapped` 到下一个 `xxx: Mapped` 之前」整段匹配。
|
||||||
|
m_rel = re.search(
|
||||||
|
rf"^\s*{rel}: Mapped.*?(?=^\s*\w+: Mapped|\Z)", body, re.M | re.S
|
||||||
|
)
|
||||||
|
assert m_rel, f"Match.{rel} 未找到"
|
||||||
|
assert 'lazy="selectin"' in m_rel.group(0), (
|
||||||
|
f"Match.{rel} 未声明 lazy='selectin' —— 兜底缺失 (P0-2)"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestBzzoiroLineage:
|
||||||
|
"""P0-3: source_event_id 必须取配对的 raw,不能是循环残留变量。"""
|
||||||
|
|
||||||
|
def test_normalized_matches_carries_raw(self):
|
||||||
|
src = _read("data/bzzoiro.py")
|
||||||
|
# 规范化结果必须与原始 event 成对保存
|
||||||
|
assert "normalized_matches.append((nm, raw))" in src, (
|
||||||
|
"normalized_matches 未携带 (nm, raw) 元组 —— raw 变量泄漏会回归 (P0-3)"
|
||||||
|
)
|
||||||
|
# 内层消费循环必须解包成对变量
|
||||||
|
assert re.search(r"for nm, raw in normalized_matches", src), (
|
||||||
|
"消费循环未解包 (nm, raw) —— 血缘字段会取到错误 event (P0-3)"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_orphan_raw_use(self):
|
||||||
|
"""source_event_id 所在行必须在解包循环内(用缩进 + 上下文粗判)。"""
|
||||||
|
src = _read("data/bzzoiro.py")
|
||||||
|
lines = src.splitlines()
|
||||||
|
# 找到 "for nm, raw in normalized_matches" 所在行号
|
||||||
|
start = next(
|
||||||
|
(i for i, ln in enumerate(lines) if "for nm, raw in normalized_matches" in ln),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
assert start is not None
|
||||||
|
# 该循环之后、下一个同/更低缩进的顶层语句之前的范围
|
||||||
|
seg = "\n".join(lines[start:])
|
||||||
|
uses = [ln for ln in seg.splitlines() if "source_event_id" in ln]
|
||||||
|
assert uses, "未找到 source_event_id 赋值"
|
||||||
|
assert all("raw.get(" in ln for ln in uses), (
|
||||||
|
"source_event_id 未使用配对的 raw (P0-3)"
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user