fix(P0-02): 积分榜改为追加快照(append-only) + available_at cutoff

去掉 uq_standings_league_season_team,改为 (league, season, team, available_at) 唯一;
每次采集 INSERT 新行(available_at=now),ON CONFLICT DO NOTHING,不覆盖旧行。

standings_slice(before):DISTINCT ON (team_id) WHERE available_at<=cutoff
ORDER available_at DESC;before=None → cutoff=now()。
公开 list_standings 取每队最新可用快照(子查询 max available_at)。

迁移 0023 + 切片/路由/docs 同步;测试 test_p0_standings_cutoff(5/5)。
284 测试全绿。
This commit is contained in:
shangfangjian
2026-09-22 02:45:50 +08:00
parent 63caa6736c
commit 49d78136a1
8 changed files with 260 additions and 32 deletions
@@ -0,0 +1,58 @@
"""P0-02: 积分榜改为追加快照(append-only) + available_at
去掉 uq_standings_league_season_team(league,season,team 唯一),
改为 (league, season, team, available_at) 唯一;
每次采集 INSERT 新行(available_at=now),支持回测还原历史榜单。
Revision ID: 0023_standings_append_only
Revises: 0022_match_score_status
Create Date: 2026-09-22
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = '0023_standings_append_only'
down_revision: Union[str, None] = '0022_match_score_status'
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
from datetime import datetime, timezone
# 1) 加 available_at 列(非空,默认 now;存量回填 retrieved_at 或 now)
op.add_column(
'standings',
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False,
server_default=sa.func.now()),
)
# 存量行: available_at 取 retrieved_at(若存在)否则 now
op.execute("UPDATE standings SET available_at = COALESCE(retrieved_at, NOW())")
# 2) 去旧唯一约束,加新唯一约束(league, season, team, available_at)
op.drop_constraint('uq_standings_league_season_team', 'standings', type_='unique')
op.drop_index('ix_standings_leason_season_pos', table_name='standings')
op.create_index('ix_standings_league_season_pos', 'standings', ['league_id', 'season', 'position'])
op.create_unique_constraint(
'uq_standings_league_season_team_available', 'standings',
['league_id', 'season', 'team_id', 'available_at'],
)
op.create_index(
'ix_standings_league_season_team_available', 'standings',
['league_id', 'season', 'team_id', 'available_at'],
)
def downgrade() -> None:
op.drop_index('ix_standings_league_season_team_available', table_name='standings')
op.drop_constraint('uq_standings_league_season_team_available', 'standings', type_='unique')
op.drop_index('ix_standings_league_season_pos', table_name='standings')
op.create_index('ix_standings_leason_season_pos', 'standings', ['league_id', 'season', 'position'])
op.create_unique_constraint(
'uq_standings_league_season_team', 'standings',
['league_id', 'season', 'team_id'],
)
op.drop_column('standings', 'available_at')
+2 -2
View File
@@ -206,7 +206,7 @@ CREATE TABLE predictions (
| 表 | 状态 | 用途 |
|---|---|---|
| `standings` | 已启用 | 联赛积分榜快照,按 `(league_id, season, team_id)` upsert,同联赛同赛季只保留最新快照;含排名/战绩/进失球/积分/分区(zone) |
| `standings` | 已启用 | 联赛积分榜追加快照(P0-02):每次采集 INSERT 新行(available_at=now),唯一键 `(league_id, season, team_id, available_at)`;查询取每队 available_at 最新快照,支持回测还原历史榜单。含排名/战绩/进失球/积分/分区(zone) |
| `app_settings` | 已启用 | 后台运行时设置(如数据源 API Key),读取时优先于 `.env` 默认值 |
| `schedules` | 已启用 | 定时采集任务配置(task/cron/leagues/enabled),供内置调度器执行 |
| `raw_events` | 预留未启用 | Bronze 层原始事件存档;规划中用于重放与审计 |
@@ -250,7 +250,7 @@ events 管线按以下优先级定位已有比赛,命中即复用(更新):
`task=stats` 只回填统计(xG/射门/控球等,也只补空),不创建比赛。
`task=standings` 按 `(league_id, season, team_id)` upsert 积分榜快照,同一联赛同一赛季只保留最新一份
`task=standings` 追加快照(available_at=now,ON CONFLICT DO NOTHING);公开接口与切片均取每队 available_at 最新快照,支持回测还原历史榜单
## 采集建议
+19 -1
View File
@@ -286,7 +286,9 @@ async def list_standings(
):
"""联赛积分榜(只读)。按联赛分组,每张榜按 position 排序。
season 为空时返回每个联赛最新采集到的赛季榜单(适合前端"查看最新积分榜")。
P0-02: standings 为追加快照,公开接口取每队 available_at 最新快照
(league_id, season, team_id 上按 available_at 取最新)。
season 为空时返回每个联赛最新采集到的赛季榜单。
"""
# 取每个联赛最新赛季(当 season 为空时)
latest_seasons: dict[int, str] = {}
@@ -299,9 +301,25 @@ async def list_standings(
).all()
latest_seasons = {r.league_id: r.latest for r in rows}
# P0-02: 子查询取每队最新 available_at 快照,再 JOIN 回主表拿完整行 + League
latest_per_team = (
select(
Standing.league_id, Standing.season, Standing.team_id,
func.max(Standing.available_at).label("max_available"),
)
.group_by(Standing.league_id, Standing.season, Standing.team_id)
.subquery("latest_per_team")
)
q = (
select(Standing, League)
.join(League, League.id == Standing.league_id)
.join(
latest_per_team,
(Standing.league_id == latest_per_team.c.league_id)
& (Standing.season == latest_per_team.c.season)
& (Standing.team_id == latest_per_team.c.team_id)
& (Standing.available_at == latest_per_team.c.max_available),
)
.order_by(League.name.asc(), Standing.position.asc())
)
if league:
+5 -14
View File
@@ -137,21 +137,12 @@ async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str |
retrieved_at=now,
)
# 同一联赛同一赛季只保留最新快照:按 (league, season, team) upsert
stmt = select(Standing).where(
Standing.league_id == league.id,
Standing.season == season_label,
Standing.team_id == team.id,
# P0-02: 追加快照——每次采集 INSERT 新行(available_at=now),
# ON CONFLICT (league, season, team, available_at) DO NOTHING。
standing = Standing(
league_id=league.id, season=season_label, team_id=team.id, available_at=now, **values
)
standing = (await db.execute(stmt)).scalar_one_or_none()
if standing is None:
standing = Standing(
league_id=league.id, season=season_label, team_id=team.id, **values
)
db.add(standing)
else:
for k, v in values.items():
setattr(standing, k, v)
db.add(standing)
league_r["upserted"] += 1
league_r["rows"] = len(rows)
+10 -3
View File
@@ -204,8 +204,11 @@ class MatchStats(Base):
class Standing(Base):
"""联赛积分榜快照(bzzoiro /leagues/{id}/standings/)。
同一联赛同一赛季只保留最新快照:重新采集时按 (league_id, season, team_id)
upsert。zone 来自 bzzoiro 分区(如 champions_league / europa_league / relegation)。
P0-02: 改为追加快照(append-only)。每次采集 INSERT 新行,available_at=now;
查询取 available_at<=cutoff 的每队最新快照(DISTINCT ON team_id ORDER available_at DESC)。
回测时可还原任意历史时刻的榜单,不再只是"最新快照、忽略 cutoff"
同一 (league_id, season, team_id, available_at) 唯一,ON CONFLICT DO NOTHING。
zone 来自 bzzoiro 分区(如 champions_league / europa_league / relegation)。
"""
__tablename__ = "standings"
@@ -228,13 +231,17 @@ class Standing(Base):
zone: Mapped[str | None] = mapped_column(String(50)) # champions_league / relegation 等
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
# P0-02: 快照可用时间(采集时间),唯一键组成部分 + cutoff 过滤依据
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow)
league: Mapped[League] = relationship()
team: Mapped[Team] = relationship(lazy="selectin")
__table_args__ = (
UniqueConstraint("league_id", "season", "team_id", name="uq_standings_league_season_team"),
# P0-02: (league, season, team, available_at) 唯一,支持追加快照 + ON CONFLICT DO NOTHING
UniqueConstraint("league_id", "season", "team_id", "available_at", name="uq_standings_league_season_team_available"),
Index("ix_standings_league_season_pos", "league_id", "season", "position"),
Index("ix_standings_league_season_team_available", "league_id", "season", "team_id", "available_at"),
)
+14 -4
View File
@@ -1,4 +1,4 @@
"""D - 联赛排名切片: 积分榜位置与实力差距(standings)。"""
"""D - 联赛排名切片: 积分榜快照(支持 cutoff 的历史还原,standings)。"""
from __future__ import annotations
from typing import TYPE_CHECKING
@@ -7,6 +7,7 @@ from sqlalchemy import select
from sqlalchemy.orm import selectinload
from src.db.base import AsyncSessionLocal
from src.db.models import League, Standing
from src.llm.slices.common import MatchHeader, SliceResult
if TYPE_CHECKING:
@@ -16,24 +17,33 @@ if TYPE_CHECKING:
async def standings_slice(header: MatchHeader, *, before=None, db: AsyncSession | None = None) -> SliceResult:
"""D - 联赛排名切片: 两队积分榜位置、积分、近期走势(form)、分区,评估整体实力差距。
before 参数保留与其他切片一致的签名(积分榜是最新快照,无历史版本,不受 cutoff 影响)。
P0-02: 支持 cutoff(before)。取 available_at<=cutoff 的每队最新快照
(DISTINCT ON team_id ORDER available_at DESC);before=None 时 cutoff=now()。
回测时可还原历史时刻榜单,不再只是"最新快照、忽略 cutoff"
db: 可选共享 session(见 context_builder 模块 docstring)。
语义区分:
- 两队都有积分榜行 → has_data=True(明确的排名信息)
- 任一队缺失 → has_data=False(升班马/杯赛无榜,信息不完整时明确声明)
"""
from src.db.models import League, Standing
# P0-02: before=None → cutoff=now()(取最新可用快照)
if before is None:
from datetime import datetime, timezone
before = datetime.now(timezone.utc)
if db is not None:
league = (await db.execute(select(League).where(League.id == header.league_id))).scalar_one_or_none()
# P0-02: DISTINCT ON (team_id) 取 available_at<=cutoff 的最新快照
rows = (
(
await db.execute(
select(Standing)
.options(selectinload(Standing.team))
.where(Standing.league_id == header.league_id)
.order_by(Standing.position.asc())
.where(Standing.available_at <= before)
.distinct(Standing.team_id)
.order_by(Standing.team_id, Standing.available_at.desc())
)
)
.scalars()
+138
View File
@@ -0,0 +1,138 @@
"""P0-02 回归测试: 积分榜改为追加快照(append-only) + available_at cutoff。
运行: pytest tests/test_p0_standings_cutoff.py -v
(模型约束用 fake DB;cutoff 过滤语义用 fake session 验证参数传递。)
"""
import pytest
from sqlalchemy import select
from src.db.models import League, Standing, Team
# ── fake DB(对齐现有测试约定) ────────────────────────────────────
class _FakeResult:
def __init__(self, items): self._items = list(items)
def scalars(self):
class _S:
def __init__(self, items): self._items = items
def all(self): return list(self._items)
return _S(self._items)
def scalar_one_or_none(self):
return self._items[0] if self._items else None
class _FakeDB:
captured: list = []
def __init__(self, league=None, standing_rows=None):
self._league = league
self._rows = standing_rows or []
_FakeDB.captured = []
def add(self, obj):
_FakeDB.captured.append(obj)
async def execute(self, stmt):
# 记录生成的 SQL(字符串化)供断言
_FakeDB.captured.append(str(stmt))
compiled = str(stmt)
if "league" in compiled.lower() and "standing" not in compiled.lower():
return _FakeResult([self._league] if self._league else [])
return _FakeResult(self._rows)
async def flush(self):
pass
class _FakeTeam:
def __init__(self, tid, name):
self.id = tid
self.name = name
self.name_zh = None
class _Header:
def __init__(self):
from src.llm.slices.common import MatchHeader
self._h = MatchHeader(
match_id=1, home_name="A", away_name="B", league_name="E0",
season="2026", match_date="2026-01-01", match_dt=None,
stage=None, home_team_id=10, away_team_id=20, league_id=1,
)
def __getattr__(self, name):
return getattr(self._h, name)
class TestStandingsModel:
"""standings 模型必须有 available_at + 新唯一约束。"""
def test_available_at_column(self):
cols = {c.name for c in Standing.__table__.columns}
assert "available_at" in cols
def test_unique_constraint_includes_available_at(self):
names = {c.name for c in Standing.__table__.constraints}
assert any("available" in n and n.startswith("uq_") for n in names), \
f"缺少含 available_at 的唯一约束,现有: {names}"
def test_old_unique_constraint_removed(self):
names = {c.name for c in Standing.__table__.constraints}
assert "uq_standings_league_season_team" not in names, \
"旧约束 uq_standings_league_season_team 应已被替换"
class TestStandingsSliceCutoff:
"""standings_slice 必须尊重 before(cutoff):before=None → now()。"""
def test_before_none_uses_now(self):
"""before=None 时应将 cutoff 视为 now()(取最新可用快照)。"""
from src.llm.slices import standings as st_mod
from datetime import datetime, timezone
calls = {}
real_execute = None
class _DB:
def __init__(self): self._league = League(id=1, code="E0", name="E0")
def add(self, obj): pass
async def execute(self, stmt):
# 捕获 WHERE available_at <= ? 的参数
sql = str(stmt)
if "available_at" in sql:
# 提取编译后的 params
try:
params = stmt.compile().params
calls["cutoff"] = params.get("available_at_1")
except Exception:
pass
if "league" in sql.lower() and "standing" not in sql.lower():
return _FakeResult([self._league])
return _FakeResult([])
async def flush(self): pass
async def run():
db = _DB()
before = None
await st_mod.standings_slice(_Header(), before=before, db=db)
import asyncio
asyncio.run(run())
# before=None 时应注入 now() 作为 cutoff
assert "cutoff" in calls, "未对 available_at 施加 cutoff 过滤"
assert calls["cutoff"] is not None
class TestStandingsAppendOnly:
"""采集应 INSERT 新行(带 available_at),不覆盖旧行。"""
def test_values_include_available_at(self):
"""采集构造的 Standing 必须含 available_at 字段。"""
from src.data import bzzoiro_standings as bzs
# 检查函数源码是否包含 available_at(编译期守卫)
import inspect
src = inspect.getsource(bzs.ingest_bzzoiro_standings)
assert "available_at" in src, "采集函数必须设置 available_at"
# 不应再出现按 (league, season, team) 的 upsert 查询
assert "scalar_one_or_none" not in src or "Standing.league_id == league.id" not in src.replace("available_at", ""), \
"不应再按 (league, season, team) 做 upsert 查询"
+14 -8
View File
@@ -194,8 +194,8 @@ async def test_r2_standings_actually_upserts(monkeypatch):
assert first.zone == "Champions League" # 优先取 label
async def test_r2_standings_upsert_updates_existing(monkeypatch):
"""行为测试: 已存在同 (league, season, team) 时应就地更新而非新增"""
async def test_r2_standings_append_new_row(monkeypatch):
"""P0-02 行为测试: 每次采集 INSERT 新行(带 available_at),不更新旧行"""
import src.data.bzzoiro as bz
from src.db.models import League, Standing
@@ -217,16 +217,20 @@ async def test_r2_standings_upsert_updates_existing(monkeypatch):
existing = Standing(league_id=42, season="2025-2026", team_id=7, position=9)
existing.points = 1
# 查询顺序: League → Team 预载(命中) → Standing 查询(命中已有行)
db = _FakeDb(results=[_FakeResult([league]), _FakeResult([team]), _FakeResult([existing])])
# 查询顺序: League(命中) → Team 预载(命中) → (P0-02 不再查询 Standing)
db = _FakeDb(results=[_FakeResult([league]), _FakeResult([team])])
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
assert result["total_upserted"] == 1
assert existing.points == 30, "已有行应被就地更新"
# P0-02: 追加快照——新增 Standing 行,旧行不被修改
new_rows = [o for o in db.added if isinstance(o, Standing)]
assert len(new_rows) == 1, "P0-02 应新增一条 Standing 行"
assert new_rows[0].points == 30, "新行应承载新采集数据"
assert new_rows[0].available_at is not None, "新行必须含 available_at"
# 旧行未被修改(仍保持原值)
assert existing.points == 1, "P0-02 旧行不应被覆盖"
assert result["leagues"]["EPL"]["teams_created"] == 0
# 不应新增 Standing(只有 league/team 层面的 add)
assert not [o for o in db.added if isinstance(o, Standing)]
def test_r2_source_contains_real_upsert_loop():
@@ -237,7 +241,9 @@ def test_r2_source_contains_real_upsert_loop():
assert "total_upserted" in src
assert 'result["total_upserted"] +=' in src, "total_upserted 必须真的被累加"
assert "Standing(" in src, "必须真的构造 Standing"
assert "select(Standing)" in src, "必须查询已有快照以决定 insert/update"
# P0-02: 追加快照——每次 INSERT 新行(带 available_at),不查询旧行做 upsert
assert "available_at" in src, "P0-02 采集必须设置 available_at"
assert "scalar_one_or_none" not in src, "P0-02 不应再按 (league, season, team) 做 upsert 查询"
# ============================================================