Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7be14c518 | ||
|
|
983363dab7 |
@@ -21,7 +21,7 @@
|
|||||||
│ │
|
│ │
|
||||||
┌──────────▼──────────┐ ┌─────────────▼────────────┐
|
┌──────────▼──────────┐ ┌─────────────▼────────────┐
|
||||||
│ PostgreSQL │ │ LLM (OpenAI-compatible) │
|
│ PostgreSQL │ │ LLM (OpenAI-compatible) │
|
||||||
│ 12 张表 │ │ OpenAI / Deepseek / │
|
│ 13 张表 │ │ OpenAI / Deepseek / │
|
||||||
│ leagues/teams/ │ │ Ollama / 任意网关 │
|
│ leagues/teams/ │ │ Ollama / 任意网关 │
|
||||||
│ matches/match_ │ └──────────────────────────┘
|
│ matches/match_ │ └──────────────────────────┘
|
||||||
│ stats/standings/ │
|
│ stats/standings/ │
|
||||||
|
|||||||
@@ -1,15 +1,20 @@
|
|||||||
"""采集任务状态表 ingest_jobs
|
"""新增 ingest_jobs 表
|
||||||
|
|
||||||
Revision ID: 0019_ingest_jobs
|
Revision ID: 0019_ingest_jobs
|
||||||
Revises: 0018_match_checks
|
Revises: 0018_match_checks
|
||||||
Create Date: 2026-09-22
|
Create Date: 2026-09-21
|
||||||
|
|
||||||
|
采集任务状态跟踪:POST /ingest/bzzoiro 创建 pending job 后台执行,
|
||||||
|
解决 fire-and-forget 不可观测问题(admin 可查询任务级状态与结果摘要)。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
|
||||||
|
# revision identifiers, used by Alembic.
|
||||||
revision: str = '0019_ingest_jobs'
|
revision: str = '0019_ingest_jobs'
|
||||||
down_revision: Union[str, None] = '0018_match_checks'
|
down_revision: Union[str, None] = '0018_match_checks'
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
@@ -21,22 +26,21 @@ def upgrade() -> None:
|
|||||||
'ingest_jobs',
|
'ingest_jobs',
|
||||||
sa.Column('id', sa.String(36), primary_key=True),
|
sa.Column('id', sa.String(36), primary_key=True),
|
||||||
sa.Column('task', sa.String(20), nullable=False),
|
sa.Column('task', sa.String(20), nullable=False),
|
||||||
sa.Column('params', sa.JSON(), nullable=False, server_default='{}'),
|
sa.Column('params', JSONB, nullable=False),
|
||||||
sa.Column('status', sa.String(20), nullable=False, server_default='pending'),
|
sa.Column('status', sa.String(20), nullable=False, server_default='pending'),
|
||||||
sa.Column('result', sa.JSON(), nullable=True),
|
sa.Column('result', JSONB),
|
||||||
sa.Column('error', sa.Text(), nullable=True),
|
sa.Column('error', sa.Text()),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now()),
|
sa.Column('created_at', sa.DateTime(timezone=True)),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('started_at', sa.DateTime(timezone=True)),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True)),
|
||||||
)
|
sa.CheckConstraint(
|
||||||
op.create_index('ix_ingest_job_status_created', 'ingest_jobs', ['status', 'created_at'])
|
|
||||||
op.create_check_constraint(
|
|
||||||
'ck_ingest_job_status', 'ingest_jobs',
|
|
||||||
"status IN ('pending','running','success','failed')",
|
"status IN ('pending','running','success','failed')",
|
||||||
|
name='ck_ingest_jobs_status',
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
op.create_index('ix_ingest_jobs_created', 'ingest_jobs', ['created_at'])
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.drop_constraint('ck_ingest_job_status', 'ingest_jobs', type_='check')
|
op.drop_index('ix_ingest_jobs_created', table_name='ingest_jobs')
|
||||||
op.drop_index('ix_ingest_job_status_created', table_name='ingest_jobs')
|
|
||||||
op.drop_table('ingest_jobs')
|
op.drop_table('ingest_jobs')
|
||||||
|
|||||||
@@ -1,32 +0,0 @@
|
|||||||
"""球队别名表 team_aliases
|
|
||||||
|
|
||||||
Revision ID: 0020_team_aliases
|
|
||||||
Revises: 0019_ingest_jobs
|
|
||||||
Create Date: 2026-09-22
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
from alembic import op
|
|
||||||
import sqlalchemy as sa
|
|
||||||
|
|
||||||
revision: str = '0020_team_aliases'
|
|
||||||
down_revision: Union[str, None] = '0019_ingest_jobs'
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.create_table(
|
|
||||||
'team_aliases',
|
|
||||||
sa.Column('alias_normalized', sa.String(120), primary_key=True),
|
|
||||||
sa.Column('team_id', sa.Integer, sa.ForeignKey('teams.id', ondelete='CASCADE'), nullable=False),
|
|
||||||
sa.Column('original_alias', sa.String(120), nullable=False),
|
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
||||||
)
|
|
||||||
op.create_index('ix_team_aliases_team_id', 'team_aliases', ['team_id'])
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_index('ix_team_aliases_team_id', table_name='team_aliases')
|
|
||||||
op.drop_table('team_aliases')
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
"""matches.source_event_id 部分唯一索引
|
|
||||||
|
|
||||||
业务唯一键:同联赛同主客同自然天一条(ix_matches_unique,既有)。
|
|
||||||
source_event_id 是上游 bzzoiro 的比赛 id,用于统计回填与血缘追踪;
|
|
||||||
当它非空时应全局唯一(同一 upstream 比赛只对应一行 matches),
|
|
||||||
避免同一场比赛因自然键天级舍入差异产生重复。
|
|
||||||
|
|
||||||
partial unique(WHERE source_event_id IS NOT NULL):
|
|
||||||
- 兼容存量空 source_event_id 的历史行(不强制回填);
|
|
||||||
- 新采集行均带 source_event_id,从此具备 upstream 唯一性。
|
|
||||||
|
|
||||||
Revision ID: 0021_match_source_event_id_unique
|
|
||||||
Revises: 0020_team_aliases
|
|
||||||
Create Date: 2026-09-22
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = '0021_match_source_event_id_unique'
|
|
||||||
down_revision: Union[str, None] = '0020_team_aliases'
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.create_index(
|
|
||||||
'ix_matches_source_event_id_unique',
|
|
||||||
'matches',
|
|
||||||
['source_event_id'],
|
|
||||||
unique=True,
|
|
||||||
postgresql_where=op.text('source_event_id IS NOT NULL'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_index('ix_matches_source_event_id_unique', table_name='matches')
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
"""P0-01: 比分可信度——score_status + 允许完赛缺分(NULL,禁止伪造 0:0)
|
|
||||||
|
|
||||||
替换 ck_matches_finished_has_score:引入 score_status(known/missing/unknown),
|
|
||||||
完赛 + score_status=missing 时 home/away_goals 必须 NULL(不伪造比分)。
|
|
||||||
|
|
||||||
Revision ID: 0022_match_score_status
|
|
||||||
Revises: 0021_match_source_event_id_unique
|
|
||||||
Create Date: 2026-09-22
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
from alembic import op
|
|
||||||
import sqlalchemy as sa
|
|
||||||
|
|
||||||
revision: str = '0022_match_score_status'
|
|
||||||
down_revision: Union[str, None] = '0021_match_source_event_id_unique'
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
# 1) 新增 score_status 列(默认 unknown)
|
|
||||||
op.add_column(
|
|
||||||
'matches',
|
|
||||||
sa.Column('score_status', sa.String(20), server_default='unknown', nullable=False),
|
|
||||||
)
|
|
||||||
|
|
||||||
# 2) 按现有数据回填 score_status(绝不写 goals=0):
|
|
||||||
# - 有比分(两列均非 NULL) → known
|
|
||||||
# - 无比分 + 完赛 → missing(缺分)
|
|
||||||
# - 其余 → unknown
|
|
||||||
op.execute(
|
|
||||||
"UPDATE matches SET score_status = 'known'"
|
|
||||||
" WHERE home_goals IS NOT NULL AND away_goals IS NOT NULL"
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"UPDATE matches SET score_status = 'missing'"
|
|
||||||
" WHERE match_status = 'finished' AND home_goals IS NULL AND away_goals IS NULL"
|
|
||||||
)
|
|
||||||
|
|
||||||
# 3) 删除旧约束,加新约束
|
|
||||||
op.drop_constraint('ck_matches_finished_has_score', 'matches', type_='check')
|
|
||||||
op.create_check_constraint(
|
|
||||||
'ck_matches_score_status_enum', 'matches',
|
|
||||||
"score_status IN ('known', 'missing', 'unknown')",
|
|
||||||
)
|
|
||||||
op.create_check_constraint(
|
|
||||||
'ck_matches_score_integrity', 'matches',
|
|
||||||
"match_status <> 'finished'"
|
|
||||||
" OR (score_status = 'known' AND home_goals IS NOT NULL AND away_goals IS NOT NULL)"
|
|
||||||
" OR (score_status IN ('missing', 'unknown') AND home_goals IS NULL AND away_goals IS NULL)",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_constraint('ck_matches_score_integrity', 'matches', type_='check')
|
|
||||||
op.drop_constraint('ck_matches_score_status_enum', 'matches', type_='check')
|
|
||||||
op.create_check_constraint(
|
|
||||||
'ck_matches_finished_has_score', 'matches',
|
|
||||||
"match_status <> 'finished' OR (home_goals IS NOT NULL AND away_goals IS NOT NULL)",
|
|
||||||
)
|
|
||||||
op.remove_column('matches', 'score_status')
|
|
||||||
@@ -1,58 +0,0 @@
|
|||||||
"""P0-02: 积分榜改为追加快照(append-only) + available_at
|
|
||||||
|
|
||||||
去掉 uq_standings_league_season_team(league,season,team 唯一),
|
|
||||||
改为 (league, season, team, available_at) 唯一;
|
|
||||||
每次采集 INSERT 新行(available_at=now),支持回测还原历史榜单。
|
|
||||||
|
|
||||||
Revision ID: 0023_standings_append_only
|
|
||||||
Revises: 0022_match_score_status
|
|
||||||
Create Date: 2026-09-22
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
from alembic import op
|
|
||||||
import sqlalchemy as sa
|
|
||||||
|
|
||||||
revision: str = '0023_standings_append_only'
|
|
||||||
down_revision: Union[str, None] = '0022_match_score_status'
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
# 1) 加 available_at 列(非空,默认 now;存量回填 retrieved_at 或 now)
|
|
||||||
op.add_column(
|
|
||||||
'standings',
|
|
||||||
sa.Column('available_at', sa.DateTime(timezone=True), nullable=False,
|
|
||||||
server_default=sa.func.now()),
|
|
||||||
)
|
|
||||||
# 存量行: available_at 取 retrieved_at(若存在)否则 now
|
|
||||||
op.execute("UPDATE standings SET available_at = COALESCE(retrieved_at, NOW())")
|
|
||||||
|
|
||||||
# 2) 去旧唯一约束,加新唯一约束(league, season, team, available_at)
|
|
||||||
op.drop_constraint('uq_standings_league_season_team', 'standings', type_='unique')
|
|
||||||
op.drop_index('ix_standings_leason_season_pos', table_name='standings')
|
|
||||||
op.create_index('ix_standings_league_season_pos', 'standings', ['league_id', 'season', 'position'])
|
|
||||||
op.create_unique_constraint(
|
|
||||||
'uq_standings_league_season_team_available', 'standings',
|
|
||||||
['league_id', 'season', 'team_id', 'available_at'],
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
'ix_standings_league_season_team_available', 'standings',
|
|
||||||
['league_id', 'season', 'team_id', 'available_at'],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_index('ix_standings_league_season_team_available', table_name='standings')
|
|
||||||
op.drop_constraint('uq_standings_league_season_team_available', 'standings', type_='unique')
|
|
||||||
op.drop_index('ix_standings_league_season_pos', table_name='standings')
|
|
||||||
op.create_index('ix_standings_leason_season_pos', 'standings', ['league_id', 'season', 'position'])
|
|
||||||
op.create_unique_constraint(
|
|
||||||
'uq_standings_league_season_team', 'standings',
|
|
||||||
['league_id', 'season', 'team_id'],
|
|
||||||
)
|
|
||||||
op.drop_column('standings', 'available_at')
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
"""P0-03: Prediction 幂等指纹——移除旧唯一约束,改为 partial unique on input_hash
|
|
||||||
|
|
||||||
input_hash 非空时唯一(同指纹返回已有行,不 UPDATE/INSERT);
|
|
||||||
兼容旧数据 NULL input_hash(不强制回填)。
|
|
||||||
|
|
||||||
Revision ID: 0024_prediction_idempotent_fingerprint
|
|
||||||
Revises: 0023_standings_append_only
|
|
||||||
Create Date: 2026-09-22
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
from alembic import op
|
|
||||||
import sqlalchemy as sa
|
|
||||||
|
|
||||||
revision: str = '0024_prediction_idempotent_fingerprint'
|
|
||||||
down_revision: Union[str, None] = '0023_standings_append_only'
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
# 移除旧唯一约束(match, provider, model, mode, run_type)
|
|
||||||
op.drop_constraint(
|
|
||||||
'uq_predictions_match_provider_model_mode_run_type',
|
|
||||||
'predictions', type_unique=True,
|
|
||||||
)
|
|
||||||
# P0-03: partial unique on input_hash(非空时唯一)
|
|
||||||
op.create_index(
|
|
||||||
'ix_predictions_input_hash_unique', 'predictions', ['input_hash'], unique=True,
|
|
||||||
postgresql_where=sa.text('input_hash IS NOT NULL'),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_index('ix_predictions_input_hash_unique', table_name='predictions')
|
|
||||||
op.create_unique_constraint(
|
|
||||||
'uq_predictions_match_provider_model_mode_run_type',
|
|
||||||
'predictions',
|
|
||||||
['match_id', 'provider', 'model', 'mode', 'run_type'],
|
|
||||||
)
|
|
||||||
@@ -20,7 +20,7 @@
|
|||||||
│ │ │ └─ aggregator 终裁(强模型) │ │
|
│ │ │ └─ aggregator 终裁(强模型) │ │
|
||||||
│ │ └────────────┘ │ events / standings │ │
|
│ │ └────────────┘ │ events / standings │ │
|
||||||
│ ┌──┴──────────────┴──┐ │ /stats 三条管线 │ │
|
│ ┌──┴──────────────┴──┐ │ /stats 三条管线 │ │
|
||||||
│ │ PostgreSQL (12 张表)│ └───────────────────┘ │
|
│ │ PostgreSQL (13 张表)│ └───────────────────┘ │
|
||||||
│ └────────────────────┘ httpx → 外部 API │
|
│ └────────────────────┘ httpx → 外部 API │
|
||||||
└─────────────────────────────────────────────────────┘
|
└─────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
@@ -70,7 +70,7 @@ Profeto/
|
|||||||
│ │ └── schedules.py # 定时任务 + 死信重试(router 级鉴权)
|
│ │ └── schedules.py # 定时任务 + 死信重试(router 级鉴权)
|
||||||
│ ├── db/
|
│ ├── db/
|
||||||
│ │ ├── base.py # async engine + get_db/get_db_read
|
│ │ ├── base.py # async engine + get_db/get_db_read
|
||||||
│ │ ├── models.py # 12 张表 ORM
|
│ │ ├── models.py # 13 张表 ORM
|
||||||
│ │ ├── repositories.py # 仓储层
|
│ │ ├── repositories.py # 仓储层
|
||||||
│ │ └── unit_of_work.py # 事务边界
|
│ │ └── unit_of_work.py # 事务边界
|
||||||
│ ├── data/
|
│ ├── data/
|
||||||
|
|||||||
+19
-1
@@ -190,9 +190,27 @@ Base URL: `http://localhost:8000` · 交互式文档: `/docs`(Swagger)与 `/redo
|
|||||||
| `season` | 当前赛季 | standings 赛季,如 `"2026-2027"` |
|
| `season` | 当前赛季 | standings 赛季,如 `"2026-2027"` |
|
||||||
| `limit` | 100 | stats 回填单次最大比赛数(1–500) |
|
| `limit` | 100 | stats 回填单次最大比赛数(1–500) |
|
||||||
|
|
||||||
- 响应含每联赛 `inserted`/`updated`/`errors` 统计
|
- 响应:`{"ok": true, "job_id": "<uuid>", "message": "……"}`,`job_id` 用于查询任务状态
|
||||||
- `task=stats` 只补空字段、不创建比赛(xG/射门/控球等统计回填)
|
- `task=stats` 只补空字段、不创建比赛(xG/射门/控球等统计回填)
|
||||||
|
|
||||||
|
### `GET /api/v1/admin/ingest/jobs/{job_id}`(需管理员)
|
||||||
|
|
||||||
|
查询一次采集任务的状态(`ingest_jobs` 表,任务级可观测性):
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"id": "…", "task": "standings", "params": {"leagues": ["E0"]},
|
||||||
|
"status": "success", "result": {"total_upserted": 20, "errors": []},
|
||||||
|
"error": null, "created_at": "…", "started_at": "…", "finished_at": "…"}
|
||||||
|
```
|
||||||
|
|
||||||
|
- `status` 取值:`pending`(已创建未开始)/ `running` / `success` / `failed`
|
||||||
|
- 管理端采集页提交后凭 `job_id` 轮询本端点直至终态
|
||||||
|
- 与 `ingest_failures` 死信独立:死信记录单条管线抓取失败(行级),job 记录整次任务结果
|
||||||
|
|
||||||
|
### `GET /api/v1/admin/ingest/jobs?limit=20&status=`(需管理员)
|
||||||
|
|
||||||
|
列出最近采集任务(最新在前),可按 `status` 过滤,`limit` 1–100。
|
||||||
|
|
||||||
> 历史版本曾有独立的 understat(xG)与 injuries(伤停)采集端点,
|
> 历史版本曾有独立的 understat(xG)与 injuries(伤停)采集端点,
|
||||||
> 已随数据源收敛为 bzzoiro 唯一来源而移除。
|
> 已随数据源收敛为 bzzoiro 唯一来源而移除。
|
||||||
|
|
||||||
|
|||||||
+9
-78
@@ -60,68 +60,12 @@
|
|||||||
|
|
||||||
### 队名归一化
|
### 队名归一化
|
||||||
|
|
||||||
`src/data.team_names.py` 维护 `NORMALIZE_MAP`(如 `Man City` → `Manchester City`),未命中映射的队名原样返回。
|
`src/data/team_names.py` 维护 `NORMALIZE_MAP`(如 `Man City` → `Manchester City`),未命中映射的队名原样返回。
|
||||||
归一前先做 Unicode NFKD 去重音。
|
归一前先做 Unicode NFKD 去重音。
|
||||||
|
|
||||||
**唯一键是归一后英文名**:`teams.name` 带 `UNIQUE` 约束,所有入库路径均经 `TeamRepository.get_or_create` 收敛归一化
|
|
||||||
(events / standings 管线在调用前归一,仓库层再做一次幂等归一作为兜底)。创建新 Team 时打 `info` 日志记录「原始名 → 归一后名」。
|
|
||||||
|
|
||||||
> ⚠️ **`normalize` 当前大小写敏感**:仅当入参大小写与 `NORMALIZE_MAP` 键完全匹配时才触发映射
|
|
||||||
>(如 `"Man City"` → `"Manchester City"`,但 `"man city"` 原样保留)。上游 bzzoiro 返回的队名首字母大写,
|
|
||||||
>实际命中无问题;若新增数据源返回全小写/全大写队名,需先 `title()` 再归一,否则会绕过映射产生重复 Team。
|
|
||||||
|
|
||||||
#### 别名机制(`team_aliases`)
|
|
||||||
|
|
||||||
归一仍可能遗漏历史重复队(如 `"Bayern Munich"` 与 `"Bayern München"` 经 NFKD 后相同则命中,
|
|
||||||
但 `"Man United"` vs `"Manchester United"` 若漏映射)。`team_aliases` 表提供**显式别名→teams.id** 映射:
|
|
||||||
|
|
||||||
| 列 | 说明 |
|
|
||||||
|----|------|
|
|
||||||
| `alias_normalized` | PK,`normalize(别名)` 后的稳定幂等键 |
|
|
||||||
| `team_id` | FK → `teams.id`(ON DELETE CASCADE) |
|
|
||||||
| `original_alias` | 原始写法(保留供参考) |
|
|
||||||
|
|
||||||
**定位三步链**(`get_or_create`):`normalize(name)` → 查 `teams.name` → 查 `team_aliases`(以 `normalize(name)` 为 PK)→ 都没有才 insert 新 Team。别名命中即复用已有 Team,避免产生重复。
|
|
||||||
|
|
||||||
**添加别名**(不自动合并历史重复队):
|
|
||||||
|
|
||||||
- **Admin 接口**(推荐):`POST /api/v1/admin/teams/aliases {"alias": "Man United", "team_id": 42}`(require_admin,幂等)
|
|
||||||
- **直接 SQL**:
|
|
||||||
```sql
|
|
||||||
INSERT INTO team_aliases(alias_normalized, team_id, original_alias)
|
|
||||||
VALUES ('man united', 42, 'Man United')
|
|
||||||
ON CONFLICT (alias_normalized) DO UPDATE SET team_id = EXCLUDED.team_id, original_alias = EXCLUDED.original_alias;
|
|
||||||
```
|
|
||||||
|
|
||||||
> ⚠️ **别名不自动合并**:发现历史重复队 A/B 后,需人工确认归一目标(如保留 B),再为 A 的归一名添加别名指向 B。
|
|
||||||
> 合并前请确认 A 的 `matches`/`standings` 引用是否需要迁移(可先 `SELECT COUNT(*) FROM matches WHERE home_team_id = A.id OR away_team_id = A.id` 评估)。
|
|
||||||
|
|
||||||
**改名 / 合并流程**(人工):
|
|
||||||
|
|
||||||
当发现两个 `teams` 行实际是同一球队(如 `Manchester City` 与 `Man City` 因历史数据大小写差异各占一行):
|
|
||||||
|
|
||||||
1. 确定**保留行**(通常选归一后规范名、且被更多 Match 引用的那行)。
|
|
||||||
2. 将被删行的所有引用指向保留行(`UPDATE matches SET home_team_id = 保留id WHERE home_team_id = 删行id`,客场同理;
|
|
||||||
`standings` / `match_stats` 按 `team_id` 同理)。
|
|
||||||
3. 删掉多余行:`DELETE FROM teams WHERE id = 删行id`。
|
|
||||||
|
|
||||||
> 此过程引入外键约束风险,务必在事务中执行并先 `BEGIN; ... ` 验证行数后再 `COMMIT`。
|
|
||||||
> 暂不做自动合并(避免误合相似名),仅通过下方 Admin 接口列出「近似重名」候选,由人工判定。
|
|
||||||
|
|
||||||
## Admin:近似重名候选
|
|
||||||
|
|
||||||
`GET /api/v1/admin/team-name-duplicates` 只读列出启发式相似候选(大小写差异、子串包含、前缀碰撞),不做自动合并。
|
|
||||||
典型用途:定期巡检,发现候选后走上方人工 SQL 合并。启发式规则:
|
|
||||||
|
|
||||||
- **大小写变体**:`lower(name)` 相同但 `name` 不同(如 `Arsenal FC` / `arsenal fc`)。
|
|
||||||
- **子串包含**:A 是 B 的子串且长度 ≥ 5(如 `Manchester` / `Manchester City`)。
|
|
||||||
- **前缀碰撞**:前 8 个字符相同的两队。
|
|
||||||
|
|
||||||
命中任一规则即列为候选,按相似度分组返回。
|
|
||||||
|
|
||||||
## 数据库 Schema
|
## 数据库 Schema
|
||||||
|
|
||||||
12 张表:核心业务表 5 张见下方 DDL,其余 7 张(积分榜/配置/调度/治理)见后文表格。
|
12 张业务/配置表 + 1 张任务状态表(`ingest_jobs`):核心业务表 5 张见下方 DDL,其余见后文表格。
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
-- 联赛
|
-- 联赛
|
||||||
@@ -202,13 +146,14 @@ CREATE TABLE predictions (
|
|||||||
);
|
);
|
||||||
```
|
```
|
||||||
|
|
||||||
其余 7 张表(DDL 略,详见 `src/db/models.py` 与 alembic 迁移):
|
其余 8 张表(DDL 略,详见 `src/db/models.py` 与 alembic 迁移):
|
||||||
|
|
||||||
| 表 | 状态 | 用途 |
|
| 表 | 状态 | 用途 |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| `standings` | 已启用 | 联赛积分榜追加快照(P0-02):每次采集 INSERT 新行(available_at=now),唯一键 `(league_id, season, team_id, available_at)`;查询取每队 available_at 最新快照,支持回测还原历史榜单。含排名/战绩/进失球/积分/分区(zone) |
|
| `standings` | 已启用 | 联赛积分榜快照,按 `(league_id, season, team_id)` upsert,同联赛同赛季只保留最新快照;含排名/战绩/进失球/积分/分区(zone) |
|
||||||
| `app_settings` | 已启用 | 后台运行时设置(如数据源 API Key),读取时优先于 `.env` 默认值 |
|
| `app_settings` | 已启用 | 后台运行时设置(如数据源 API Key),读取时优先于 `.env` 默认值 |
|
||||||
| `schedules` | 已启用 | 定时采集任务配置(task/cron/leagues/enabled),供内置调度器执行 |
|
| `schedules` | 已启用 | 定时采集任务配置(task/cron/leagues/enabled),供内置调度器执行 |
|
||||||
|
| `ingest_jobs` | 已启用 | 采集任务状态:POST ingest 创建 pending,后台流转 running→success/failed;result 存统计摘要,供 admin 轮询(任务级,与死信互补) |
|
||||||
| `raw_events` | 预留未启用 | Bronze 层原始事件存档;规划中用于重放与审计 |
|
| `raw_events` | 预留未启用 | Bronze 层原始事件存档;规划中用于重放与审计 |
|
||||||
| `ingest_failures` | 已启用 | 采集失败死信:bzzoiro 三条管线(events/standings/stats)抓取失败时写入,admin 后台可查看与重试 |
|
| `ingest_failures` | 已启用 | 采集失败死信:bzzoiro 三条管线(events/standings/stats)抓取失败时写入,admin 后台可查看与重试 |
|
||||||
| `data_quality_checks` | 预留未启用 | 数据质量检查结果;规划中定时检查比赛/统计/积分榜完整性 |
|
| `data_quality_checks` | 预留未启用 | 数据质量检查结果;规划中定时检查比赛/统计/积分榜完整性 |
|
||||||
@@ -218,25 +163,11 @@ CREATE TABLE predictions (
|
|||||||
|
|
||||||
1. **`match_date_date`(天级日期)**: 用于天级去重。bzzoiro 返回的时间带时分秒,精确匹配不可靠,故拆出 `DATE` 列做唯一键。
|
1. **`match_date_date`(天级日期)**: 用于天级去重。bzzoiro 返回的时间带时分秒,精确匹配不可靠,故拆出 `DATE` 列做唯一键。
|
||||||
|
|
||||||
2. **`ix_matches_unique`**: `(league_id, home_team_id, away_team_id, match_date_date)` 唯一,保证同一场比赛重复采集时 upsert 而非插入重复行。**业务唯一:同联赛同主客同自然天一条。**
|
2. **`ix_matches_unique`**: `(league_id, home_team_id, away_team_id, match_date_date)` 唯一,保证同一场比赛重复采集时 upsert 而非插入重复行。
|
||||||
|
|
||||||
3. **`source_event_id` 部分唯一**: `ix_matches_source_event_id_unique`(WHERE source_event_id IS NOT NULL)——上游 bzzoiro 的比赛 id,当非空时全局唯一。作用:
|
3. **`predictions` 级联删除**: `ON DELETE CASCADE`,删比赛时自动清其预测。
|
||||||
- 统计回填(`/events/{id}/stats/`)与 Bronze 血缘(/events/ 采集)通过它定位比赛,不依赖自然键天级舍入;
|
|
||||||
- 新采集行均带此 id,避免同一 upstream 比赛因时间戳差异绕开自然键产生重复。
|
|
||||||
- 存量空 source_event_id 历史行不受影响(不强制回填)。
|
|
||||||
|
|
||||||
4. **`predictions` 级联删除**: `ON DELETE CASCADE`,删比赛时自动清其预测。
|
4. **`mode` + `prompt_version`**: `single` 模式存 `v1`/`v2`,`multi` 模式存 `multi_v1`/`multi_v2`,eval summary 按这两列天然分组对比。
|
||||||
|
|
||||||
5. **`mode` + `prompt_version`**: `single` 模式存 `v1`/`v2`,`multi` 模式存 `multi_v1`/`multi_v2`,eval summary 按这两列天然分组对比。
|
|
||||||
|
|
||||||
## 采集 upsert 查找顺序
|
|
||||||
|
|
||||||
events 管线按以下优先级定位已有比赛,命中即复用(更新):
|
|
||||||
|
|
||||||
1. **`source_event_id`**(upstream event id,唯一索引命中)——最精确,跨自然键舍入差异;
|
|
||||||
2. **自然键**:`(league_id, home_team_id, away_team_id, match_date_date)`(内存去重,覆盖无 event id 的采集)。
|
|
||||||
|
|
||||||
两者都未命中 → insert 新比赛。
|
|
||||||
|
|
||||||
## 入库语义(幂等)
|
## 入库语义(幂等)
|
||||||
|
|
||||||
@@ -250,7 +181,7 @@ events 管线按以下优先级定位已有比赛,命中即复用(更新):
|
|||||||
|
|
||||||
`task=stats` 只回填统计(xG/射门/控球等,也只补空),不创建比赛。
|
`task=stats` 只回填统计(xG/射门/控球等,也只补空),不创建比赛。
|
||||||
|
|
||||||
`task=standings` 追加快照(available_at=now,ON CONFLICT DO NOTHING);公开接口与切片均取每队 available_at 最新快照,支持回测还原历史榜单。
|
`task=standings` 按 `(league_id, season, team_id)` upsert 积分榜快照,同一联赛同一赛季只保留最新一份。
|
||||||
|
|
||||||
## 采集建议
|
## 采集建议
|
||||||
|
|
||||||
|
|||||||
@@ -46,9 +46,6 @@ curl http://localhost:8000/health
|
|||||||
- [ ] **6. 反代信任头** — `TRUST_PROXY_HEADERS=True`,且**仅可信反代可达 API**;反代需设置 `X-Forwarded-For`(`$proxy_add_x_forwarded_for`)与 `X-Real-IP`,否则限流/日志按反代 IP 计数
|
- [ ] **6. 反代信任头** — `TRUST_PROXY_HEADERS=True`,且**仅可信反代可达 API**;反代需设置 `X-Forwarded-For`(`$proxy_add_x_forwarded_for`)与 `X-Real-IP`,否则限流/日志按反代 IP 计数
|
||||||
- [ ] **7. 限流前置到网关** — 推荐 Nginx `limit_req`(配置见[安全与限流](#安全与限流));应用内限流与 KeyRing 为**单进程内存实现**,多 worker 各自独立计数会把实际配额放大 N 倍(启动时会打印一次性告警)
|
- [ ] **7. 限流前置到网关** — 推荐 Nginx `limit_req`(配置见[安全与限流](#安全与限流));应用内限流与 KeyRing 为**单进程内存实现**,多 worker 各自独立计数会把实际配额放大 N 倍(启动时会打印一次性告警)
|
||||||
- [ ] **8. uvicorn 单 worker** — compose/Dockerfile 默认单 worker,保持即可;需横向扩容时先在网关统一限流,再起多实例(每实例仍单 worker)
|
- [ ] **8. uvicorn 单 worker** — compose/Dockerfile 默认单 worker,保持即可;需横向扩容时先在网关统一限流,再起多实例(每实例仍单 worker)
|
||||||
|
|
||||||
> ⚠️ **多 worker 陷阱**:应用内限流(`_RateLimiter`)与 KeyRing 均为**进程内纯内存状态**,多 worker 部署(如 `uvicorn --workers 4`)时各进程**各自独立计数、互不共享**——实际限流配额会被放大 N 倍、KeyRing 限流状态也不同步。
|
|
||||||
> 若确需多 worker,必须前置 Nginx/网关做**全局限流**(见[安全与限流](#安全与限流)),并设环境变量 `STRICT_SINGLE_WORKER=True`(见下)在启动期强制拒绝多 worker,避免静默配额漂移。
|
|
||||||
- [ ] **9. 启动后健康检查** — `curl /health` 返回 200(存活);`curl /health/ready` 返回 200(就绪,校验数据库连通,不可达时 503)
|
- [ ] **9. 启动后健康检查** — `curl /health` 返回 200(存活);`curl /health/ready` 返回 200(就绪,校验数据库连通,不可达时 503)
|
||||||
- [ ] **10. 数据库迁移** — compose/Dockerfile 启动命令已内置 `alembic upgrade head && uvicorn …`,升级镜像重启即自动迁移,无需手动执行
|
- [ ] **10. 数据库迁移** — compose/Dockerfile 启动命令已内置 `alembic upgrade head && uvicorn …`,升级镜像重启即自动迁移,无需手动执行
|
||||||
|
|
||||||
@@ -102,29 +99,10 @@ cd frontend && npm install && npm run dev
|
|||||||
| `LLM_AGGREGATOR_MODEL` | ❌ | — | 终裁模型(回落 `LLM_MODEL`) |
|
| `LLM_AGGREGATOR_MODEL` | ❌ | — | 终裁模型(回落 `LLM_MODEL`) |
|
||||||
| `BZZOIRO_KEY` | ✅ | — | bzzoiro 数据源 Key(唯一数据源) |
|
| `BZZOIRO_KEY` | ✅ | — | bzzoiro 数据源 Key(唯一数据源) |
|
||||||
| `CORS_ORIGINS` | ❌ | `http://localhost:5173,...` | 允许的跨域来源 |
|
| `CORS_ORIGINS` | ❌ | `http://localhost:5173,...` | 允许的跨域来源 |
|
||||||
| `STRICT_SINGLE_WORKER` | ❌ | `False` | `True` 时若以多 worker 启动则拒绝(防限流配额漂移) |
|
|
||||||
| `SECRET_KEY` | ❌ | — | 加密主密钥(生产环境必填) |
|
| `SECRET_KEY` | ❌ | — | 加密主密钥(生产环境必填) |
|
||||||
| `ADMIN_PASSWORD` | ❌ | — | 管理后台密码(留空=不启用) |
|
| `ADMIN_PASSWORD` | ❌ | — | 管理后台密码(留空=不启用) |
|
||||||
| `ADMIN_API_KEY` | ❌ | — | 机器/脚本调用的 API Key |
|
| `ADMIN_API_KEY` | ❌ | — | 机器/脚本调用的 API Key |
|
||||||
|
|
||||||
## 同站部署 vs 跨站 CSRF
|
|
||||||
|
|
||||||
Profeto 管理鉴权使用 **HttpOnly Cookie 会话**(登录后服务端写入),`allow_credentials=True` 的 CORS 配置允许浏览器跨域携带 Cookie——这也引入了 CSRF 面。部署拓扑决定风险等级:
|
|
||||||
|
|
||||||
**同站部署(推荐)**: 前端与 API 同域(反代把 `/` 与 `/api` 都转发到同一后端,或同源端口)。
|
|
||||||
- 浏览器视为 **same-origin**,CORS 不触发;`SameSite=Lax` 会话 Cookie 天然阻断跨站请求携带。
|
|
||||||
- 风险最低。`CORS_ORIGINS` 可设为空或同域来源,仅作兜底。
|
|
||||||
|
|
||||||
**跨站部署**: 前端与 API 不同域(如前端 `app.example.com`、API `api.example.com`,或开发时 `localhost:3000` → `localhost:8000`)。
|
|
||||||
- 必须把 API 域名列入 `CORS_ORIGINS`,且 `allow_credentials=True` 才能携带 Cookie。
|
|
||||||
- 此时任何被允许域下的页面都能构造带 Cookie 的请求 → **CSRF 面**:
|
|
||||||
- 状态变更接口(采集/回测/改密等写操作)要求**管理员 Cookie + 同域**,攻击者无法从第三方站点读取 Cookie,但可构造跨域表单/请求——`SameSite=Lax` 会阻断跨站 POST 表单提交(顶级导航 GET 仍放行),这是当前主要防线。
|
|
||||||
- `GET /api/v1/admin/*` 只读接口受 `SameSite=Lax` 下顶级导航可能被利用,但攻击者无法读取响应(CORS 不匹配时浏览器拦截)。
|
|
||||||
- **加固建议**:
|
|
||||||
1. 反代层加 `Origin`/`Referer` 校验,仅放行 `CORS_ORIGINS` 列表中的来源(即便 FastAPI CORS 已通过,反代校验是多一层纵深)。
|
|
||||||
2. 写操作要求自定义请求头(如 `X-Requested-With: XMLHttpRequest`),第三方站点无法在无预检下添加自定义头,天然阻断简单跨站 POST。
|
|
||||||
3. 生产强制 HTTPS(`APP_ENV=production` 下 Cookie 自动 `Secure`),防中间人窃 Cookie。
|
|
||||||
|
|
||||||
## LLM 提供商配置示例
|
## LLM 提供商配置示例
|
||||||
|
|
||||||
### OpenAI
|
### OpenAI
|
||||||
|
|||||||
@@ -129,7 +129,8 @@ Profeto/
|
|||||||
│ ├── 0003_injuries.py
|
│ ├── 0003_injuries.py
|
||||||
│ ├── 0004_snapshot_and_constraints.py
|
│ ├── 0004_snapshot_and_constraints.py
|
||||||
│ ├── 0005_prediction_status_and_stats_provenance.py
|
│ ├── 0005_prediction_status_and_stats_provenance.py
|
||||||
│ └── 0006_schema_model_drift_cleanup.py
|
│ ├── 0006_schema_model_drift_cleanup.py
|
||||||
|
│ └── …… (共 19 个迁移,最新 0019_ingest_jobs)
|
||||||
├── tests/ # 测试
|
├── tests/ # 测试
|
||||||
│ ├── test_core.py # 核心逻辑测试
|
│ ├── test_core.py # 核心逻辑测试
|
||||||
│ └── test_agents.py # 多 agent 测试
|
│ └── test_agents.py # 多 agent 测试
|
||||||
|
|||||||
@@ -72,6 +72,11 @@ export async function triggerCollection(req: CollectionRequest): Promise<any> {
|
|||||||
return api.post(`${API_BASE}/ingest/bzzoiro`, body)
|
return api.post(`${API_BASE}/ingest/bzzoiro`, body)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 查询采集任务状态(ingest_jobs;提交采集返回 job_id 后轮询用)
|
||||||
|
export async function fetchIngestJob(jobId: string): Promise<IngestJob> {
|
||||||
|
return api.get<IngestJob>(`${API_BASE}/admin/ingest/jobs/${jobId}`)
|
||||||
|
}
|
||||||
|
|
||||||
// ── 预测管理 ────────────────────────────────────────────────────
|
// ── 预测管理 ────────────────────────────────────────────────────
|
||||||
|
|
||||||
export async function triggerPrediction(req: { match_id: number; mode?: string }): Promise<any> {
|
export async function triggerPrediction(req: { match_id: number; mode?: string }): Promise<any> {
|
||||||
@@ -363,13 +368,6 @@ export function fetchIngestStatus(): Promise<{ sources: IngestSourceStatus[] }>
|
|||||||
return api.get<{ sources: IngestSourceStatus[] }>(`${API_BASE}/admin/ingest/status`)
|
return api.get<{ sources: IngestSourceStatus[] }>(`${API_BASE}/admin/ingest/status`)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 采集任务状态轮询(单任务)
|
|
||||||
*/
|
|
||||||
export function fetchIngestJob(jobId: string): Promise<IngestJob> {
|
|
||||||
return api.get<IngestJob>(`${API_BASE}/admin/ingest/jobs/${jobId}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 比赛详情(含最近预测摘要)
|
* 比赛详情(含最近预测摘要)
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -11,9 +11,9 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import { useEffect, useState, useCallback, useRef } from 'react'
|
import { useEffect, useState, useCallback, useRef } from 'react'
|
||||||
import { triggerCollection, fetchLeagues, fetchIngestJob } from '../dal'
|
import { triggerCollection, fetchLeagues, fetchIngestStatus, fetchIngestJob } from '../dal'
|
||||||
import type { IngestJob, League } from '../types'
|
import type { IngestSourceStatus } from '../types'
|
||||||
import type { CollectionRequest } from '../types'
|
import type { CollectionRequest, League, IngestJob } from '../types'
|
||||||
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
|
import { Card, CardBody, CardHeader, Badge, SectionHeader, Alert, Spinner } from '../components'
|
||||||
|
|
||||||
const TASKS = [
|
const TASKS = [
|
||||||
@@ -23,9 +23,33 @@ const TASKS = [
|
|||||||
{ value: 'all', label: '全量采集', desc: '依次采集比赛 + 积分榜 + 统计回填', icon: '⏵⏵' },
|
{ value: 'all', label: '全量采集', desc: '依次采集比赛 + 积分榜 + 统计回填', icon: '⏵⏵' },
|
||||||
] as const
|
] as const
|
||||||
|
|
||||||
type TaskUIStatus = 'idle' | 'running' | 'done' | 'error'
|
type TaskStatus = 'idle' | 'running' | 'done' | 'error'
|
||||||
|
|
||||||
const TERMINAL_STATUSES: ReadonlySet<string> = new Set(['success', 'failed'])
|
const JOB_STATUS_LABEL: Record<IngestJob['status'], string> = {
|
||||||
|
pending: '等待中',
|
||||||
|
running: '执行中',
|
||||||
|
success: '已完成',
|
||||||
|
failed: '失败',
|
||||||
|
}
|
||||||
|
|
||||||
|
/** job.result 摘要 → 多行文本(错误只显示条数,明细见「采集失败」页与系统日志) */
|
||||||
|
function summarizeResult(result: Record<string, any> | null): string[] {
|
||||||
|
if (!result) return []
|
||||||
|
return Object.entries(result).map(([k, v]) => {
|
||||||
|
if (v && typeof v === 'object') {
|
||||||
|
const bits: string[] = []
|
||||||
|
for (const [kk, vv] of Object.entries(v)) {
|
||||||
|
if (kk === 'errors') {
|
||||||
|
if (Array.isArray(vv) && vv.length > 0) bits.push(`错误 ${vv.length} 条`)
|
||||||
|
} else if (vv !== null && vv !== undefined) {
|
||||||
|
bits.push(`${kk} ${vv}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return `${k}: ${bits.length ? bits.join(' · ') : '无变更'}`
|
||||||
|
}
|
||||||
|
return `${k}: ${String(v)}`
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export default function CollectionPage() {
|
export default function CollectionPage() {
|
||||||
const [leagues, setLeagues] = useState<League[]>([])
|
const [leagues, setLeagues] = useState<League[]>([])
|
||||||
@@ -51,13 +75,16 @@ export default function CollectionPage() {
|
|||||||
const [limit, setLimit] = useState(100)
|
const [limit, setLimit] = useState(100)
|
||||||
const [loading, setLoading] = useState(false)
|
const [loading, setLoading] = useState(false)
|
||||||
const [error, setError] = useState<string | null>(null)
|
const [error, setError] = useState<string | null>(null)
|
||||||
|
const [result, setResult] = useState<{ title: string; detail: string } | null>(null)
|
||||||
|
|
||||||
// 任务进度反馈:跟踪真实 ingest_job 状态
|
// 任务进度反馈
|
||||||
const [taskStatus, setTaskStatus] = useState<TaskUIStatus>('idle')
|
const [taskStatus, setTaskStatus] = useState<TaskStatus>('idle')
|
||||||
const [jobId, setJobId] = useState<string | null>(null)
|
|
||||||
const [jobInfo, setJobInfo] = useState<IngestJob | null>(null)
|
|
||||||
const [taskStartedAt, setTaskStartedAt] = useState<number | null>(null)
|
const [taskStartedAt, setTaskStartedAt] = useState<number | null>(null)
|
||||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||||
|
const jobPollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||||
|
const [ingestSnap, setIngestSnap] = useState<IngestSourceStatus | null>(null)
|
||||||
|
// 采集任务状态(ingest_jobs):提交后有 job_id 即轮询,替代「30 秒盲等」
|
||||||
|
const [job, setJob] = useState<IngestJob | null>(null)
|
||||||
|
|
||||||
const loadLeagues = useCallback(async () => {
|
const loadLeagues = useCallback(async () => {
|
||||||
const lg = await fetchLeagues()
|
const lg = await fetchLeagues()
|
||||||
@@ -66,65 +93,51 @@ export default function CollectionPage() {
|
|||||||
|
|
||||||
useEffect(() => { loadLeagues() }, [loadLeagues])
|
useEffect(() => { loadLeagues() }, [loadLeagues])
|
||||||
|
|
||||||
// 轮询采集 job 直到终态(success/failed)
|
// 轮询采集状态(任务启动后)
|
||||||
|
const startPolling = useCallback(() => {
|
||||||
|
if (pollRef.current) clearInterval(pollRef.current)
|
||||||
|
pollRef.current = setInterval(async () => {
|
||||||
|
try {
|
||||||
|
const { sources } = await fetchIngestStatus()
|
||||||
|
const bz = sources.find(s => s.name === 'bzzoiro')
|
||||||
|
if (bz) setIngestSnap(bz)
|
||||||
|
} catch { /* ignore */ }
|
||||||
|
}, 5_000)
|
||||||
|
}, [])
|
||||||
|
|
||||||
const stopPolling = useCallback(() => {
|
const stopPolling = useCallback(() => {
|
||||||
if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null }
|
if (pollRef.current) { clearInterval(pollRef.current); pollRef.current = null }
|
||||||
}, [])
|
}, [])
|
||||||
|
|
||||||
useEffect(() => () => stopPolling(), [stopPolling])
|
// job 轮询:3 秒一次,终态(success/failed)自动停止;10 分钟兜底防泄漏
|
||||||
|
const stopJobPolling = useCallback(() => {
|
||||||
|
if (jobPollRef.current) { clearInterval(jobPollRef.current); jobPollRef.current = null }
|
||||||
|
}, [])
|
||||||
|
|
||||||
const startJobPolling = useCallback((id: string) => {
|
const startJobPolling = useCallback((jobId: string) => {
|
||||||
stopPolling()
|
stopJobPolling()
|
||||||
const tick = async () => {
|
let ticks = 0
|
||||||
|
jobPollRef.current = setInterval(async () => {
|
||||||
|
ticks += 1
|
||||||
|
if (ticks > 200) { stopJobPolling(); setTaskStatus('done'); return }
|
||||||
try {
|
try {
|
||||||
const job = await fetchIngestJob(id)
|
const j = await fetchIngestJob(jobId)
|
||||||
setJobInfo(job)
|
setJob(j)
|
||||||
if (TERMINAL_STATUSES.has(job.status)) {
|
if (j.status === 'success') { stopJobPolling(); setTaskStatus('done') }
|
||||||
setTaskStatus(job.status === 'success' ? 'done' : 'error')
|
else if (j.status === 'failed') { stopJobPolling(); setTaskStatus('error') }
|
||||||
stopPolling()
|
} catch { /* 网络抖动忽略,下个周期重试 */ }
|
||||||
}
|
}, 3_000)
|
||||||
} catch { /* 单次轮询失败不影响后续 */ }
|
}, [stopJobPolling])
|
||||||
}
|
|
||||||
tick()
|
useEffect(() => () => { stopPolling(); stopJobPolling() }, [stopPolling, stopJobPolling])
|
||||||
pollRef.current = setInterval(tick, 3_000)
|
|
||||||
}, [stopPolling])
|
|
||||||
|
|
||||||
const isEventsTask = task === 'events' || task === 'all'
|
const isEventsTask = task === 'events' || task === 'all'
|
||||||
|
|
||||||
// 友好汇总 job.result
|
|
||||||
const jobSummary = (j: IngestJob | null): { title: string; detail: string } | null => {
|
|
||||||
if (!j) return null
|
|
||||||
if (j.status === 'failed') {
|
|
||||||
return { title: '采集失败', detail: j.error || '采集任务异常终止,请到「系统日志」查看详细堆栈。' }
|
|
||||||
}
|
|
||||||
if (j.status !== 'success') return null
|
|
||||||
const r = j.result as Record<string, unknown> | null
|
|
||||||
if (!r) return { title: '采集完成', detail: '任务成功(无汇总数据)。' }
|
|
||||||
const ev = r.events as Record<string, unknown> | undefined
|
|
||||||
const evTotal = ev ? (ev.total_inserted as number ?? 0) + (ev.total_updated as number ?? 0) : 0
|
|
||||||
const st = r.standings as Record<string, unknown> | undefined
|
|
||||||
const stTotal = st ? (st.total_upserted as number ?? 0) : 0
|
|
||||||
const stats = r.stats as Record<string, unknown> | undefined
|
|
||||||
const statsTotal = stats ? (stats.created as number ?? 0) + (stats.updated as number ?? 0) : 0
|
|
||||||
const evErr = (ev?.errors as string[] | undefined)?.length ?? 0
|
|
||||||
const stErr = (st?.errors as string[] | undefined)?.length ?? 0
|
|
||||||
const statsErr = (stats?.errors as string[] | undefined)?.length ?? 0
|
|
||||||
const totalErr = evErr + stErr + statsErr
|
|
||||||
const parts: string[] = []
|
|
||||||
if (ev) parts.push(`比赛 +${evTotal}`)
|
|
||||||
if (st) parts.push(`积分榜 +${stTotal}`)
|
|
||||||
if (stats) parts.push(`统计 +${statsTotal}`)
|
|
||||||
const detail = parts.length
|
|
||||||
? `共更新: ${parts.join(' / ')}${totalErr ? `,错误 ${totalErr} 条(见日志)` : ''}`
|
|
||||||
: '任务成功'
|
|
||||||
return { title: '采集完成', detail }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleSubmit(e: React.FormEvent) {
|
async function handleSubmit(e: React.FormEvent) {
|
||||||
e.preventDefault()
|
e.preventDefault()
|
||||||
setError(null)
|
setError(null)
|
||||||
setJobInfo(null)
|
setResult(null)
|
||||||
setJobId(null)
|
setJob(null)
|
||||||
setLoading(true)
|
setLoading(true)
|
||||||
setTaskStatus('running')
|
setTaskStatus('running')
|
||||||
setTaskStartedAt(Date.now())
|
setTaskStartedAt(Date.now())
|
||||||
@@ -140,26 +153,38 @@ export default function CollectionPage() {
|
|||||||
date_from: isEventsTask ? dateFrom || undefined : undefined,
|
date_from: isEventsTask ? dateFrom || undefined : undefined,
|
||||||
date_to: isEventsTask ? dateTo || undefined : undefined,
|
date_to: isEventsTask ? dateTo || undefined : undefined,
|
||||||
}
|
}
|
||||||
const res = await triggerCollection(body)
|
const resp = await triggerCollection(body)
|
||||||
const id: string | undefined = res?.job_id
|
const jobId: string | undefined = resp?.job_id
|
||||||
if (id) {
|
if (jobId) {
|
||||||
setJobId(id)
|
// 有 job_id:轮询任务状态直至终态(3 秒/次)
|
||||||
startJobPolling(id)
|
setResult({
|
||||||
|
title: '采集任务已启动',
|
||||||
|
detail: `任务 ID: ${jobId}。正在跟踪执行进度,完成后展示结果摘要。`,
|
||||||
|
})
|
||||||
|
startJobPolling(jobId)
|
||||||
} else {
|
} else {
|
||||||
// 后端未返回 job_id(旧版兼容):退化为原逻辑
|
// 兜底:后端未返回 job_id(旧版本),退回「30 秒盲等 + 系统日志」
|
||||||
setTimeout(() => { setTaskStatus('done'); }, 30_000)
|
setResult({
|
||||||
|
title: '采集任务已启动',
|
||||||
|
detail: '正在后台执行(上游限速时可能需要数分钟)。完成结果与错误请到「系统日志」页查看(支持自动刷新)。',
|
||||||
|
})
|
||||||
|
startPolling()
|
||||||
|
setTimeout(() => {
|
||||||
|
setTaskStatus('done')
|
||||||
|
stopPolling()
|
||||||
|
}, 30_000)
|
||||||
}
|
}
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
setTaskStatus('error')
|
setTaskStatus('error')
|
||||||
setError(err instanceof Error ? err.message : '采集触发失败')
|
setError(err instanceof Error ? err.message : '采集触发失败')
|
||||||
stopPolling()
|
stopPolling()
|
||||||
|
stopJobPolling()
|
||||||
} finally {
|
} finally {
|
||||||
setLoading(false)
|
setLoading(false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const elapsed = taskStartedAt ? Math.round((Date.now() - taskStartedAt) / 1000) : 0
|
const elapsed = taskStartedAt ? Math.round((Date.now() - taskStartedAt) / 1000) : 0
|
||||||
const summary = jobInfo ? jobSummary(jobInfo) : null
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
@@ -283,11 +308,12 @@ export default function CollectionPage() {
|
|||||||
|
|
||||||
{/* 消息提示 */}
|
{/* 消息提示 */}
|
||||||
{error && <Alert kind="error" title="采集失败" message={error} onClose={() => setError(null)} />}
|
{error && <Alert kind="error" title="采集失败" message={error} onClose={() => setError(null)} />}
|
||||||
{summary && (
|
{result && (
|
||||||
<Alert
|
<Alert
|
||||||
kind={jobInfo?.status === 'success' ? 'ok' : 'error'}
|
kind="ok"
|
||||||
title={summary.title}
|
title={result.title}
|
||||||
message={summary.detail}
|
message={result.detail || undefined}
|
||||||
|
onClose={() => setResult(null)}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
@@ -312,10 +338,14 @@ export default function CollectionPage() {
|
|||||||
<div className="space-y-3">
|
<div className="space-y-3">
|
||||||
<div className="flex items-center gap-2 text-xs text-ink-700">
|
<div className="flex items-center gap-2 text-xs text-ink-700">
|
||||||
<Spinner />
|
<Spinner />
|
||||||
<span>任务执行中{jobId ? `(job ${jobId.slice(0, 8)}…)` : ''},已运行 {elapsed}s…</span>
|
<span>
|
||||||
|
{job
|
||||||
|
? `${JOB_STATUS_LABEL[job.status]}(轮询中),已运行 ${elapsed}s…`
|
||||||
|
: `任务执行中,已运行 ${elapsed}s…`}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<p className="text-2xs text-ink-400">
|
<p className="text-2xs text-ink-400">
|
||||||
后台异步执行,关闭页面不影响结果。每 3 秒自动轮询进度。
|
后台异步执行,关闭页面不影响结果。可稍后查看「系统日志」确认完成。
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
@@ -323,24 +353,44 @@ export default function CollectionPage() {
|
|||||||
<div className="space-y-2">
|
<div className="space-y-2">
|
||||||
<div className="flex items-center gap-2 text-xs text-emerald-700">
|
<div className="flex items-center gap-2 text-xs text-emerald-700">
|
||||||
<span className="inline-block h-2 w-2 rounded-full bg-emerald-500" />
|
<span className="inline-block h-2 w-2 rounded-full bg-emerald-500" />
|
||||||
<span>采集完成{jobId ? `(job ${jobId.slice(0, 8)}…)` : ''}</span>
|
<span>
|
||||||
|
{job
|
||||||
|
? `采集完成(${JOB_STATUS_LABEL[job.status]})`
|
||||||
|
: '任务已提交,后台执行中(可能尚未完成)'}
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
{summary && <p className="text-2xs text-ink-500">{summary.detail}</p>}
|
{job?.result && summarizeResult(job.result).length > 0 && (
|
||||||
|
<div className="rounded-md bg-ink-50 px-3 py-2">
|
||||||
|
{summarizeResult(job.result).map((line, i) => (
|
||||||
|
<p key={i} className="font-mono text-2xs leading-relaxed text-ink-600">{line}</p>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{!job && (
|
||||||
|
<p className="text-2xs text-ink-400">
|
||||||
|
采集耗时取决于数据量。请到「系统日志」页查看最终结果。
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{taskStatus === 'error' && (
|
{taskStatus === 'error' && (
|
||||||
<div className="space-y-1">
|
<div className="space-y-2">
|
||||||
<p className="text-xs text-press">采集失败{jobId ? `(job ${jobId.slice(0, 8)}…)` : ''}</p>
|
<p className="text-xs text-press">
|
||||||
{jobInfo?.error && (
|
{job?.status === 'failed' ? '采集任务失败' : '任务触发失败,请检查配置或网络。'}
|
||||||
<p className="text-2xs text-ink-500">{jobInfo.error.slice(0, 200)}</p>
|
</p>
|
||||||
|
{job?.error && (
|
||||||
|
<p className="break-all rounded-md bg-red-50 px-3 py-2 font-mono text-2xs text-press">
|
||||||
|
{job.error}
|
||||||
|
</p>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
{jobInfo?.created_at && (
|
{ingestSnap?.last_success_at && (
|
||||||
<p className="mt-2 text-2xs text-ink-400">
|
<div className="mt-3 border-t border-ink-100 pt-3">
|
||||||
创建于 {new Date(jobInfo.created_at).toLocaleString('zh-CN', { hour12: false })}
|
<p className="text-2xs text-ink-400">
|
||||||
{jobInfo.finished_at && ` · 完成于 ${new Date(jobInfo.finished_at).toLocaleString('zh-CN', { hour12: false })}`}
|
bzzoiro 最近一次采集: {new Date(ingestSnap.last_success_at).toLocaleString('zh-CN', { hour12: false })}
|
||||||
</p>
|
</p>
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
</CardBody>
|
</CardBody>
|
||||||
</Card>
|
</Card>
|
||||||
|
|||||||
+13
-14
@@ -103,6 +103,19 @@ export interface CollectionRequest {
|
|||||||
date_to?: string
|
date_to?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 采集任务状态(ingest_jobs,提交采集后轮询)
|
||||||
|
export interface IngestJob {
|
||||||
|
id: string
|
||||||
|
task: 'events' | 'standings' | 'stats' | 'all'
|
||||||
|
params: Record<string, unknown>
|
||||||
|
status: 'pending' | 'running' | 'success' | 'failed'
|
||||||
|
result: Record<string, any> | null
|
||||||
|
error: string | null
|
||||||
|
created_at: string | null
|
||||||
|
started_at: string | null
|
||||||
|
finished_at: string | null
|
||||||
|
}
|
||||||
|
|
||||||
// ── 评估 & 回测 ─────────────────────────────────────────────────
|
// ── 评估 & 回测 ─────────────────────────────────────────────────
|
||||||
|
|
||||||
export interface EvalCalibrationBucket {
|
export interface EvalCalibrationBucket {
|
||||||
@@ -283,20 +296,6 @@ export interface IngestSourceStatus {
|
|||||||
last_failure: IngestLastFailure | null
|
last_failure: IngestLastFailure | null
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 采集任务状态 ──────────────────────────────────────────────
|
|
||||||
|
|
||||||
export interface IngestJob {
|
|
||||||
id: string
|
|
||||||
task: string
|
|
||||||
params: Record<string, unknown>
|
|
||||||
status: 'pending' | 'running' | 'success' | 'failed'
|
|
||||||
result: Record<string, unknown> | null
|
|
||||||
error: string | null
|
|
||||||
created_at: string | null
|
|
||||||
started_at: string | null
|
|
||||||
finished_at: string | null
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── 比赛详情 ─────────────────────────────────────────────────────
|
// ── 比赛详情 ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
export interface MatchRecentPrediction {
|
export interface MatchRecentPrediction {
|
||||||
|
|||||||
@@ -1,65 +0,0 @@
|
|||||||
/**
|
|
||||||
* P0-00: HTTP client method/body/headers 可信度测试。
|
|
||||||
* 运行: node --experimental-strip-types frontend/src/lib/http.test.ts
|
|
||||||
*
|
|
||||||
* 最小环境 polyfill:Node 22 自带 fetch/AbortController,本测试不触发 401 路径,
|
|
||||||
* 故 window.dispatchEvent 不会被调用,无需完整 DOM。
|
|
||||||
*/
|
|
||||||
import { test } from 'node:test'
|
|
||||||
import assert from 'node:assert/strict'
|
|
||||||
|
|
||||||
// 最小浏览器环境 polyfill(仅覆盖 http.ts 在 happy path 用到的全局)
|
|
||||||
const store: Record<string, string> = {}
|
|
||||||
// @ts-expect-error 测试用最小 window stub
|
|
||||||
globalThis.window = {
|
|
||||||
dispatchEvent: () => false,
|
|
||||||
localStorage: {
|
|
||||||
getItem: (k: string) => store[k] ?? null,
|
|
||||||
setItem: (k: string, v: string) => { store[k] = v },
|
|
||||||
removeItem: (k: string) => { delete store[k] },
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
// 捕获每次 fetch 的入参供断言
|
|
||||||
let lastInit: RequestInit | undefined
|
|
||||||
globalThis.fetch = async (_url: string, init?: RequestInit) => {
|
|
||||||
lastInit = init
|
|
||||||
return new Response(JSON.stringify({ ok: true }), { status: 200, headers: { 'content-type': 'application/json' } })
|
|
||||||
}
|
|
||||||
|
|
||||||
const { http } = await import('./http.ts')
|
|
||||||
|
|
||||||
test('GET: method=GET, 无 body, 无 Content-Type', async () => {
|
|
||||||
await http.get('/api/v1/matches')
|
|
||||||
assert.equal(lastInit?.method, 'GET')
|
|
||||||
assert.equal(lastInit?.body, undefined)
|
|
||||||
assert.equal((lastInit?.headers as Record<string, string>)?.['Content-Type'], undefined)
|
|
||||||
})
|
|
||||||
|
|
||||||
test('POST: method=POST, 序列化 body, 有 Content-Type', async () => {
|
|
||||||
await http.post('/api/v1/matches', { a: 1 })
|
|
||||||
assert.equal(lastInit?.method, 'POST')
|
|
||||||
assert.equal(lastInit?.body, JSON.stringify({ a: 1 }))
|
|
||||||
assert.equal((lastInit?.headers as Record<string, string>)?.['Content-Type'], 'application/json')
|
|
||||||
})
|
|
||||||
|
|
||||||
test('POST 空 body: 不设 Content-Type', async () => {
|
|
||||||
await http.post('/api/v1/matches', undefined)
|
|
||||||
assert.equal(lastInit?.method, 'POST')
|
|
||||||
assert.equal(lastInit?.body, undefined)
|
|
||||||
assert.equal((lastInit?.headers as Record<string, string>)?.['Content-Type'], undefined)
|
|
||||||
})
|
|
||||||
|
|
||||||
test('PUT: method=PUT, 有 body 与 Content-Type', async () => {
|
|
||||||
await http.put('/api/v1/x', { b: 2 })
|
|
||||||
assert.equal(lastInit?.method, 'PUT')
|
|
||||||
assert.equal(lastInit?.body, JSON.stringify({ b: 2 }))
|
|
||||||
assert.equal((lastInit?.headers as Record<string, string>)?.['Content-Type'], 'application/json')
|
|
||||||
})
|
|
||||||
|
|
||||||
test('DELETE: method=DELETE, 无 body, 无 Content-Type', async () => {
|
|
||||||
await http.delete('/api/v1/x/1')
|
|
||||||
assert.equal(lastInit?.method, 'DELETE')
|
|
||||||
assert.equal(lastInit?.body, undefined)
|
|
||||||
assert.equal((lastInit?.headers as Record<string, string>)?.['Content-Type'], undefined)
|
|
||||||
})
|
|
||||||
@@ -49,11 +49,10 @@ async function request<T>(path: string, options: RequestOptions = {}): Promise<T
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const method = (options.method ?? 'GET').toUpperCase()
|
const res = await fetch(url, {
|
||||||
const body = options.body
|
signal: controller.signal,
|
||||||
// 仅当有 body 时设置 Content-Type,避免 GET/DELETE 等无 body 请求被误标
|
headers: { 'Content-Type': 'application/json' },
|
||||||
const headers: Record<string, string> = body ? { 'Content-Type': 'application/json' } : {}
|
})
|
||||||
const res = await fetch(url, { signal: controller.signal, method, body, headers })
|
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const rawText = await res.text()
|
const rawText = await res.text()
|
||||||
|
|||||||
@@ -8,9 +8,18 @@
|
|||||||
import { useEffect, useState, useCallback } from 'react'
|
import { useEffect, useState, useCallback } from 'react'
|
||||||
import { fetchStandings } from '../admin/dal'
|
import { fetchStandings } from '../admin/dal'
|
||||||
import type { StandingsLeague, StandingRow } from '../admin/dal'
|
import type { StandingsLeague, StandingRow } from '../admin/dal'
|
||||||
import { useLeagues } from './matches/hooks/useLeagues'
|
|
||||||
import { Spinner } from '../admin/components'
|
import { Spinner } from '../admin/components'
|
||||||
|
|
||||||
|
const LEAGUES = [
|
||||||
|
{ code: 'E0', name: '英超' },
|
||||||
|
{ code: 'SP1', name: '西甲' },
|
||||||
|
{ code: 'D1', name: '德甲' },
|
||||||
|
{ code: 'I1', name: '意甲' },
|
||||||
|
{ code: 'F1', name: '法甲' },
|
||||||
|
{ code: 'CL', name: '欧冠' },
|
||||||
|
{ code: 'EL', name: '欧联' },
|
||||||
|
]
|
||||||
|
|
||||||
const ZONE_META: Record<string, { label: string; cls: string }> = {
|
const ZONE_META: Record<string, { label: string; cls: string }> = {
|
||||||
// 欧战资格
|
// 欧战资格
|
||||||
'Champions League': { label: '欧冠区', cls: 'bg-emerald-100 text-emerald-700' },
|
'Champions League': { label: '欧冠区', cls: 'bg-emerald-100 text-emerald-700' },
|
||||||
@@ -55,9 +64,7 @@ function FormDots({ form }: { form?: string | null }) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export default function StandingsPage() {
|
export default function StandingsPage() {
|
||||||
// 统一数据源:复用 useLeagues hook(优先 API,失败回退本地常量)
|
const [leagues, setLeagues] = useState<StandingsLeague[]>([])
|
||||||
const leagues = useLeagues()
|
|
||||||
const [standings, setStandings] = useState<StandingsLeague[]>([])
|
|
||||||
const [activeLeague, setActiveLeague] = useState<string>('')
|
const [activeLeague, setActiveLeague] = useState<string>('')
|
||||||
const [loading, setLoading] = useState(true)
|
const [loading, setLoading] = useState(true)
|
||||||
const [switching, setSwitching] = useState(false) // 切换联赛中
|
const [switching, setSwitching] = useState(false) // 切换联赛中
|
||||||
@@ -78,7 +85,7 @@ export default function StandingsPage() {
|
|||||||
setError(null)
|
setError(null)
|
||||||
try {
|
try {
|
||||||
const data = await fetchStandings(code)
|
const data = await fetchStandings(code)
|
||||||
setStandings(data.leagues)
|
setLeagues(data.leagues)
|
||||||
if (!activeLeague && data.leagues.length > 0) {
|
if (!activeLeague && data.leagues.length > 0) {
|
||||||
setActiveLeague(data.leagues[0].league_code)
|
setActiveLeague(data.leagues[0].league_code)
|
||||||
}
|
}
|
||||||
@@ -97,7 +104,7 @@ export default function StandingsPage() {
|
|||||||
setSwitching(true)
|
setSwitching(true)
|
||||||
setActiveLeague(code)
|
setActiveLeague(code)
|
||||||
try {
|
try {
|
||||||
await fetchStandings(code).then(data => setStandings(data.leagues))
|
await fetchStandings(code).then(data => setLeagues(data.leagues))
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
setError(err instanceof Error ? err.message : '加载失败')
|
setError(err instanceof Error ? err.message : '加载失败')
|
||||||
} finally {
|
} finally {
|
||||||
@@ -105,32 +112,26 @@ export default function StandingsPage() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const active = standings.find(l => l.league_code === activeLeague) ?? standings[0]
|
const active = leagues.find(l => l.league_code === activeLeague) ?? leagues[0]
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
{/* 联赛切换 */}
|
{/* 联赛切换 */}
|
||||||
<div className="flex flex-wrap gap-2">
|
<div className="flex flex-wrap gap-2">
|
||||||
{leagues.map(l => {
|
{LEAGUES.map(l => (
|
||||||
// 标记该联赛是否有积分榜数据:有数据可正常切换,无数据也可选中但显示空态
|
|
||||||
const hasData = standings.some(s => s.league_code === l.code)
|
|
||||||
const isEmpty = activeLeague === l.code && !hasData
|
|
||||||
return (
|
|
||||||
<button
|
<button
|
||||||
key={l.code}
|
key={l.code}
|
||||||
onClick={() => switchLeague(l.code)}
|
onClick={() => switchLeague(l.code)}
|
||||||
disabled={switching}
|
disabled={switching}
|
||||||
title={hasData ? undefined : '暂无积分榜数据'}
|
|
||||||
className={`rounded border px-3 py-1.5 text-xs transition-colors disabled:opacity-50 ${
|
className={`rounded border px-3 py-1.5 text-xs transition-colors disabled:opacity-50 ${
|
||||||
activeLeague === l.code
|
activeLeague === l.code
|
||||||
? 'border-ink-900 bg-ink-900 text-paper-50'
|
? 'border-ink-900 bg-ink-900 text-paper-50'
|
||||||
: 'border-ink-200 text-ink-500 hover:border-ink-300'
|
: 'border-ink-200 text-ink-500 hover:border-ink-300'
|
||||||
} ${!hasData ? 'border-dashed' : ''}`}
|
}`}
|
||||||
>
|
>
|
||||||
{l.name}
|
{l.name}
|
||||||
</button>
|
</button>
|
||||||
)
|
))}
|
||||||
})}
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{error && (
|
{error && (
|
||||||
|
|||||||
@@ -1,171 +0,0 @@
|
|||||||
/**
|
|
||||||
* AgentsPanel: 五路专家意见 —— 可折叠 + 状态摘要 + 权重条形图 + 单路详情。
|
|
||||||
*
|
|
||||||
* P3-1: 从 MatchPredictPanel.PredictionPanel 拆出,渲染逻辑原样搬迁。
|
|
||||||
*/
|
|
||||||
import { useState } from 'react'
|
|
||||||
import type { AgentReport, Prediction } from '../../types'
|
|
||||||
import { AGENT_LABELS, CN_NUM } from '../../types'
|
|
||||||
|
|
||||||
const STATUS_BADGE: Record<string, { label: string; cls: string }> = {
|
|
||||||
ok: { label: '正常', cls: 'text-ink-500' },
|
|
||||||
no_data: { label: '无数据', cls: 'text-ink-400' },
|
|
||||||
error: { label: '调用失败', cls: 'text-press' },
|
|
||||||
parse_error: { label: '解析失败', cls: 'text-press' },
|
|
||||||
}
|
|
||||||
|
|
||||||
const SUFFICIENCY_LABEL: Record<string, string> = {
|
|
||||||
high: '充分',
|
|
||||||
medium: '一般',
|
|
||||||
low: '偏少',
|
|
||||||
none: '无',
|
|
||||||
}
|
|
||||||
|
|
||||||
/** home_edge(-1~1,正=利主队)的可视化:以中线为原点的双向细条 */
|
|
||||||
function EdgeBar({ value }: { value: number }) {
|
|
||||||
const v = Math.max(-1, Math.min(1, value))
|
|
||||||
const half = Math.abs(v) * 50
|
|
||||||
return (
|
|
||||||
<div className="relative h-px w-full bg-ink-200" role="presentation">
|
|
||||||
<span className="absolute left-1/2 top-1/2 h-2 w-px -translate-x-1/2 -translate-y-1/2 bg-ink-400" />
|
|
||||||
<span
|
|
||||||
className={`absolute top-0 h-px transition-all duration-500 ${v >= 0 ? 'bg-press' : 'bg-ink-600'}`}
|
|
||||||
style={
|
|
||||||
v >= 0
|
|
||||||
? { left: '50%', width: `${half}%` }
|
|
||||||
: { right: '50%', width: `${half}%` }
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 单路专家意见:汉字编号 + 细线行 */
|
|
||||||
function AgentCard({ report: r, no }: { report: AgentReport; no: string }) {
|
|
||||||
const badge = STATUS_BADGE[r.status] ?? { label: r.status, cls: 'text-ink-400' }
|
|
||||||
const inactive = r.status !== 'ok'
|
|
||||||
|
|
||||||
return (
|
|
||||||
<details className="group border-b border-ink-200">
|
|
||||||
<summary className="flex cursor-pointer list-none items-baseline gap-2.5 px-1 py-3">
|
|
||||||
<span className="font-serif text-sm text-ink-400">{no}</span>
|
|
||||||
<span className="text-sm font-medium text-ink-900">{AGENT_LABELS[r.agent] ?? r.agent}</span>
|
|
||||||
<span className={`text-2xs ${badge.cls}`}>{badge.label}</span>
|
|
||||||
|
|
||||||
<span className="ml-auto flex items-baseline gap-3 text-2xs tabular-nums text-ink-500">
|
|
||||||
{r.status === 'ok' && r.subjective_confidence !== null && (
|
|
||||||
<span>信心 {Math.round(r.subjective_confidence * 100)}%</span>
|
|
||||||
)}
|
|
||||||
{r.status === 'ok' && r.probable_score && (
|
|
||||||
<span className="font-serif font-bold text-ink-800">{r.probable_score}</span>
|
|
||||||
)}
|
|
||||||
<svg viewBox="0 0 20 20" className="h-3 w-3 self-center text-ink-300 transition-transform group-open:rotate-90" fill="currentColor" aria-hidden="true">
|
|
||||||
<path d="M7.3 5.3a1 1 0 011.4 0l4 4a1 1 0 010 1.4l-4 4a1 1 0 01-1.4-1.4L10.6 10 7.3 6.7a1 1 0 010-1.4z" />
|
|
||||||
</svg>
|
|
||||||
</span>
|
|
||||||
</summary>
|
|
||||||
|
|
||||||
<div className="space-y-3 px-1 pb-4 pl-7">
|
|
||||||
{inactive && (
|
|
||||||
<p className="text-xs leading-relaxed text-ink-500">
|
|
||||||
{r.status === 'no_data' && '该维度没有可用数据,已跳过 LLM 分析以节省额度(不影响其他专家)。'}
|
|
||||||
{r.status === 'error' && '该专家调用失败,本次结论未纳入其视角(fail-open 设计,不阻断整体预测)。'}
|
|
||||||
{r.status === 'parse_error' && '模型输出未通过格式校验,该报告已丢弃。'}
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!inactive && r.home_edge !== null && (
|
|
||||||
<div>
|
|
||||||
<div className="mb-1.5 flex items-baseline justify-between text-2xs">
|
|
||||||
<span className="text-ink-500">主队优势</span>
|
|
||||||
<span className={`font-semibold tabular-nums ${r.home_edge > 0 ? 'text-press' : r.home_edge < 0 ? 'text-ink-700' : 'text-ink-500'}`}>
|
|
||||||
{r.home_edge > 0 ? '+' : ''}{r.home_edge.toFixed(2)}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<EdgeBar value={r.home_edge} />
|
|
||||||
<div className="mt-1 flex justify-between text-2xs text-ink-400">
|
|
||||||
<span>利客队</span>
|
|
||||||
<span>利主队</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{r.analysis && (
|
|
||||||
<p className="font-serif text-sm leading-loose text-ink-700">{r.analysis}</p>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{r.key_evidence.length > 0 && (
|
|
||||||
<ul className="space-y-1.5">
|
|
||||||
{r.key_evidence.map((e, i) => (
|
|
||||||
<li key={i} className="flex gap-2 text-xs leading-relaxed text-ink-600">
|
|
||||||
<span className="flex-shrink-0 text-ink-300" aria-hidden="true">—</span>
|
|
||||||
<span>{e}</span>
|
|
||||||
</li>
|
|
||||||
))}
|
|
||||||
</ul>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{r.exp_home_goals !== null && r.exp_away_goals !== null && (
|
|
||||||
<p className="text-xs text-ink-500">
|
|
||||||
进球期望 <span className="font-serif font-bold tabular-nums text-ink-900">{r.exp_home_goals.toFixed(1)} - {r.exp_away_goals.toFixed(1)}</span>
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!inactive && (
|
|
||||||
<p className="border-t border-ink-100 pt-2.5 text-2xs text-ink-400">
|
|
||||||
数据充分度 {SUFFICIENCY_LABEL[r.data_sufficiency] ?? r.data_sufficiency}
|
|
||||||
<span className="mx-2 text-ink-200">|</span>
|
|
||||||
<span className="font-mono">{r.model}</span>
|
|
||||||
{r.latency_ms !== null && <span className="ml-2 tabular-nums">{r.latency_ms}ms</span>}
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</details>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export function AgentsPanel({ prediction }: { prediction: Prediction }) {
|
|
||||||
const [expertsOpen, setExpertsOpen] = useState(false)
|
|
||||||
const degraded = prediction.status === 'degraded' || prediction.status === 'failed'
|
|
||||||
const reports = prediction.agent_outputs ?? []
|
|
||||||
const okReports = reports.filter(r => r.status === 'ok')
|
|
||||||
|
|
||||||
if (reports.length === 0) return null
|
|
||||||
|
|
||||||
return (
|
|
||||||
<section>
|
|
||||||
<button
|
|
||||||
onClick={() => setExpertsOpen(o => !o)}
|
|
||||||
className="flex w-full items-center justify-between border-b border-ink-200 pb-2 text-left"
|
|
||||||
>
|
|
||||||
<span className="section-head mb-0">五路专家意见({okReports.length}/{reports.length} 路有效)</span>
|
|
||||||
<span className="text-2xs text-ink-400">{expertsOpen ? '收起' : '展开'}</span>
|
|
||||||
</button>
|
|
||||||
|
|
||||||
{!degraded && prediction.agent_weights && Object.keys(prediction.agent_weights).length > 0 && (
|
|
||||||
<div className="mt-3 space-y-1.5">
|
|
||||||
<span className="text-2xs text-ink-500">终裁权重分布</span>
|
|
||||||
{Object.entries(prediction.agent_weights)
|
|
||||||
.sort((a, b) => b[1] - a[1])
|
|
||||||
.map(([k, v]) => (
|
|
||||||
<div key={k} className="grid grid-cols-[96px_minmax(0,1fr)_40px] items-center gap-2">
|
|
||||||
<span className="truncate text-2xs text-ink-500">{AGENT_LABELS[k] ?? k}</span>
|
|
||||||
<div className="h-1.5 bg-paper-100">
|
|
||||||
<div className="h-full bg-press" style={{ width: `${Math.round(v * 100)}%` }} />
|
|
||||||
</div>
|
|
||||||
<span className="text-right text-2xs tabular-nums text-ink-500">{Math.round(v * 100)}%</span>
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{expertsOpen && (
|
|
||||||
<div className="mt-2">
|
|
||||||
{reports.map((r, i) => (
|
|
||||||
<AgentCard key={r.agent} report={r} no={CN_NUM[i] ?? String(i + 1)} />
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</section>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -6,11 +6,37 @@
|
|||||||
*/
|
*/
|
||||||
import { useEffect, useState } from 'react'
|
import { useEffect, useState } from 'react'
|
||||||
import TeamSideTag from '../../../components/TeamSideTag'
|
import TeamSideTag from '../../../components/TeamSideTag'
|
||||||
import type { Match, Prediction } from '../types'
|
import type { AgentReport, Match, Prediction } from '../types'
|
||||||
import { AGENT_LABELS } from '../types'
|
import { AGENT_LABELS, CN_NUM, OUTCOME_LABEL } from '../types'
|
||||||
import { AgentsPanel } from './AgentsPanel'
|
|
||||||
import { OutcomePanel } from './OutcomePanel'
|
/** 置信度细线:0~1 数值的低调可视化 */
|
||||||
import { ReasoningPanel } from './ReasoningPanel'
|
function Meter({ value }: { value: number }) {
|
||||||
|
const pct = Math.max(0, Math.min(100, Math.round(value * 100)))
|
||||||
|
return (
|
||||||
|
<div className="h-px w-full bg-ink-200" role="presentation">
|
||||||
|
<div className="h-px bg-press transition-[width] duration-500" style={{ width: `${pct}%` }} />
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** home_edge(-1~1,正=利主队)的可视化:以中线为原点的双向细条 */
|
||||||
|
function EdgeBar({ value }: { value: number }) {
|
||||||
|
const v = Math.max(-1, Math.min(1, value))
|
||||||
|
const half = Math.abs(v) * 50
|
||||||
|
return (
|
||||||
|
<div className="relative h-px w-full bg-ink-200" role="presentation">
|
||||||
|
<span className="absolute left-1/2 top-1/2 h-2 w-px -translate-x-1/2 -translate-y-1/2 bg-ink-400" />
|
||||||
|
<span
|
||||||
|
className={`absolute top-0 h-px transition-all duration-500 ${v >= 0 ? 'bg-press' : 'bg-ink-600'}`}
|
||||||
|
style={
|
||||||
|
v >= 0
|
||||||
|
? { left: '50%', width: `${half}%` }
|
||||||
|
: { right: '50%', width: `${half}%` }
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
function Spinner({ className = '' }: { className?: string }) {
|
function Spinner({ className = '' }: { className?: string }) {
|
||||||
return (
|
return (
|
||||||
@@ -27,73 +53,77 @@ function Spinner({ className = '' }: { className?: string }) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** 胜平负一行文字:选中的红字加方块标记,未选中的退灰 */
|
/** 胜平负一行文字:选中的红字加方块标记,未选中的退灰 */
|
||||||
.**
|
function OutcomeLine({
|
||||||
* P3-1:PredictionPanel 不再自绘,改为组合三个子组件:
|
pick,
|
||||||
* OutcomePanel(比分/胜平负/成本) / AgentsPanel(专家意见) / ReasoningPanel(终裁/降级)。
|
confidence,
|
||||||
* 渲染输出与拆分前完全一致(仅降级警示 + 报头 + 元信息仍在此处)。
|
|
||||||
*/
|
|
||||||
function PredictionPanel({
|
|
||||||
prediction,
|
|
||||||
match,
|
|
||||||
embedded = false,
|
|
||||||
}: {
|
}: {
|
||||||
prediction: Prediction
|
pick: string | null
|
||||||
match: Match
|
confidence: number | null
|
||||||
/** 弹窗嵌入模式:弹窗已提供报头,这里省略自带版头 */
|
|
||||||
embedded?: boolean
|
|
||||||
}) {
|
}) {
|
||||||
const homeName = match.home_team_zh || match.home_team
|
const options = ['1', 'X', '2'] as const
|
||||||
const awayName = match.away_team_zh || match.away_team
|
|
||||||
const degraded = prediction.status === 'degraded' || prediction.status === 'failed'
|
|
||||||
const reports = prediction.agent_outputs ?? []
|
|
||||||
const okReports = reports.filter(r => r.status === 'ok')
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<article className={embedded ? 'bg-paper-50' : 'border border-ink-900 bg-paper-50'}>
|
<div>
|
||||||
{!embedded && (
|
<div className="flex items-baseline justify-center gap-6 sm:gap-10">
|
||||||
<div className="flex flex-wrap items-baseline justify-between gap-2 border-b border-ink-900 bg-paper-100 px-4 py-2.5 sm:px-5">
|
{options.map(o => {
|
||||||
<h3 className="flex flex-wrap items-center gap-1.5 font-serif text-sm font-bold text-ink-900">
|
const on = pick === o
|
||||||
预测版 ·
|
return (
|
||||||
<TeamSideTag side="home" />
|
<div key={o} className="flex flex-col items-center gap-1">
|
||||||
{homeName}
|
<span className={`flex items-center gap-1.5 text-sm ${on ? 'font-semibold text-press' : 'text-ink-400'}`}>
|
||||||
<span>对</span>
|
{on && <span className="inline-block h-2 w-2 bg-press" aria-hidden="true" />}
|
||||||
<TeamSideTag side="away" />
|
{OUTCOME_LABEL[o]}
|
||||||
{awayName}
|
|
||||||
</h3>
|
|
||||||
<span className="text-2xs tabular-nums text-ink-500">
|
|
||||||
{prediction.provider} / {prediction.model}
|
|
||||||
{prediction.latency_ms !== null && ` · ${(prediction.latency_ms / 1000).toFixed(1)}s`}
|
|
||||||
</span>
|
</span>
|
||||||
|
{on && confidence !== null && (
|
||||||
|
<span className="text-2xs tabular-nums text-ink-500">
|
||||||
|
置信 {Math.round(confidence * 100)}%
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
{pick && confidence !== null && (
|
||||||
|
<div className="mx-auto mt-3 max-w-xs">
|
||||||
|
<Meter value={confidence} />
|
||||||
|
<p className="mt-1 text-center text-2xs text-ink-400">主观置信度,非统计概率</p>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
<div className="space-y-7 px-4 py-6 sm:px-5">
|
|
||||||
{degraded && (
|
|
||||||
<div className="border-l-2 border-press bg-press-wash/40 px-4 py-3">
|
|
||||||
<p className="font-serif text-sm font-bold text-press-dark">
|
|
||||||
{prediction.status === 'failed' ? '预测失败' : '预测降级(degraded)'}
|
|
||||||
</p>
|
|
||||||
<p className="mt-1.5 whitespace-pre-wrap text-xs leading-relaxed text-ink-600">
|
|
||||||
{prediction.reasoning || '所有专家均无有效数据或调用失败,无法生成可靠比分。'}
|
|
||||||
</p>
|
|
||||||
</div>
|
</div>
|
||||||
)}
|
|
||||||
|
|
||||||
{!degraded && <OutcomePanel prediction={prediction} match={match} />}
|
|
||||||
|
|
||||||
<p className="text-center text-2xs text-ink-500">
|
|
||||||
`多专家模式 · ${okReports.length}/${reports.length} 路有效`
|
|
||||||
{prediction.prompt_version && ` · prompt ${prediction.prompt_version}`}
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<AgentsPanel prediction={prediction} />
|
|
||||||
|
|
||||||
<ReasoningPanel prediction={prediction} />
|
|
||||||
</div>
|
|
||||||
</article>
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** 预测成本展示:耗时 + token + 限流余量 */
|
||||||
|
function PredictionCost({ prediction }: { prediction: Prediction }) {
|
||||||
|
const latency = prediction.latency_ms != null ? `${(prediction.latency_ms / 1000).toFixed(1)}s` : null
|
||||||
|
const tokens = prediction.prompt_tokens != null || prediction.completion_tokens != null
|
||||||
|
? `${prediction.prompt_tokens ?? '?'}/${prediction.completion_tokens ?? '?'}`
|
||||||
|
: null
|
||||||
|
|
||||||
|
if (!latency && !tokens && prediction.rate_limit_remaining == null) return null
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="border-t border-ink-200 pt-3 text-2xs text-ink-500">
|
||||||
|
<div className="flex flex-wrap items-center justify-center gap-x-4 gap-y-1">
|
||||||
|
{latency && (
|
||||||
|
<span className="inline-flex items-center gap-1">
|
||||||
|
<span aria-hidden="true" className="opacity-60">⏱</span>耗时 {latency}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{tokens && (
|
||||||
|
<span className="inline-flex items-center gap-1">
|
||||||
|
<span aria-hidden="true" className="opacity-60">Tok</span>prompt/completion: {tokens}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{prediction.rate_limit_remaining != null && prediction.rate_limit_remaining <= 3 && (
|
||||||
|
<span className="text-press" title="每分钟最多 10 次预测">
|
||||||
|
剩余配额: {prediction.rate_limit_remaining}/10(分钟)
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 预测过程阶段(按时长模拟;结果到达即跳到完成) */
|
||||||
function PredictProgress() {
|
function PredictProgress() {
|
||||||
const [elapsed, setElapsed] = useState(0)
|
const [elapsed, setElapsed] = useState(0)
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -169,6 +199,256 @@ function PredictProgress() {
|
|||||||
</div>
|
</div>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const STATUS_BADGE: Record<string, { label: string; cls: string }> = {
|
||||||
|
ok: { label: '正常', cls: 'text-ink-500' },
|
||||||
|
no_data: { label: '无数据', cls: 'text-ink-400' },
|
||||||
|
error: { label: '调用失败', cls: 'text-press' },
|
||||||
|
parse_error: { label: '解析失败', cls: 'text-press' },
|
||||||
|
}
|
||||||
|
|
||||||
|
const SUFFICIENCY_LABEL: Record<string, string> = {
|
||||||
|
high: '充分',
|
||||||
|
medium: '一般',
|
||||||
|
low: '偏少',
|
||||||
|
none: '无',
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 单路专家意见:汉字编号 + 细线行 */
|
||||||
|
function AgentCard({ report: r, no }: { report: AgentReport; no: string }) {
|
||||||
|
const badge = STATUS_BADGE[r.status] ?? { label: r.status, cls: 'text-ink-400' }
|
||||||
|
const inactive = r.status !== 'ok'
|
||||||
|
|
||||||
|
return (
|
||||||
|
<details className="group border-b border-ink-200">
|
||||||
|
<summary className="flex cursor-pointer list-none items-baseline gap-2.5 px-1 py-3">
|
||||||
|
<span className="font-serif text-sm text-ink-400">{no}</span>
|
||||||
|
<span className="text-sm font-medium text-ink-900">{AGENT_LABELS[r.agent] ?? r.agent}</span>
|
||||||
|
<span className={`text-2xs ${badge.cls}`}>{badge.label}</span>
|
||||||
|
|
||||||
|
<span className="ml-auto flex items-baseline gap-3 text-2xs tabular-nums text-ink-500">
|
||||||
|
{r.status === 'ok' && r.subjective_confidence !== null && (
|
||||||
|
<span>信心 {Math.round(r.subjective_confidence * 100)}%</span>
|
||||||
|
)}
|
||||||
|
{r.status === 'ok' && r.probable_score && (
|
||||||
|
<span className="font-serif font-bold text-ink-800">{r.probable_score}</span>
|
||||||
|
)}
|
||||||
|
<svg viewBox="0 0 20 20" className="h-3 w-3 self-center text-ink-300 transition-transform group-open:rotate-90" fill="currentColor" aria-hidden="true">
|
||||||
|
<path d="M7.3 5.3a1 1 0 011.4 0l4 4a1 1 0 010 1.4l-4 4a1 1 0 01-1.4-1.4L10.6 10 7.3 6.7a1 1 0 010-1.4z" />
|
||||||
|
</svg>
|
||||||
|
</span>
|
||||||
|
</summary>
|
||||||
|
|
||||||
|
<div className="space-y-3 px-1 pb-4 pl-7">
|
||||||
|
{/* 无数据 / 失败时给出明确说明,避免用户以为是空白 bug */}
|
||||||
|
{inactive && (
|
||||||
|
<p className="text-xs leading-relaxed text-ink-500">
|
||||||
|
{r.status === 'no_data' && '该维度没有可用数据,已跳过 LLM 分析以节省额度(不影响其他专家)。'}
|
||||||
|
{r.status === 'error' && '该专家调用失败,本次结论未纳入其视角(fail-open 设计,不阻断整体预测)。'}
|
||||||
|
{r.status === 'parse_error' && '模型输出未通过格式校验,该报告已丢弃。'}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!inactive && r.home_edge !== null && (
|
||||||
|
<div>
|
||||||
|
<div className="mb-1.5 flex items-baseline justify-between text-2xs">
|
||||||
|
<span className="text-ink-500">主队优势</span>
|
||||||
|
<span className={`font-semibold tabular-nums ${r.home_edge > 0 ? 'text-press' : r.home_edge < 0 ? 'text-ink-700' : 'text-ink-500'}`}>
|
||||||
|
{r.home_edge > 0 ? '+' : ''}{r.home_edge.toFixed(2)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<EdgeBar value={r.home_edge} />
|
||||||
|
<div className="mt-1 flex justify-between text-2xs text-ink-400">
|
||||||
|
<span>利客队</span>
|
||||||
|
<span>利主队</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{r.analysis && (
|
||||||
|
<p className="font-serif text-sm leading-loose text-ink-700">{r.analysis}</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{r.key_evidence.length > 0 && (
|
||||||
|
<ul className="space-y-1.5">
|
||||||
|
{r.key_evidence.map((e, i) => (
|
||||||
|
<li key={i} className="flex gap-2 text-xs leading-relaxed text-ink-600">
|
||||||
|
<span className="flex-shrink-0 text-ink-300" aria-hidden="true">—</span>
|
||||||
|
<span>{e}</span>
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{r.exp_home_goals !== null && r.exp_away_goals !== null && (
|
||||||
|
<p className="text-xs text-ink-500">
|
||||||
|
进球期望 <span className="font-serif font-bold tabular-nums text-ink-900">{r.exp_home_goals.toFixed(1)} - {r.exp_away_goals.toFixed(1)}</span>
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!inactive && (
|
||||||
|
<p className="border-t border-ink-100 pt-2.5 text-2xs text-ink-400">
|
||||||
|
数据充分度 {SUFFICIENCY_LABEL[r.data_sufficiency] ?? r.data_sufficiency}
|
||||||
|
<span className="mx-2 text-ink-200">|</span>
|
||||||
|
<span className="font-mono">{r.model}</span>
|
||||||
|
{r.latency_ms !== null && <span className="ml-2 tabular-nums">{r.latency_ms}ms</span>}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function PredictionPanel({
|
||||||
|
prediction,
|
||||||
|
match,
|
||||||
|
embedded = false,
|
||||||
|
}: {
|
||||||
|
prediction: Prediction
|
||||||
|
match: Match
|
||||||
|
/** 弹窗嵌入模式:弹窗已提供报头,这里省略自带版头 */
|
||||||
|
embedded?: boolean
|
||||||
|
}) {
|
||||||
|
const homeName = match.home_team_zh || match.home_team
|
||||||
|
const [expertsOpen, setExpertsOpen] = useState(false)
|
||||||
|
const awayName = match.away_team_zh || match.away_team
|
||||||
|
const degraded = prediction.status === 'degraded' || prediction.status === 'failed'
|
||||||
|
const reports = prediction.agent_outputs ?? []
|
||||||
|
const okReports = reports.filter(r => r.status === 'ok')
|
||||||
|
|
||||||
|
return (
|
||||||
|
<article className={embedded ? 'bg-paper-50' : 'border border-ink-900 bg-paper-50'}>
|
||||||
|
{!embedded && (
|
||||||
|
<div className="flex flex-wrap items-baseline justify-between gap-2 border-b border-ink-900 bg-paper-100 px-4 py-2.5 sm:px-5">
|
||||||
|
<h3 className="flex flex-wrap items-center gap-1.5 font-serif text-sm font-bold text-ink-900">
|
||||||
|
预测版 ·
|
||||||
|
<TeamSideTag side="home" />
|
||||||
|
{homeName}
|
||||||
|
<span>对</span>
|
||||||
|
<TeamSideTag side="away" />
|
||||||
|
{awayName}
|
||||||
|
</h3>
|
||||||
|
<span className="text-2xs tabular-nums text-ink-500">
|
||||||
|
{prediction.provider} / {prediction.model}
|
||||||
|
{prediction.latency_ms !== null && ` · ${(prediction.latency_ms / 1000).toFixed(1)}s`}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div className="space-y-7 px-4 py-6 sm:px-5">
|
||||||
|
{/* ── degraded / failed 态:醒目警示 + 原因,不展示虚假比分 ── */}
|
||||||
|
{degraded && (
|
||||||
|
<div className="border-l-2 border-press bg-press-wash/40 px-4 py-3">
|
||||||
|
<p className="font-serif text-sm font-bold text-press-dark">
|
||||||
|
{prediction.status === 'failed' ? '预测失败' : '预测降级(degraded)'}
|
||||||
|
</p>
|
||||||
|
<p className="mt-1.5 whitespace-pre-wrap text-xs leading-relaxed text-ink-600">
|
||||||
|
{prediction.reasoning || '所有专家均无有效数据或调用失败,无法生成可靠比分。'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 主结论(仅 success 展示) ── */}
|
||||||
|
{!degraded && (
|
||||||
|
<>
|
||||||
|
<div className="text-center">
|
||||||
|
<p className="font-serif text-5xl font-bold tabular-nums leading-none text-ink-900 sm:text-6xl">
|
||||||
|
{prediction.pred_home_goals ?? '-'}
|
||||||
|
<span className="mx-3 font-normal text-ink-300">:</span>
|
||||||
|
{prediction.pred_away_goals ?? '-'}
|
||||||
|
</p>
|
||||||
|
<p className="mt-3 text-2xs tracking-[0.5em] text-ink-400">预测比分</p>
|
||||||
|
{prediction.alt_pred_home_goals != null && prediction.alt_pred_away_goals != null && (
|
||||||
|
<p className="mt-2 text-2xs tabular-nums text-ink-400">
|
||||||
|
备选{' '}
|
||||||
|
<span className="font-serif text-sm font-bold tabular-nums text-ink-600">
|
||||||
|
{prediction.alt_pred_home_goals}<span className="mx-0.5 font-normal text-ink-300">:</span>{prediction.alt_pred_away_goals}
|
||||||
|
</span>
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="border-y border-ink-200 py-4">
|
||||||
|
<OutcomeLine pick={prediction.pred_1x2} confidence={prediction.subjective_confidence} />
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 成本信息(耗时 + token + 限流余量) ── */}
|
||||||
|
{!degraded && (
|
||||||
|
<PredictionCost prediction={prediction} />
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 元信息 ── */}
|
||||||
|
<p className="text-center text-2xs text-ink-500">
|
||||||
|
`多专家模式 · ${okReports.length}/${reports.length} 路有效`
|
||||||
|
{prediction.prompt_version && ` · prompt ${prediction.prompt_version}`}
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{/* ── 终裁/降级说明意见 ── */}
|
||||||
|
{prediction.reasoning && degraded && (
|
||||||
|
<section>
|
||||||
|
<h4 className="section-head mb-2">降级原因</h4>
|
||||||
|
<blockquote className="border-l-2 border-press pl-4">
|
||||||
|
<p className="whitespace-pre-wrap font-serif text-sm leading-loose text-ink-700">{prediction.reasoning}</p>
|
||||||
|
</blockquote>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 专家意见(多模式):可折叠 + 状态摘要 + 权重条形图 ── */}
|
||||||
|
{reports.length > 0 && (
|
||||||
|
<section>
|
||||||
|
<button
|
||||||
|
onClick={() => setExpertsOpen(o => !o)}
|
||||||
|
className="flex w-full items-center justify-between border-b border-ink-200 pb-2 text-left"
|
||||||
|
>
|
||||||
|
<span className="section-head mb-0">五路专家意见({okReports.length}/{reports.length} 路有效)</span>
|
||||||
|
<span className="text-2xs text-ink-400">{expertsOpen ? '收起' : '展开'}</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{/* 权重条形图(仅 success 且有权重时显示) */}
|
||||||
|
{!degraded && prediction.agent_weights && Object.keys(prediction.agent_weights).length > 0 && (
|
||||||
|
<div className="mt-3 space-y-1.5">
|
||||||
|
<span className="text-2xs text-ink-500">终裁权重分布</span>
|
||||||
|
{Object.entries(prediction.agent_weights)
|
||||||
|
.sort((a, b) => b[1] - a[1])
|
||||||
|
.map(([k, v]) => (
|
||||||
|
<div key={k} className="grid grid-cols-[96px_minmax(0,1fr)_40px] items-center gap-2">
|
||||||
|
<span className="truncate text-2xs text-ink-500">{AGENT_LABELS[k] ?? k}</span>
|
||||||
|
<div className="h-1.5 bg-paper-100">
|
||||||
|
<div className="h-full bg-press" style={{ width: `${Math.round(v * 100)}%` }} />
|
||||||
|
</div>
|
||||||
|
<span className="text-right text-2xs tabular-nums text-ink-500">{Math.round(v * 100)}%</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{expertsOpen && (
|
||||||
|
<div className="mt-2">
|
||||||
|
{reports.map((r, i) => (
|
||||||
|
<AgentCard key={r.agent} report={r} no={CN_NUM[i] ?? String(i + 1)} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* ── 终裁意见(success) ── */}
|
||||||
|
{prediction.reasoning && !degraded && (
|
||||||
|
<section>
|
||||||
|
<h4 className="section-head mb-3">终裁意见</h4>
|
||||||
|
<blockquote className="border-l-2 border-press pl-4">
|
||||||
|
<p className="whitespace-pre-wrap font-serif text-sm leading-loose text-ink-700">{prediction.reasoning}</p>
|
||||||
|
</blockquote>
|
||||||
|
</section>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</article>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 预测弹窗:进行中显示过程可视化,完成后显示预测版,失败显示原因 */
|
||||||
export function PredictModal({
|
export function PredictModal({
|
||||||
match,
|
match,
|
||||||
predicting,
|
predicting,
|
||||||
|
|||||||
@@ -1,126 +0,0 @@
|
|||||||
/**
|
|
||||||
* OutcomePanel: 预测主结论 —— 比分 / 胜平负 / 置信度 / 成本。
|
|
||||||
*
|
|
||||||
* P3-1: 从 MatchPredictPanel.PredictionPanel 拆出,渲染逻辑原样搬迁。
|
|
||||||
*/
|
|
||||||
import TeamSideTag from '../../../../components/TeamSideTag'
|
|
||||||
import type { Match, Prediction } from '../../types'
|
|
||||||
import { OUTCOME_LABEL } from '../../types'
|
|
||||||
|
|
||||||
/** 置信度细线:0~1 数值的低调可视化 */
|
|
||||||
function Meter({ value }: { value: number }) {
|
|
||||||
const pct = Math.max(0, Math.min(100, Math.round(value * 100)))
|
|
||||||
return (
|
|
||||||
<div className="h-px w-full bg-ink-200" role="presentation">
|
|
||||||
<div className="h-px bg-press transition-[width] duration-500" style={{ width: `${pct}%` }} />
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 胜平负一行文字:选中的红字加方块标记,未选中的退灰 */
|
|
||||||
function OutcomeLine({
|
|
||||||
pick,
|
|
||||||
confidence,
|
|
||||||
}: {
|
|
||||||
pick: string | null
|
|
||||||
confidence: number | null
|
|
||||||
}) {
|
|
||||||
const options = ['1', 'X', '2'] as const
|
|
||||||
return (
|
|
||||||
<div>
|
|
||||||
<div className="flex items-baseline justify-center gap-6 sm:gap-10">
|
|
||||||
{options.map(o => {
|
|
||||||
const on = pick === o
|
|
||||||
return (
|
|
||||||
<div key={o} className="flex flex-col items-center gap-1">
|
|
||||||
<span className={`flex items-center gap-1.5 text-sm ${on ? 'font-semibold text-press' : 'text-ink-400'}`}>
|
|
||||||
{on && <span className="inline-block h-2 w-2 bg-press" aria-hidden="true" />}
|
|
||||||
{OUTCOME_LABEL[o]}
|
|
||||||
</span>
|
|
||||||
{on && confidence !== null && (
|
|
||||||
<span className="text-2xs tabular-nums text-ink-500">
|
|
||||||
置信 {Math.round(confidence * 100)}%
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
})}
|
|
||||||
</div>
|
|
||||||
{pick && confidence !== null && (
|
|
||||||
<div className="mx-auto mt-3 max-w-xs">
|
|
||||||
<Meter value={confidence} />
|
|
||||||
<p className="mt-1 text-center text-2xs text-ink-400">主观置信度,非统计概率</p>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** 预测成本展示:耗时 + token + 限流余量 */
|
|
||||||
function PredictionCost({ prediction }: { prediction: Prediction }) {
|
|
||||||
const latency = prediction.latency_ms != null ? `${(prediction.latency_ms / 1000).toFixed(1)}s` : null
|
|
||||||
const tokens = prediction.prompt_tokens != null || prediction.completion_tokens != null
|
|
||||||
? `${prediction.prompt_tokens ?? '?'}/${prediction.completion_tokens ?? '?'}`
|
|
||||||
: null
|
|
||||||
|
|
||||||
if (!latency && !tokens && prediction.rate_limit_remaining == null) return null
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="border-t border-ink-200 pt-3 text-2xs text-ink-500">
|
|
||||||
<div className="flex flex-wrap items-center justify-center gap-x-4 gap-y-1">
|
|
||||||
{latency && (
|
|
||||||
<span className="inline-flex items-center gap-1">
|
|
||||||
<span aria-hidden="true" className="opacity-60">⏱</span>耗时 {latency}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
{tokens && (
|
|
||||||
<span className="inline-flex items-center gap-1">
|
|
||||||
<span aria-hidden="true" className="opacity-60">Tok</span>prompt/completion: {tokens}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
{prediction.rate_limit_remaining != null && prediction.rate_limit_remaining <= 3 && (
|
|
||||||
<span className="text-press" title="每分钟最多 10 次预测">
|
|
||||||
剩余配额: {prediction.rate_limit_remaining}/10(分钟)
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
export function OutcomePanel({ prediction, match }: { prediction: Prediction; match: Match }) {
|
|
||||||
const homeName = match.home_team_zh || match.home_team
|
|
||||||
const awayName = match.away_team_zh || match.away_team
|
|
||||||
const degraded = prediction.status === 'degraded' || prediction.status === 'failed'
|
|
||||||
|
|
||||||
return (
|
|
||||||
<>
|
|
||||||
{!degraded && (
|
|
||||||
<div className="text-center">
|
|
||||||
<p className="font-serif text-5xl font-bold tabular-nums leading-none text-ink-900 sm:text-6xl">
|
|
||||||
{prediction.pred_home_goals ?? '-'}
|
|
||||||
<span className="mx-3 font-normal text-ink-300">:</span>
|
|
||||||
{prediction.pred_away_goals ?? '-'}
|
|
||||||
</p>
|
|
||||||
<p className="mt-3 text-2xs tracking-[0.5em] text-ink-400">预测比分</p>
|
|
||||||
{prediction.alt_pred_home_goals != null && prediction.alt_pred_away_goals != null && (
|
|
||||||
<p className="mt-2 text-2xs tabular-nums text-ink-400">
|
|
||||||
备选{' '}
|
|
||||||
<span className="font-serif text-sm font-bold tabular-nums text-ink-600">
|
|
||||||
{prediction.alt_pred_home_goals}<span className="mx-0.5 font-normal text-ink-300">:</span>{prediction.alt_pred_away_goals}
|
|
||||||
</span>
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!degraded && (
|
|
||||||
<div className="border-y border-ink-200 py-4">
|
|
||||||
<OutcomeLine pick={prediction.pred_1x2} confidence={prediction.subjective_confidence} />
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!degraded && <PredictionCost prediction={prediction} />}
|
|
||||||
</>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -1,34 +0,0 @@
|
|||||||
/**
|
|
||||||
* ReasoningPanel: 终裁意见 / 降级原因 —— 预测的文本解释。
|
|
||||||
*
|
|
||||||
* P3-1: 从 MatchPredictPanel.PredictionPanel 拆出,渲染逻辑原样搬迁。
|
|
||||||
*/
|
|
||||||
import type { Prediction } from '../../types'
|
|
||||||
|
|
||||||
export function ReasoningPanel({ prediction }: { prediction: Prediction }) {
|
|
||||||
const degraded = prediction.status === 'degraded' || prediction.status === 'failed'
|
|
||||||
|
|
||||||
if (!prediction.reasoning) return null
|
|
||||||
|
|
||||||
// 降级态:reasoning 展示为「降级原因」
|
|
||||||
if (degraded) {
|
|
||||||
return (
|
|
||||||
<section>
|
|
||||||
<h4 className="section-head mb-2">降级原因</h4>
|
|
||||||
<blockquote className="border-l-2 border-press pl-4">
|
|
||||||
<p className="whitespace-pre-wrap font-serif text-sm leading-loose text-ink-700">{prediction.reasoning}</p>
|
|
||||||
</blockquote>
|
|
||||||
</section>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// 成功态:reasoning 展示为「终裁意见」
|
|
||||||
return (
|
|
||||||
<section>
|
|
||||||
<h4 className="section-head mb-3">终裁意见</h4>
|
|
||||||
<blockquote className="border-l-2 border-press pl-4">
|
|
||||||
<p className="whitespace-pre-wrap font-serif text-sm leading-loose text-ink-700">{prediction.reasoning}</p>
|
|
||||||
</blockquote>
|
|
||||||
</section>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@@ -78,8 +78,6 @@ export const LEAGUES = [
|
|||||||
{ code: 'D1', name: '德甲' },
|
{ code: 'D1', name: '德甲' },
|
||||||
{ code: 'I1', name: '意甲' },
|
{ code: 'I1', name: '意甲' },
|
||||||
{ code: 'F1', name: '法甲' },
|
{ code: 'F1', name: '法甲' },
|
||||||
{ code: 'CL', name: '欧冠' },
|
|
||||||
{ code: 'EL', name: '欧联' },
|
|
||||||
]
|
]
|
||||||
|
|
||||||
/** 汉字编号,给专家意见排版用 */
|
/** 汉字编号,给专家意见排版用 */
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
|
|
||||||
@@ -41,24 +40,6 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
|||||||
"多 worker 部署请将限流前置到 Nginx/网关,或以单 worker 运行"
|
"多 worker 部署请将限流前置到 Nginx/网关,或以单 worker 运行"
|
||||||
)
|
)
|
||||||
|
|
||||||
# P3-3:STRICT_SINGLE_WORKER 启动期强制校验,拒绝多 worker 静默配额漂移。
|
|
||||||
# uvicorn 通过 --workers 传入;此处以环境变量 UVICORN_WORKERS 或启动参数判定。
|
|
||||||
# 为避免耦合 uvicorn 内部,仅校验一个显式传入的标记:当 STRICT_SINGLE_WORKER=True 时,
|
|
||||||
# 要求环境变量 UVICORN_WORKERS 不为空且 <=1,否则拒绝启动。
|
|
||||||
if settings.STRICT_SINGLE_WORKER:
|
|
||||||
workers = os.environ.get("UVICORN_WORKERS", "1")
|
|
||||||
try:
|
|
||||||
n_workers = int(workers)
|
|
||||||
except ValueError:
|
|
||||||
n_workers = 1
|
|
||||||
if n_workers > 1:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"STRICT_SINGLE_WORKER=True 但以 {n_workers} worker 启动会被拒绝 "
|
|
||||||
f"(应用内限流/KeyRing 多 worker 下各自独立计数,配额放大 {n_workers} 倍)。"
|
|
||||||
f"请前置 Nginx/网关全局限流后再启用多 worker,或保持单 worker。"
|
|
||||||
)
|
|
||||||
logger.info("STRICT_SINGLE_WORKER=True:已确认单 worker 启动,限流配额不会漂移")
|
|
||||||
|
|
||||||
# 注册默认定时任务(如果数据库中没有)
|
# 注册默认定时任务(如果数据库中没有)
|
||||||
from src.db.base import AsyncSessionLocal
|
from src.db.base import AsyncSessionLocal
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
|
|||||||
@@ -1,88 +0,0 @@
|
|||||||
"""后台管理:配置项 CRUD(settings)与运行日志查询。
|
|
||||||
|
|
||||||
所有接口需管理员鉴权(require_admin)。路由前缀 /api/v1/admin。
|
|
||||||
配置项白名单见 src/core/runtime_config.py SETTING_DEFS,之外的 key 一律拒绝。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
from pydantic import BaseModel
|
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
|
||||||
from src.core.log_buffer import get_entries
|
|
||||||
from src.core.runtime_config import (
|
|
||||||
SETTING_DEFS,
|
|
||||||
clear_runtime_value,
|
|
||||||
get_setting_origin,
|
|
||||||
mask_value,
|
|
||||||
set_runtime_value,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
|
||||||
|
|
||||||
|
|
||||||
class SettingUpdateIn(BaseModel):
|
|
||||||
value: str
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/settings")
|
|
||||||
async def list_settings():
|
|
||||||
"""全部可配置项(脱敏),供后台各配置页渲染。"""
|
|
||||||
out = []
|
|
||||||
for key, defn in SETTING_DEFS.items():
|
|
||||||
origin, value = await get_setting_origin(key)
|
|
||||||
out.append(
|
|
||||||
{
|
|
||||||
"key": key,
|
|
||||||
"label": defn.label,
|
|
||||||
"description": defn.description,
|
|
||||||
"sensitive": defn.sensitive,
|
|
||||||
"configured": origin != "none",
|
|
||||||
"masked": mask_value(value, defn.sensitive),
|
|
||||||
"origin": origin,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/logs")
|
|
||||||
async def read_logs(
|
|
||||||
level: str | None = Query(None, description="最低级别: DEBUG/INFO/WARNING/ERROR"),
|
|
||||||
keyword: str | None = Query(None, description="消息或 logger 关键字"),
|
|
||||||
limit: int = Query(200, ge=1, le=1000),
|
|
||||||
):
|
|
||||||
"""查询应用运行日志(内存环形缓冲,最新在前;进程重启后清零)。"""
|
|
||||||
entries = get_entries(level, keyword, limit)
|
|
||||||
return {"entries": entries, "count": len(entries)}
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/settings/{key}")
|
|
||||||
async def update_setting(key: str, body: SettingUpdateIn):
|
|
||||||
"""更新配置项(写入 app_settings 覆盖 .env)。传空值请改用 DELETE。"""
|
|
||||||
if key not in SETTING_DEFS:
|
|
||||||
raise HTTPException(404, f"不支持的配置项: {key}")
|
|
||||||
value = body.value.strip()
|
|
||||||
if not value:
|
|
||||||
raise HTTPException(400, "值不能为空;如需回落 .env 请调用清除接口")
|
|
||||||
await set_runtime_value(key, value)
|
|
||||||
defn = SETTING_DEFS[key]
|
|
||||||
return {"key": key, "masked": mask_value(value, defn.sensitive), "origin": "db"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.delete("/settings/{key}")
|
|
||||||
async def clear_setting(key: str):
|
|
||||||
"""清除 DB 覆盖值,回落 .env 默认。"""
|
|
||||||
if key not in SETTING_DEFS:
|
|
||||||
raise HTTPException(404, f"不支持的配置项: {key}")
|
|
||||||
await clear_runtime_value(key)
|
|
||||||
origin, value = await get_setting_origin(key)
|
|
||||||
defn = SETTING_DEFS[key]
|
|
||||||
return {
|
|
||||||
"key": key,
|
|
||||||
"masked": mask_value(value, defn.sensitive),
|
|
||||||
"origin": origin,
|
|
||||||
}
|
|
||||||
@@ -1,228 +0,0 @@
|
|||||||
"""后台管理:数据源列表/连通性测试、KeyRing 状态、采集健康概览。
|
|
||||||
|
|
||||||
所有接口需管理员鉴权(require_admin)。路由前缀 /api/v1/admin。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import time
|
|
||||||
from datetime import date, datetime
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
|
||||||
from src.core.config import settings
|
|
||||||
from src.core.http_client import get_client
|
|
||||||
from src.core.runtime_config import (
|
|
||||||
SETTING_DEFS,
|
|
||||||
get_runtime_value,
|
|
||||||
get_setting_origin,
|
|
||||||
mask_value,
|
|
||||||
)
|
|
||||||
from src.db.base import AsyncSession, get_db_read
|
|
||||||
from src.db.models import Match, MatchStats, Standing
|
|
||||||
from src.data.key_ring import get_key_ring
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
|
||||||
|
|
||||||
# ── 数据源元数据(bzzoiro 单一数据源) ────────────────────────────
|
|
||||||
|
|
||||||
_SOURCES: list[dict] = [
|
|
||||||
{
|
|
||||||
"name": "bzzoiro",
|
|
||||||
"label": "Bzzoiro",
|
|
||||||
"description": "唯一数据源:赛程比分 + 积分榜 + 比赛详细统计(xG/射门/控球等)",
|
|
||||||
"setting_keys": ["BZZOIRO_KEY", "BZZOIRO_BASE"],
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
async def _last_ingestion(db: AsyncSession, source: str) -> datetime | None:
|
|
||||||
"""源最近一次采集时间(取自数据血缘字段,无记录返回 None)。"""
|
|
||||||
return (
|
|
||||||
await db.execute(
|
|
||||||
select(func.max(MatchStats.retrieved_at)).where(MatchStats.source == source)
|
|
||||||
)
|
|
||||||
).scalar()
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/datasources")
|
|
||||||
async def list_datasources(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""数据源列表:各配置项的脱敏值、来源(db/env/none)与最近采集时间。"""
|
|
||||||
result = []
|
|
||||||
for src in _SOURCES:
|
|
||||||
settings_out = []
|
|
||||||
for key in src["setting_keys"]:
|
|
||||||
origin, value = await get_setting_origin(key)
|
|
||||||
defn = SETTING_DEFS[key]
|
|
||||||
settings_out.append(
|
|
||||||
{
|
|
||||||
"key": key,
|
|
||||||
"label": defn.label,
|
|
||||||
"description": defn.description,
|
|
||||||
"sensitive": defn.sensitive,
|
|
||||||
"configured": origin != "none",
|
|
||||||
"masked": mask_value(value, defn.sensitive),
|
|
||||||
"origin": origin,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
key_configured = all(s["configured"] for s in settings_out) if settings_out else True
|
|
||||||
last = await _last_ingestion(db, src["name"])
|
|
||||||
result.append(
|
|
||||||
{
|
|
||||||
"name": src["name"],
|
|
||||||
"label": src["label"],
|
|
||||||
"description": src["description"],
|
|
||||||
"key_configured": key_configured,
|
|
||||||
"last_ingestion": last.isoformat() if last else None,
|
|
||||||
"settings": settings_out,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ── 连通性测试 ──────────────────────────────────────────────────
|
|
||||||
|
|
||||||
_TEST_TIMEOUT = 15
|
|
||||||
|
|
||||||
|
|
||||||
async def _probe(url: str, headers: dict | None = None, params: dict | None = None) -> dict:
|
|
||||||
"""单次 HTTP 探测,返回 (ok, status, latency_ms, detail)。不重试。"""
|
|
||||||
client = get_client()
|
|
||||||
start = time.monotonic()
|
|
||||||
try:
|
|
||||||
resp = await client.get(url, headers=headers, params=params, timeout=_TEST_TIMEOUT)
|
|
||||||
except Exception as e:
|
|
||||||
return {
|
|
||||||
"ok": False,
|
|
||||||
"status": None,
|
|
||||||
"latency_ms": int((time.monotonic() - start) * 1000),
|
|
||||||
"detail": f"无法连接: {e}",
|
|
||||||
}
|
|
||||||
latency = int((time.monotonic() - start) * 1000)
|
|
||||||
status = resp.status_code
|
|
||||||
if status == 200:
|
|
||||||
detail = "连接成功"
|
|
||||||
elif status in (401, 403):
|
|
||||||
detail = "服务可达,但密钥无效或无权限"
|
|
||||||
else:
|
|
||||||
detail = f"服务返回 HTTP {status}"
|
|
||||||
return {"ok": status == 200, "status": status, "latency_ms": latency, "detail": detail}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/datasources/{name}/test")
|
|
||||||
async def test_datasource(name: str):
|
|
||||||
"""轻量连通性测试:真实请求上游一次,不触发任何入库。"""
|
|
||||||
src = next((s for s in _SOURCES if s["name"] == name), None)
|
|
||||||
if src is None:
|
|
||||||
raise HTTPException(404, f"未知数据源: {name}")
|
|
||||||
|
|
||||||
if name == "bzzoiro":
|
|
||||||
key = await get_runtime_value("BZZOIRO_KEY")
|
|
||||||
if not key:
|
|
||||||
return {"ok": False, "status": None, "latency_ms": 0, "detail": "BZZOIRO_KEY 未配置"}
|
|
||||||
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
|
||||||
today = date.today().isoformat()
|
|
||||||
return await _probe(
|
|
||||||
f"{base}/events/",
|
|
||||||
headers={"Authorization": f"Token {key}", "Accept": "application/json"},
|
|
||||||
params={"date_from": today, "date_to": today},
|
|
||||||
)
|
|
||||||
|
|
||||||
raise HTTPException(404, f"未知数据源: {name}")
|
|
||||||
|
|
||||||
|
|
||||||
# ── 数据源健康/最近采集状态(只读,不触发采集) ──────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/ingest/status")
|
|
||||||
async def ingest_status(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""数据源采集健康概览(bzzoiro 单源;只读,不触发任何采集)。"""
|
|
||||||
bzzoiro_key = await get_runtime_value("BZZOIRO_KEY")
|
|
||||||
bzzoiro_base = await get_runtime_value("BZZOIRO_BASE")
|
|
||||||
|
|
||||||
# 比赛覆盖
|
|
||||||
match_row = (
|
|
||||||
await db.execute(
|
|
||||||
select(
|
|
||||||
func.count().label("cnt"),
|
|
||||||
func.max(Match.match_date).label("latest_match_date"),
|
|
||||||
func.max(Match.created_at).label("latest_row_at"),
|
|
||||||
).where(Match.match_status == "finished")
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
# 统计覆盖(精确 retrieved_at)
|
|
||||||
stats_row = (
|
|
||||||
await db.execute(
|
|
||||||
select(
|
|
||||||
func.count().label("cnt"),
|
|
||||||
func.max(MatchStats.retrieved_at).label("latest_retrieved"),
|
|
||||||
).where(MatchStats.source == "bzzoiro")
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
# 积分榜覆盖
|
|
||||||
standings_row = (
|
|
||||||
await db.execute(select(func.count()).select_from(Standing))
|
|
||||||
).scalar()
|
|
||||||
|
|
||||||
bzzoiro = {
|
|
||||||
"name": "bzzoiro",
|
|
||||||
"label": "Bzzoiro",
|
|
||||||
"key_configured": bool(bzzoiro_key),
|
|
||||||
"base_url": (bzzoiro_base.rstrip("/") if bzzoiro_base else None) or settings.BZZOIRO_BASE,
|
|
||||||
"reachable": None, # 不主动探测
|
|
||||||
"last_success_at": (stats_row.latest_retrieved or match_row.latest_row_at),
|
|
||||||
"last_success_at_iso": (
|
|
||||||
stats_row.latest_retrieved or match_row.latest_row_at
|
|
||||||
).isoformat() if (stats_row.latest_retrieved or match_row.latest_row_at) else None,
|
|
||||||
"latest_match_date": match_row.latest_match_date.isoformat() if match_row.latest_match_date else None,
|
|
||||||
"recent_count": match_row.cnt or 0,
|
|
||||||
"stats_count": stats_row.cnt or 0,
|
|
||||||
"standings_count": standings_row or 0,
|
|
||||||
"note": "last_success_at 取 match_stats.retrieved_at(统计回填)与 matches.created_at(比赛行)的较大者",
|
|
||||||
"last_failure": _last_failure_log("bzzoiro"),
|
|
||||||
}
|
|
||||||
|
|
||||||
return {"sources": [bzzoiro]}
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/keyring/status")
|
|
||||||
async def keyring_status():
|
|
||||||
"""KeyRing 运行状态:当前使用的 key、冷却状态、轮转信息(供管理后台展示)。"""
|
|
||||||
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
|
||||||
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
|
||||||
ring = get_key_ring(base, raw_keys)
|
|
||||||
st = ring.stats()
|
|
||||||
st["base_url"] = base
|
|
||||||
st["cooldown_seconds"] = ring._cooldown
|
|
||||||
st["has_multiple"] = ring.has_multiple
|
|
||||||
st["active_key"] = ring.active_key
|
|
||||||
return st
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/keyring/cooldown/reset")
|
|
||||||
async def keyring_reset_cooldown():
|
|
||||||
"""手动重置所有 key 的冷却状态(用于紧急恢复)。"""
|
|
||||||
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
|
||||||
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
|
||||||
ring = get_key_ring(base, raw_keys)
|
|
||||||
ring._blocked_until.clear()
|
|
||||||
return {"ok": True, "message": "已重置所有 key 冷却状态", "stats": ring.stats()}
|
|
||||||
|
|
||||||
|
|
||||||
def _last_failure_log(source: str) -> dict | None:
|
|
||||||
"""从系统日志缓冲中查找某数据源的最近一次错误(仅作参考,非专用失败表)。"""
|
|
||||||
from src.core.log_buffer import get_entries
|
|
||||||
entries = get_entries(min_level="ERROR", keyword=source, limit=5)
|
|
||||||
if not entries:
|
|
||||||
return None
|
|
||||||
e = entries[0]
|
|
||||||
return {
|
|
||||||
"at": datetime.fromtimestamp(e["ts"]).isoformat(),
|
|
||||||
"logger": e["logger"],
|
|
||||||
"detail": e["message"][:200],
|
|
||||||
"note": "approx:来自内存日志缓冲,非专用采集失败表;进程重启后清零",
|
|
||||||
}
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
"""后台管理:采集任务状态查询(只读)。
|
|
||||||
|
|
||||||
GET /api/v1/admin/ingest/jobs/{job_id} — 单任务详情
|
|
||||||
GET /api/v1/admin/ingest/jobs?limit=N — 最近任务列表(默认 20)
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
|
||||||
from sqlalchemy import desc, select
|
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
|
||||||
from src.api.schemas import IngestJobOut
|
|
||||||
from src.db.base import AsyncSession, get_db_read
|
|
||||||
from src.db.models import IngestJob
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/ingest/jobs/{job_id}", response_model=IngestJobOut)
|
|
||||||
async def get_ingest_job(job_id: str, db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""查询单个采集任务状态。"""
|
|
||||||
job = await db.get(IngestJob, job_id)
|
|
||||||
if job is None:
|
|
||||||
raise HTTPException(404, f"采集任务不存在: {job_id}")
|
|
||||||
return _job_to_out(job)
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/ingest/jobs", response_model=list[IngestJobOut])
|
|
||||||
async def list_ingest_jobs(
|
|
||||||
limit: int = Query(20, ge=1, le=100, description="返回条数"),
|
|
||||||
db: AsyncSession = Depends(get_db_read),
|
|
||||||
):
|
|
||||||
"""查询最近采集任务(最新在前)。"""
|
|
||||||
rows = (
|
|
||||||
await db.execute(select(IngestJob).order_by(desc(IngestJob.created_at)).limit(limit))
|
|
||||||
).scalars().all()
|
|
||||||
return [_job_to_out(j) for j in rows]
|
|
||||||
|
|
||||||
|
|
||||||
def _job_to_out(job: IngestJob) -> IngestJobOut:
|
|
||||||
return IngestJobOut(
|
|
||||||
id=job.id,
|
|
||||||
task=job.task,
|
|
||||||
params=job.params or {},
|
|
||||||
status=job.status,
|
|
||||||
result=job.result,
|
|
||||||
error=job.error,
|
|
||||||
created_at=job.created_at,
|
|
||||||
started_at=job.started_at,
|
|
||||||
finished_at=job.finished_at,
|
|
||||||
)
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
"""后台管理:LLM 专家/终裁配置、可用模型探测、连通性测试。
|
|
||||||
|
|
||||||
所有接口需管理员鉴权(require_admin)。路由前缀 /api/v1/admin。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import time
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
from fastapi import APIRouter, Depends
|
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
|
||||||
from src.core.config import settings
|
|
||||||
from src.core.http_client import get_client
|
|
||||||
from src.core.runtime_config import (
|
|
||||||
AGENT_META,
|
|
||||||
SETTING_DEFS,
|
|
||||||
get_runtime_value,
|
|
||||||
get_setting_origin,
|
|
||||||
mask_value,
|
|
||||||
)
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/llm/agents")
|
|
||||||
async def list_llm_agents():
|
|
||||||
"""各专家/终裁的独立 LLM 配置状态(含当前生效模型的解析结果)。"""
|
|
||||||
out = []
|
|
||||||
for agent in AGENT_META:
|
|
||||||
aid = agent["id"].upper()
|
|
||||||
pfx = f"AGENT_{aid}_"
|
|
||||||
fields = {}
|
|
||||||
for suffix in ("MODEL", "BASE_URL", "API_KEY"):
|
|
||||||
origin, value = await get_setting_origin(f"{pfx}{suffix}")
|
|
||||||
defn = SETTING_DEFS[f"{pfx}{suffix}"]
|
|
||||||
fields[suffix.lower()] = {
|
|
||||||
"configured": origin != "none",
|
|
||||||
"masked": mask_value(value, defn.sensitive),
|
|
||||||
"origin": origin,
|
|
||||||
}
|
|
||||||
# 生效模型 = 覆盖 → 层级默认(专家/终裁 env) → 全局 LLM_MODEL
|
|
||||||
tier_default = (
|
|
||||||
settings.LLM_AGGREGATOR_MODEL if agent["id"] == "aggregator" else settings.LLM_SPECIALIST_MODEL
|
|
||||||
)
|
|
||||||
effective_model = (
|
|
||||||
fields["model"]["masked"]
|
|
||||||
if fields["model"]["configured"]
|
|
||||||
else (tier_default or await get_runtime_value("LLM_MODEL"))
|
|
||||||
)
|
|
||||||
out.append(
|
|
||||||
{
|
|
||||||
"id": agent["id"],
|
|
||||||
"label": agent["label"],
|
|
||||||
"fields": fields,
|
|
||||||
"effective_model": effective_model,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/llm/models")
|
|
||||||
async def list_llm_models():
|
|
||||||
"""探测当前 LLM 服务可用的模型列表(OpenAI 兼容 GET /models)。
|
|
||||||
|
|
||||||
只读探测,不产生费用;配置缺失或服务不可达时返回 ok=false 与原因。
|
|
||||||
"""
|
|
||||||
base_url = (await get_runtime_value("LLM_BASE_URL")).rstrip("/")
|
|
||||||
api_key = await get_runtime_value("LLM_API_KEY")
|
|
||||||
if not base_url or not api_key:
|
|
||||||
return {"ok": False, "models": [], "detail": "LLM_BASE_URL 或 LLM_API_KEY 未配置"}
|
|
||||||
|
|
||||||
client = get_client()
|
|
||||||
start = time.monotonic()
|
|
||||||
try:
|
|
||||||
resp = await client.get(
|
|
||||||
f"{base_url}/models",
|
|
||||||
headers={"Authorization": f"Bearer {api_key}"},
|
|
||||||
timeout=httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0),
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
return {
|
|
||||||
"ok": False,
|
|
||||||
"models": [],
|
|
||||||
"latency_ms": int((time.monotonic() - start) * 1000),
|
|
||||||
"detail": f"无法连接 LLM 服务: {e}",
|
|
||||||
}
|
|
||||||
|
|
||||||
latency = int((time.monotonic() - start) * 1000)
|
|
||||||
if resp.status_code in (401, 403):
|
|
||||||
return {"ok": False, "models": [], "latency_ms": latency, "detail": "密钥无效或无权限(HTTP 401/403)"}
|
|
||||||
if resp.status_code != 200:
|
|
||||||
return {"ok": False, "models": [], "latency_ms": latency, "detail": f"服务返回 HTTP {resp.status_code}"}
|
|
||||||
|
|
||||||
try:
|
|
||||||
data = resp.json()
|
|
||||||
except Exception:
|
|
||||||
return {"ok": False, "models": [], "latency_ms": latency, "detail": "响应不是合法 JSON"}
|
|
||||||
|
|
||||||
models: list[str] = []
|
|
||||||
items = data.get("data") if isinstance(data, dict) else None
|
|
||||||
if isinstance(items, list):
|
|
||||||
models = sorted(
|
|
||||||
str(m.get("id")) for m in items if isinstance(m, dict) and m.get("id")
|
|
||||||
)
|
|
||||||
if not models:
|
|
||||||
return {"ok": False, "models": [], "latency_ms": latency, "detail": "服务未返回模型列表"}
|
|
||||||
return {"ok": True, "models": models, "latency_ms": latency, "detail": f"共 {len(models)} 个可用模型"}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/llm/ping")
|
|
||||||
async def llm_ping():
|
|
||||||
"""LLM 连通性测试(不依赖比赛)。只发一次 chat 请求验证配置。"""
|
|
||||||
from src.llm.provider import get_default_provider
|
|
||||||
p = await get_default_provider()
|
|
||||||
resp = await p.chat(
|
|
||||||
system="你是测试助手。",
|
|
||||||
user="ping",
|
|
||||||
max_tokens=10,
|
|
||||||
)
|
|
||||||
if resp.error:
|
|
||||||
return {"ok": False, "message": resp.error}
|
|
||||||
return {"ok": True, "message": "LLM 连接正常", "model": p.model}
|
|
||||||
@@ -1,343 +0,0 @@
|
|||||||
"""后台管理:管理区统计、数据完整性分析、数据质量检查。
|
|
||||||
|
|
||||||
所有接口需管理员鉴权(require_admin)。路由前缀 /api/v1/admin。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
|
||||||
from sqlalchemy import func, select
|
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
|
||||||
from src.db.base import AsyncSession, get_db_read
|
|
||||||
from src.db.models import DataQualityCheck, IngestFailure, League, Match, MatchStats, Prediction, Standing
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/stats")
|
|
||||||
async def admin_stats(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""管理区统计(只读):预测次数 + 比赛覆盖。轻量聚合,无 LLM 调用。"""
|
|
||||||
day_ago = datetime.now(timezone.utc) - timedelta(days=1)
|
|
||||||
week_ago = datetime.now(timezone.utc) - timedelta(days=7)
|
|
||||||
r = (
|
|
||||||
await db.execute(
|
|
||||||
select(
|
|
||||||
func.count().label("total"),
|
|
||||||
func.count().filter(Prediction.created_at >= day_ago).label("last_24h"),
|
|
||||||
func.count().filter(Prediction.created_at >= week_ago).label("last_7d"),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
# F3 修复: 补充真实比赛计数(非 limit=100 近似)
|
|
||||||
match_cnt = (await db.execute(select(func.count()).select_from(Match))).scalar() or 0
|
|
||||||
finished_cnt = (await db.execute(select(func.count()).where(Match.match_status == "finished"))).scalar() or 0
|
|
||||||
stats_cnt = (await db.execute(select(func.count()).select_from(MatchStats))).scalar() or 0
|
|
||||||
standings_cnt = (await db.execute(select(func.count()).select_from(Standing))).scalar() or 0
|
|
||||||
return {
|
|
||||||
"predictions": {"total": r.total, "last_24h": r.last_24h, "last_7d": r.last_7d},
|
|
||||||
"matches": {"total": match_cnt, "finished": finished_cnt},
|
|
||||||
"stats": {"total": stats_cnt},
|
|
||||||
"standings": {"total": standings_cnt},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ── 数据完整性分析(可视化数据源) ────────────────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/data-completeness")
|
|
||||||
async def data_completeness(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""按联赛统计数据完整性:比赛覆盖、字段覆盖、积分榜覆盖。
|
|
||||||
|
|
||||||
前端「数据完整性」页据此渲染,回答三个问题:
|
|
||||||
1. 数据是否齐全(各联赛比赛/统计/积分榜量级)
|
|
||||||
2. 字段是否齐全(每张统计表各字段非空率)
|
|
||||||
3. 覆盖是否新鲜(最近一场/最近一次采集)
|
|
||||||
"""
|
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_NAMES, LEAGUE_COUNTRIES
|
|
||||||
|
|
||||||
out_leagues: list[dict] = []
|
|
||||||
for code, bzz_id in BZZOIRO_LEAGUE_IDS.items():
|
|
||||||
# 比赛覆盖
|
|
||||||
m = (
|
|
||||||
await db.execute(
|
|
||||||
select(
|
|
||||||
func.count().label("total"),
|
|
||||||
func.count().filter(Match.match_status == "finished").label("finished"),
|
|
||||||
func.count().filter(Match.match_status == "scheduled").label("scheduled"),
|
|
||||||
func.count().filter(Match.source_event_id.is_not(None)).label("with_source_id"),
|
|
||||||
func.max(Match.match_date).label("latest_match"),
|
|
||||||
func.min(Match.match_date).label("earliest_match"),
|
|
||||||
)
|
|
||||||
.select_from(Match)
|
|
||||||
.join(League, League.id == Match.league_id)
|
|
||||||
.where(League.code == code)
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
# 统计字段覆盖(联表 matches)
|
|
||||||
s = (
|
|
||||||
await db.execute(
|
|
||||||
select(
|
|
||||||
func.count().label("rows"),
|
|
||||||
func.count(MatchStats.home_xg).label("xg"),
|
|
||||||
func.count(MatchStats.home_shots).label("shots"),
|
|
||||||
func.count(MatchStats.home_possession).label("possession"),
|
|
||||||
func.count(MatchStats.home_corners).label("corners"),
|
|
||||||
func.count(MatchStats.home_fouls).label("fouls"),
|
|
||||||
func.count(MatchStats.home_big_chances).label("big_chances"),
|
|
||||||
func.count(MatchStats.home_yellow_cards).label("cards"),
|
|
||||||
)
|
|
||||||
.select_from(MatchStats)
|
|
||||||
.join(Match, Match.id == MatchStats.match_id)
|
|
||||||
.join(League, League.id == Match.league_id)
|
|
||||||
.where(League.code == code)
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
# 积分榜覆盖
|
|
||||||
st = (
|
|
||||||
await db.execute(
|
|
||||||
select(
|
|
||||||
func.count().label("rows"),
|
|
||||||
func.max(Standing.retrieved_at).label("latest_retrieved"),
|
|
||||||
)
|
|
||||||
.select_from(Standing)
|
|
||||||
.join(League, League.id == Standing.league_id)
|
|
||||||
.where(League.code == code)
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
|
|
||||||
stats_rows = s.rows or 0
|
|
||||||
pct = lambda n: round(n / stats_rows * 100, 1) if stats_rows else 0.0 # noqa: E731
|
|
||||||
out_leagues.append(
|
|
||||||
{
|
|
||||||
"code": code,
|
|
||||||
"name": LEAGUE_NAMES.get(code, code),
|
|
||||||
"country": LEAGUE_COUNTRIES.get(code),
|
|
||||||
"matches": {
|
|
||||||
"total": m.total or 0,
|
|
||||||
"finished": m.finished or 0,
|
|
||||||
"scheduled": m.scheduled or 0,
|
|
||||||
"with_source_id": m.with_source_id or 0,
|
|
||||||
"earliest_match": m.earliest_match.isoformat() if m.earliest_match else None,
|
|
||||||
"latest_match": m.latest_match.isoformat() if m.latest_match else None,
|
|
||||||
},
|
|
||||||
"stats": {
|
|
||||||
"rows": stats_rows,
|
|
||||||
"fields": {
|
|
||||||
"xg": {"count": s.xg or 0, "pct": pct(s.xg or 0)},
|
|
||||||
"shots": {"count": s.shots or 0, "pct": pct(s.shots or 0)},
|
|
||||||
"possession": {"count": s.possession or 0, "pct": pct(s.possession or 0)},
|
|
||||||
"corners": {"count": s.corners or 0, "pct": pct(s.corners or 0)},
|
|
||||||
"fouls": {"count": s.fouls or 0, "pct": pct(s.fouls or 0)},
|
|
||||||
"big_chances": {"count": s.big_chances or 0, "pct": pct(s.big_chances or 0)},
|
|
||||||
"cards": {"count": s.cards or 0, "pct": pct(s.cards or 0)},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"standings": {
|
|
||||||
"rows": st.rows or 0,
|
|
||||||
"latest_retrieved": st.latest_retrieved.isoformat() if st.latest_retrieved else None,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# 整体健康信号
|
|
||||||
total_finished = sum(l["matches"]["finished"] for l in out_leagues)
|
|
||||||
total_stats = sum(l["stats"]["rows"] for l in out_leagues)
|
|
||||||
stats_coverage = round(total_stats / total_finished * 100, 1) if total_finished else 0.0
|
|
||||||
issues: list[str] = []
|
|
||||||
for l in out_leagues:
|
|
||||||
if l["matches"]["finished"] == 0:
|
|
||||||
issues.append(f"{l['name']}: 无已完赛比赛,请先运行「比赛数据」采集")
|
|
||||||
elif l["stats"]["rows"] == 0:
|
|
||||||
issues.append(f"{l['name']}: 已完赛 {l['matches']['finished']} 场但无统计回填,请运行「统计回填」采集")
|
|
||||||
elif stats_coverage < 80:
|
|
||||||
issues.append(f"{l['name']}: 统计覆盖率仅 {stats_coverage}%,建议增量回填")
|
|
||||||
if l["standings"]["rows"] == 0:
|
|
||||||
issues.append(f"{l['name']}: 无积分榜数据,请运行「积分榜」采集")
|
|
||||||
if not issues:
|
|
||||||
issues.append("各联赛数据完整度良好")
|
|
||||||
|
|
||||||
return {
|
|
||||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
||||||
"leagues": out_leagues,
|
|
||||||
"totals": {
|
|
||||||
"finished_matches": total_finished,
|
|
||||||
"stats_rows": total_stats,
|
|
||||||
"stats_coverage_pct": stats_coverage,
|
|
||||||
},
|
|
||||||
"issues": issues,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# ── 数据质量检查 API ────────────────────────────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/data-quality")
|
|
||||||
async def data_quality_checks(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""数据质量检查结果(只读)。"""
|
|
||||||
# 最近的失败记录
|
|
||||||
failures = (
|
|
||||||
await db.execute(
|
|
||||||
select(IngestFailure)
|
|
||||||
.where(IngestFailure.status.in_(["pending", "retrying"]))
|
|
||||||
.order_by(IngestFailure.created_at.desc())
|
|
||||||
.limit(20)
|
|
||||||
)
|
|
||||||
).scalars().all()
|
|
||||||
|
|
||||||
# 最近的质量检查
|
|
||||||
checks = (
|
|
||||||
await db.execute(
|
|
||||||
select(DataQualityCheck)
|
|
||||||
.order_by(DataQualityCheck.checked_at.desc())
|
|
||||||
.limit(20)
|
|
||||||
)
|
|
||||||
).scalars().all()
|
|
||||||
|
|
||||||
return {
|
|
||||||
"failures": [
|
|
||||||
{
|
|
||||||
"id": f.id,
|
|
||||||
"source": f.source_system,
|
|
||||||
"entity_type": f.entity_type,
|
|
||||||
"source_record_id": f.source_record_id,
|
|
||||||
"error_type": f.error_type,
|
|
||||||
"error_detail": f.error_detail,
|
|
||||||
"retry_count": f.retry_count,
|
|
||||||
"status": f.status,
|
|
||||||
"created_at": f.created_at.isoformat() if f.created_at else None,
|
|
||||||
}
|
|
||||||
for f in failures
|
|
||||||
],
|
|
||||||
"checks": [
|
|
||||||
{
|
|
||||||
"id": c.id,
|
|
||||||
"check_name": c.check_name,
|
|
||||||
"entity_type": c.entity_type,
|
|
||||||
"passed": c.passed,
|
|
||||||
"severity": c.severity,
|
|
||||||
"detail": c.detail,
|
|
||||||
"checked_at": c.checked_at.isoformat() if c.checked_at else None,
|
|
||||||
}
|
|
||||||
for c in checks
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/data-quality/run")
|
|
||||||
async def run_data_quality_check(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""手动触发一次数据质量检查。"""
|
|
||||||
checks = []
|
|
||||||
|
|
||||||
# 检查1: 已完赛但无统计的比赛
|
|
||||||
finished_no_stats = (
|
|
||||||
await db.execute(
|
|
||||||
select(func.count())
|
|
||||||
.select_from(Match)
|
|
||||||
.outerjoin(MatchStats, Match.id == MatchStats.match_id)
|
|
||||||
.where(Match.match_status == "finished")
|
|
||||||
.where(MatchStats.id.is_(None))
|
|
||||||
)
|
|
||||||
).scalar() or 0
|
|
||||||
|
|
||||||
checks.append(DataQualityCheck(
|
|
||||||
check_name="finished_without_stats",
|
|
||||||
entity_type="match",
|
|
||||||
actual_value=float(finished_no_stats),
|
|
||||||
passed=finished_no_stats == 0,
|
|
||||||
severity="warning" if finished_no_stats > 0 else "info",
|
|
||||||
detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
|
|
||||||
))
|
|
||||||
|
|
||||||
# 检查2: 积分榜缺失的联赛
|
|
||||||
leagues_without_standings = (
|
|
||||||
await db.execute(
|
|
||||||
select(func.count())
|
|
||||||
.select_from(League)
|
|
||||||
.outerjoin(Standing, League.id == Standing.league_id)
|
|
||||||
.where(Standing.id.is_(None))
|
|
||||||
)
|
|
||||||
).scalar() or 0
|
|
||||||
|
|
||||||
checks.append(DataQualityCheck(
|
|
||||||
check_name="league_without_standings",
|
|
||||||
entity_type="league",
|
|
||||||
actual_value=float(leagues_without_standings),
|
|
||||||
passed=leagues_without_standings == 0,
|
|
||||||
severity="warning" if leagues_without_standings > 0 else "info",
|
|
||||||
detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
|
|
||||||
))
|
|
||||||
|
|
||||||
for c in checks:
|
|
||||||
db.add(c)
|
|
||||||
await db.commit()
|
|
||||||
|
|
||||||
return {"ok": True, "checks": [{"name": c.check_name, "passed": c.passed} for c in checks]}
|
|
||||||
|
|
||||||
|
|
||||||
# ── 近似重名候选(只读,启发式,不做自动合并) ──────────────────────
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/team-name-duplicates")
|
|
||||||
async def team_name_duplicates(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""只读列出近似重名候选(大小写变体/子串包含/前缀碰撞)。
|
|
||||||
|
|
||||||
启发式规则(命中任一即列为候选):
|
|
||||||
- 大小写变体: lower(name) 相同但 name 不同
|
|
||||||
- 子串包含: A 是 B 的子串且 len(A) ≥ 5
|
|
||||||
- 前缀碰撞: 前 8 字符相同(忽略大小写)
|
|
||||||
|
|
||||||
仅作排查参考,合并需走人工 SQL(见 docs/05-data.md)。
|
|
||||||
"""
|
|
||||||
teams = (await db.execute(select(Team.id, Team.name))).all()
|
|
||||||
by_lower: dict[str, list[dict]] = {}
|
|
||||||
for t in teams:
|
|
||||||
key = (t.name or "").lower()
|
|
||||||
by_lower.setdefault(key, []).append({"id": t.id, "name": t.name})
|
|
||||||
|
|
||||||
groups: list[dict] = []
|
|
||||||
|
|
||||||
# 规则1: 大小写变体(lower 相同但原名不同)
|
|
||||||
for key, members in by_lower.items():
|
|
||||||
if len(members) > 1:
|
|
||||||
groups.append({
|
|
||||||
"rule": "case_variant",
|
|
||||||
"key": key,
|
|
||||||
"members": members,
|
|
||||||
})
|
|
||||||
|
|
||||||
# 规则2 & 3: 子串包含 / 前缀碰撞(仅在 lower 名不同的组间比较)
|
|
||||||
distinct = [m for members in by_lower.values() for m in members]
|
|
||||||
seen_pairs: set[tuple[int, int]] = set()
|
|
||||||
for i, a in enumerate(distinct):
|
|
||||||
na = (a["name"] or "").lower()
|
|
||||||
for b in distinct[i + 1:]:
|
|
||||||
nb = (b["name"] or "").lower()
|
|
||||||
if na == nb:
|
|
||||||
continue # 已被规则1覆盖
|
|
||||||
pair = (min(a["id"], b["id"]), max(a["id"], b["id"]))
|
|
||||||
if pair in seen_pairs:
|
|
||||||
continue
|
|
||||||
hit = None
|
|
||||||
if len(na) >= 5 and na in nb:
|
|
||||||
hit = "substring"
|
|
||||||
elif len(nb) >= 5 and nb in na:
|
|
||||||
hit = "substring"
|
|
||||||
elif len(na) >= 8 and len(nb) >= 8 and na[:8] == nb[:8]:
|
|
||||||
hit = "prefix"
|
|
||||||
if hit:
|
|
||||||
seen_pairs.add(pair)
|
|
||||||
groups.append({
|
|
||||||
"rule": hit,
|
|
||||||
"members": [a, b],
|
|
||||||
})
|
|
||||||
|
|
||||||
return {
|
|
||||||
"count": len(groups),
|
|
||||||
"hint": "命中任一启发式仅表示'可疑',合并前请人工确认是否同一球队",
|
|
||||||
"groups": groups,
|
|
||||||
}
|
|
||||||
@@ -1,29 +1,668 @@
|
|||||||
"""后台管理路由聚合入口:按职责拆分为四个子模块,统一挂载。
|
"""后台管理路由:数据源配置的查看、修改与连通性测试。
|
||||||
|
|
||||||
所有路由仍挂在 /api/v1/admin,且均带 dependencies=[Depends(require_admin)]
|
所有接口需管理员鉴权(require_admin)。配置项白名单见
|
||||||
(鉴权由各子路由器声明,行为与拆分前完全一致)。
|
src/core/runtime_config.py SETTING_DEFS,之外的 key 一律拒绝。
|
||||||
|
|
||||||
子模块:
|
|
||||||
- admin_datasources 数据源列表/连通性测试、KeyRing、采集健康概览
|
|
||||||
- admin_config settings CRUD、运行日志
|
|
||||||
- admin_llm LLM agents/models/ping
|
|
||||||
- admin_quality stats、data-completeness、data-quality
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter
|
import logging
|
||||||
|
import time
|
||||||
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
|
||||||
from src.api.routes.admin_config import router as admin_config_router
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
from src.api.routes.admin_datasources import router as admin_datasources_router
|
from pydantic import BaseModel
|
||||||
from src.api.routes.admin_ingest_jobs import router as admin_ingest_jobs_router
|
from sqlalchemy import func, select
|
||||||
from src.api.routes.admin_llm import router as admin_llm_router
|
import httpx
|
||||||
from src.api.routes.admin_quality import router as admin_quality_router
|
|
||||||
from src.api.routes.admin_teams import router as admin_teams_router
|
|
||||||
|
|
||||||
router = APIRouter()
|
from src.api.deps import require_admin
|
||||||
router.include_router(admin_datasources_router)
|
from src.core.config import settings
|
||||||
router.include_router(admin_config_router)
|
from src.core.http_client import get_client
|
||||||
router.include_router(admin_ingest_jobs_router)
|
from src.core.log_buffer import get_entries
|
||||||
router.include_router(admin_llm_router)
|
from src.core.runtime_config import (
|
||||||
router.include_router(admin_quality_router)
|
AGENT_META,
|
||||||
router.include_router(admin_teams_router)
|
SETTING_DEFS,
|
||||||
|
clear_runtime_value,
|
||||||
|
get_runtime_value,
|
||||||
|
get_setting_origin,
|
||||||
|
mask_value,
|
||||||
|
set_runtime_value,
|
||||||
|
)
|
||||||
|
from src.db.base import AsyncSession, get_db_read
|
||||||
|
from src.db.models import League, Match, MatchStats, Standing
|
||||||
|
from src.data.key_ring import get_key_ring, parse_keys
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
||||||
|
|
||||||
|
# ── 数据源元数据(bzzoiro 单一数据源) ────────────────────────────
|
||||||
|
|
||||||
|
_SOURCES: list[dict] = [
|
||||||
|
{
|
||||||
|
"name": "bzzoiro",
|
||||||
|
"label": "Bzzoiro",
|
||||||
|
"description": "唯一数据源:赛程比分 + 积分榜 + 比赛详细统计(xG/射门/控球等)",
|
||||||
|
"setting_keys": ["BZZOIRO_KEY", "BZZOIRO_BASE"],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class SettingUpdateIn(BaseModel):
|
||||||
|
value: str
|
||||||
|
|
||||||
|
|
||||||
|
async def _last_ingestion(db: AsyncSession, source: str) -> datetime | None:
|
||||||
|
"""源最近一次采集时间(取自数据血缘字段,无记录返回 None)。"""
|
||||||
|
return (
|
||||||
|
await db.execute(
|
||||||
|
select(func.max(MatchStats.retrieved_at)).where(MatchStats.source == source)
|
||||||
|
)
|
||||||
|
).scalar()
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/datasources")
|
||||||
|
async def list_datasources(db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""数据源列表:各配置项的脱敏值、来源(db/env/none)与最近采集时间。"""
|
||||||
|
result = []
|
||||||
|
for src in _SOURCES:
|
||||||
|
settings_out = []
|
||||||
|
for key in src["setting_keys"]:
|
||||||
|
origin, value = await get_setting_origin(key)
|
||||||
|
defn = SETTING_DEFS[key]
|
||||||
|
settings_out.append(
|
||||||
|
{
|
||||||
|
"key": key,
|
||||||
|
"label": defn.label,
|
||||||
|
"description": defn.description,
|
||||||
|
"sensitive": defn.sensitive,
|
||||||
|
"configured": origin != "none",
|
||||||
|
"masked": mask_value(value, defn.sensitive),
|
||||||
|
"origin": origin,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
key_configured = all(s["configured"] for s in settings_out) if settings_out else True
|
||||||
|
last = await _last_ingestion(db, src["name"])
|
||||||
|
result.append(
|
||||||
|
{
|
||||||
|
"name": src["name"],
|
||||||
|
"label": src["label"],
|
||||||
|
"description": src["description"],
|
||||||
|
"key_configured": key_configured,
|
||||||
|
"last_ingestion": last.isoformat() if last else None,
|
||||||
|
"settings": settings_out,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/settings")
|
||||||
|
async def list_settings():
|
||||||
|
"""全部可配置项(脱敏),供后台各配置页渲染。"""
|
||||||
|
out = []
|
||||||
|
for key, defn in SETTING_DEFS.items():
|
||||||
|
origin, value = await get_setting_origin(key)
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"key": key,
|
||||||
|
"label": defn.label,
|
||||||
|
"description": defn.description,
|
||||||
|
"sensitive": defn.sensitive,
|
||||||
|
"configured": origin != "none",
|
||||||
|
"masked": mask_value(value, defn.sensitive),
|
||||||
|
"origin": origin,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ── LLM 可用模型检测 ────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/logs")
|
||||||
|
async def read_logs(
|
||||||
|
level: str | None = Query(None, description="最低级别: DEBUG/INFO/WARNING/ERROR"),
|
||||||
|
keyword: str | None = Query(None, description="消息或 logger 关键字"),
|
||||||
|
limit: int = Query(200, ge=1, le=1000),
|
||||||
|
):
|
||||||
|
"""查询应用运行日志(内存环形缓冲,最新在前;进程重启后清零)。"""
|
||||||
|
entries = get_entries(level, keyword, limit)
|
||||||
|
return {"entries": entries, "count": len(entries)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/llm/agents")
|
||||||
|
async def list_llm_agents():
|
||||||
|
"""各专家/终裁的独立 LLM 配置状态(含当前生效模型的解析结果)。"""
|
||||||
|
out = []
|
||||||
|
for agent in AGENT_META:
|
||||||
|
aid = agent["id"].upper()
|
||||||
|
pfx = f"AGENT_{aid}_"
|
||||||
|
fields = {}
|
||||||
|
for suffix in ("MODEL", "BASE_URL", "API_KEY"):
|
||||||
|
origin, value = await get_setting_origin(f"{pfx}{suffix}")
|
||||||
|
defn = SETTING_DEFS[f"{pfx}{suffix}"]
|
||||||
|
fields[suffix.lower()] = {
|
||||||
|
"configured": origin != "none",
|
||||||
|
"masked": mask_value(value, defn.sensitive),
|
||||||
|
"origin": origin,
|
||||||
|
}
|
||||||
|
# 生效模型 = 覆盖 → 层级默认(专家/终裁 env) → 全局 LLM_MODEL
|
||||||
|
tier_default = (
|
||||||
|
settings.LLM_AGGREGATOR_MODEL if agent["id"] == "aggregator" else settings.LLM_SPECIALIST_MODEL
|
||||||
|
)
|
||||||
|
effective_model = (
|
||||||
|
fields["model"]["masked"]
|
||||||
|
if fields["model"]["configured"]
|
||||||
|
else (tier_default or await get_runtime_value("LLM_MODEL"))
|
||||||
|
)
|
||||||
|
out.append(
|
||||||
|
{
|
||||||
|
"id": agent["id"],
|
||||||
|
"label": agent["label"],
|
||||||
|
"fields": fields,
|
||||||
|
"effective_model": effective_model,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/llm/models")
|
||||||
|
async def list_llm_models():
|
||||||
|
"""探测当前 LLM 服务可用的模型列表(OpenAI 兼容 GET /models)。
|
||||||
|
|
||||||
|
只读探测,不产生费用;配置缺失或服务不可达时返回 ok=false 与原因。
|
||||||
|
"""
|
||||||
|
base_url = (await get_runtime_value("LLM_BASE_URL")).rstrip("/")
|
||||||
|
api_key = await get_runtime_value("LLM_API_KEY")
|
||||||
|
if not base_url or not api_key:
|
||||||
|
return {"ok": False, "models": [], "detail": "LLM_BASE_URL 或 LLM_API_KEY 未配置"}
|
||||||
|
|
||||||
|
client = get_client()
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
resp = await client.get(
|
||||||
|
f"{base_url}/models",
|
||||||
|
headers={"Authorization": f"Bearer {api_key}"},
|
||||||
|
timeout=httpx.Timeout(connect=10.0, read=20.0, write=10.0, pool=10.0),
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"models": [],
|
||||||
|
"latency_ms": int((time.monotonic() - start) * 1000),
|
||||||
|
"detail": f"无法连接 LLM 服务: {e}",
|
||||||
|
}
|
||||||
|
|
||||||
|
latency = int((time.monotonic() - start) * 1000)
|
||||||
|
if resp.status_code in (401, 403):
|
||||||
|
return {"ok": False, "models": [], "latency_ms": latency, "detail": "密钥无效或无权限(HTTP 401/403)"}
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return {"ok": False, "models": [], "latency_ms": latency, "detail": f"服务返回 HTTP {resp.status_code}"}
|
||||||
|
|
||||||
|
try:
|
||||||
|
data = resp.json()
|
||||||
|
except Exception:
|
||||||
|
return {"ok": False, "models": [], "latency_ms": latency, "detail": "响应不是合法 JSON"}
|
||||||
|
|
||||||
|
models: list[str] = []
|
||||||
|
items = data.get("data") if isinstance(data, dict) else None
|
||||||
|
if isinstance(items, list):
|
||||||
|
models = sorted(
|
||||||
|
str(m.get("id")) for m in items if isinstance(m, dict) and m.get("id")
|
||||||
|
)
|
||||||
|
if not models:
|
||||||
|
return {"ok": False, "models": [], "latency_ms": latency, "detail": "服务未返回模型列表"}
|
||||||
|
return {"ok": True, "models": models, "latency_ms": latency, "detail": f"共 {len(models)} 个可用模型"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/settings/{key}")
|
||||||
|
async def update_setting(key: str, body: SettingUpdateIn):
|
||||||
|
"""更新配置项(写入 app_settings 覆盖 .env)。传空值请改用 DELETE。"""
|
||||||
|
if key not in SETTING_DEFS:
|
||||||
|
raise HTTPException(404, f"不支持的配置项: {key}")
|
||||||
|
value = body.value.strip()
|
||||||
|
if not value:
|
||||||
|
raise HTTPException(400, "值不能为空;如需回落 .env 请调用清除接口")
|
||||||
|
await set_runtime_value(key, value)
|
||||||
|
defn = SETTING_DEFS[key]
|
||||||
|
return {"key": key, "masked": mask_value(value, defn.sensitive), "origin": "db"}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/settings/{key}")
|
||||||
|
async def clear_setting(key: str):
|
||||||
|
"""清除 DB 覆盖值,回落 .env 默认。"""
|
||||||
|
if key not in SETTING_DEFS:
|
||||||
|
raise HTTPException(404, f"不支持的配置项: {key}")
|
||||||
|
await clear_runtime_value(key)
|
||||||
|
origin, value = await get_setting_origin(key)
|
||||||
|
defn = SETTING_DEFS[key]
|
||||||
|
return {
|
||||||
|
"key": key,
|
||||||
|
"masked": mask_value(value, defn.sensitive),
|
||||||
|
"origin": origin,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── 连通性测试 ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
_TEST_TIMEOUT = 15
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe(url: str, headers: dict | None = None, params: dict | None = None) -> dict:
|
||||||
|
"""单次 HTTP 探测,返回 (ok, status, latency_ms, detail)。不重试。"""
|
||||||
|
client = get_client()
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
resp = await client.get(url, headers=headers, params=params, timeout=_TEST_TIMEOUT)
|
||||||
|
except Exception as e:
|
||||||
|
return {
|
||||||
|
"ok": False,
|
||||||
|
"status": None,
|
||||||
|
"latency_ms": int((time.monotonic() - start) * 1000),
|
||||||
|
"detail": f"无法连接: {e}",
|
||||||
|
}
|
||||||
|
latency = int((time.monotonic() - start) * 1000)
|
||||||
|
status = resp.status_code
|
||||||
|
if status == 200:
|
||||||
|
detail = "连接成功"
|
||||||
|
elif status in (401, 403):
|
||||||
|
detail = "服务可达,但密钥无效或无权限"
|
||||||
|
else:
|
||||||
|
detail = f"服务返回 HTTP {status}"
|
||||||
|
return {"ok": status == 200, "status": status, "latency_ms": latency, "detail": detail}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/datasources/{name}/test")
|
||||||
|
async def test_datasource(name: str):
|
||||||
|
"""轻量连通性测试:真实请求上游一次,不触发任何入库。"""
|
||||||
|
src = next((s for s in _SOURCES if s["name"] == name), None)
|
||||||
|
if src is None:
|
||||||
|
raise HTTPException(404, f"未知数据源: {name}")
|
||||||
|
|
||||||
|
if name == "bzzoiro":
|
||||||
|
key = await get_runtime_value("BZZOIRO_KEY")
|
||||||
|
if not key:
|
||||||
|
return {"ok": False, "status": None, "latency_ms": 0, "detail": "BZZOIRO_KEY 未配置"}
|
||||||
|
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
||||||
|
today = date.today().isoformat()
|
||||||
|
return await _probe(
|
||||||
|
f"{base}/events/",
|
||||||
|
headers={"Authorization": f"Token {key}", "Accept": "application/json"},
|
||||||
|
params={"date_from": today, "date_to": today},
|
||||||
|
)
|
||||||
|
|
||||||
|
raise HTTPException(404, f"未知数据源: {name}")
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/llm/ping")
|
||||||
|
async def llm_ping():
|
||||||
|
"""LLM 连通性测试(不依赖比赛)。只发一次 chat 请求验证配置。"""
|
||||||
|
from src.llm.provider import get_default_provider
|
||||||
|
p = await get_default_provider()
|
||||||
|
resp = await p.chat(
|
||||||
|
system="你是测试助手。",
|
||||||
|
user="ping",
|
||||||
|
max_tokens=10,
|
||||||
|
)
|
||||||
|
if resp.error:
|
||||||
|
return {"ok": False, "message": resp.error}
|
||||||
|
return {"ok": True, "message": "LLM 连接正常", "model": p.model}
|
||||||
|
|
||||||
|
|
||||||
|
# ── 数据源健康/最近采集状态(只读,不触发采集) ──────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/ingest/status")
|
||||||
|
async def ingest_status(db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""数据源采集健康概览(bzzoiro 单源;只读,不触发任何采集)。"""
|
||||||
|
bzzoiro_key = await get_runtime_value("BZZOIRO_KEY")
|
||||||
|
bzzoiro_base = await get_runtime_value("BZZOIRO_BASE")
|
||||||
|
|
||||||
|
# 比赛覆盖
|
||||||
|
match_row = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
func.count().label("cnt"),
|
||||||
|
func.max(Match.match_date).label("latest_match_date"),
|
||||||
|
func.max(Match.created_at).label("latest_row_at"),
|
||||||
|
).where(Match.match_status == "finished")
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
# 统计覆盖(精确 retrieved_at)
|
||||||
|
stats_row = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
func.count().label("cnt"),
|
||||||
|
func.max(MatchStats.retrieved_at).label("latest_retrieved"),
|
||||||
|
).where(MatchStats.source == "bzzoiro")
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
# 积分榜覆盖
|
||||||
|
standings_row = (
|
||||||
|
await db.execute(select(func.count()).select_from(Standing))
|
||||||
|
).scalar()
|
||||||
|
|
||||||
|
bzzoiro = {
|
||||||
|
"name": "bzzoiro",
|
||||||
|
"label": "Bzzoiro",
|
||||||
|
"key_configured": bool(bzzoiro_key),
|
||||||
|
"base_url": (bzzoiro_base.rstrip("/") if bzzoiro_base else None) or settings.BZZOIRO_BASE,
|
||||||
|
"reachable": None, # 不主动探测
|
||||||
|
"last_success_at": (stats_row.latest_retrieved or match_row.latest_row_at),
|
||||||
|
"last_success_at_iso": (
|
||||||
|
stats_row.latest_retrieved or match_row.latest_row_at
|
||||||
|
).isoformat() if (stats_row.latest_retrieved or match_row.latest_row_at) else None,
|
||||||
|
"latest_match_date": match_row.latest_match_date.isoformat() if match_row.latest_match_date else None,
|
||||||
|
"recent_count": match_row.cnt or 0,
|
||||||
|
"stats_count": stats_row.cnt or 0,
|
||||||
|
"standings_count": standings_row or 0,
|
||||||
|
"note": "last_success_at 取 match_stats.retrieved_at(统计回填)与 matches.created_at(比赛行)的较大者",
|
||||||
|
"last_failure": _last_failure_log("bzzoiro"),
|
||||||
|
}
|
||||||
|
|
||||||
|
return {"sources": [bzzoiro]}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/keyring/status")
|
||||||
|
async def keyring_status():
|
||||||
|
"""KeyRing 运行状态:当前使用的 key、冷却状态、轮转信息(供管理后台展示)。"""
|
||||||
|
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
||||||
|
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
||||||
|
ring = get_key_ring(base, raw_keys)
|
||||||
|
st = ring.stats()
|
||||||
|
st["base_url"] = base
|
||||||
|
st["cooldown_seconds"] = ring._cooldown
|
||||||
|
st["has_multiple"] = ring.has_multiple
|
||||||
|
st["active_key"] = ring.active_key
|
||||||
|
return st
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/keyring/cooldown/reset")
|
||||||
|
async def keyring_reset_cooldown():
|
||||||
|
"""手动重置所有 key 的冷却状态(用于紧急恢复)。"""
|
||||||
|
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
||||||
|
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
||||||
|
ring = get_key_ring(base, raw_keys)
|
||||||
|
ring._blocked_until.clear()
|
||||||
|
return {"ok": True, "message": "已重置所有 key 冷却状态", "stats": ring.stats()}
|
||||||
|
|
||||||
|
|
||||||
|
def _last_failure_log(source: str) -> dict | None:
|
||||||
|
"""从系统日志缓冲中查找某数据源的最近一次错误(仅作参考,非专用失败表)。"""
|
||||||
|
entries = get_entries(min_level="ERROR", keyword=source, limit=5)
|
||||||
|
if not entries:
|
||||||
|
return None
|
||||||
|
e = entries[0]
|
||||||
|
return {
|
||||||
|
"at": datetime.fromtimestamp(e["ts"]).isoformat(),
|
||||||
|
"logger": e["logger"],
|
||||||
|
"detail": e["message"][:200],
|
||||||
|
"note": "approx:来自内存日志缓冲,非专用采集失败表;进程重启后清零",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/stats")
|
||||||
|
async def admin_stats(db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""管理区统计(只读):预测次数 + 比赛覆盖。轻量聚合,无 LLM 调用。"""
|
||||||
|
from sqlalchemy import func, text
|
||||||
|
from src.db.models import Prediction, Match, MatchStats, Standing
|
||||||
|
day_ago = datetime.now(timezone.utc) - timedelta(days=1)
|
||||||
|
week_ago = datetime.now(timezone.utc) - timedelta(days=7)
|
||||||
|
r = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
func.count().label("total"),
|
||||||
|
func.count().filter(Prediction.created_at >= day_ago).label("last_24h"),
|
||||||
|
func.count().filter(Prediction.created_at >= week_ago).label("last_7d"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
# F3 修复: 补充真实比赛计数(非 limit=100 近似)
|
||||||
|
match_cnt = (await db.execute(select(func.count()).select_from(Match))).scalar() or 0
|
||||||
|
finished_cnt = (await db.execute(select(func.count()).where(Match.match_status == "finished"))).scalar() or 0
|
||||||
|
stats_cnt = (await db.execute(select(func.count()).select_from(MatchStats))).scalar() or 0
|
||||||
|
standings_cnt = (await db.execute(select(func.count()).select_from(Standing))).scalar() or 0
|
||||||
|
return {
|
||||||
|
"predictions": {"total": r.total, "last_24h": r.last_24h, "last_7d": r.last_7d},
|
||||||
|
"matches": {"total": match_cnt, "finished": finished_cnt},
|
||||||
|
"stats": {"total": stats_cnt},
|
||||||
|
"standings": {"total": standings_cnt},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── 数据完整性分析(可视化数据源) ────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/data-completeness")
|
||||||
|
async def data_completeness(db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""按联赛统计数据完整性:比赛覆盖、字段覆盖、积分榜覆盖。
|
||||||
|
|
||||||
|
前端「数据完整性」页据此渲染,回答三个问题:
|
||||||
|
1. 数据是否齐全(各联赛比赛/统计/积分榜量级)
|
||||||
|
2. 字段是否齐全(每张统计表各字段非空率)
|
||||||
|
3. 覆盖是否新鲜(最近一场/最近一次采集)
|
||||||
|
"""
|
||||||
|
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_NAMES, LEAGUE_COUNTRIES
|
||||||
|
|
||||||
|
out_leagues: list[dict] = []
|
||||||
|
for code, bzz_id in BZZOIRO_LEAGUE_IDS.items():
|
||||||
|
# 比赛覆盖
|
||||||
|
m = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
func.count().label("total"),
|
||||||
|
func.count().filter(Match.match_status == "finished").label("finished"),
|
||||||
|
func.count().filter(Match.match_status == "scheduled").label("scheduled"),
|
||||||
|
func.count().filter(Match.source_event_id.is_not(None)).label("with_source_id"),
|
||||||
|
func.max(Match.match_date).label("latest_match"),
|
||||||
|
func.min(Match.match_date).label("earliest_match"),
|
||||||
|
)
|
||||||
|
.select_from(Match)
|
||||||
|
.join(League, League.id == Match.league_id)
|
||||||
|
.where(League.code == code)
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
# 统计字段覆盖(联表 matches)
|
||||||
|
s = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
func.count().label("rows"),
|
||||||
|
func.count(MatchStats.home_xg).label("xg"),
|
||||||
|
func.count(MatchStats.home_shots).label("shots"),
|
||||||
|
func.count(MatchStats.home_possession).label("possession"),
|
||||||
|
func.count(MatchStats.home_corners).label("corners"),
|
||||||
|
func.count(MatchStats.home_fouls).label("fouls"),
|
||||||
|
func.count(MatchStats.home_big_chances).label("big_chances"),
|
||||||
|
func.count(MatchStats.home_yellow_cards).label("cards"),
|
||||||
|
)
|
||||||
|
.select_from(MatchStats)
|
||||||
|
.join(Match, Match.id == MatchStats.match_id)
|
||||||
|
.join(League, League.id == Match.league_id)
|
||||||
|
.where(League.code == code)
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
# 积分榜覆盖
|
||||||
|
st = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
func.count().label("rows"),
|
||||||
|
func.max(Standing.retrieved_at).label("latest_retrieved"),
|
||||||
|
)
|
||||||
|
.select_from(Standing)
|
||||||
|
.join(League, League.id == Standing.league_id)
|
||||||
|
.where(League.code == code)
|
||||||
|
)
|
||||||
|
).one()
|
||||||
|
|
||||||
|
stats_rows = s.rows or 0
|
||||||
|
pct = lambda n: round(n / stats_rows * 100, 1) if stats_rows else 0.0 # noqa: E731
|
||||||
|
out_leagues.append(
|
||||||
|
{
|
||||||
|
"code": code,
|
||||||
|
"name": LEAGUE_NAMES.get(code, code),
|
||||||
|
"country": LEAGUE_COUNTRIES.get(code),
|
||||||
|
"matches": {
|
||||||
|
"total": m.total or 0,
|
||||||
|
"finished": m.finished or 0,
|
||||||
|
"scheduled": m.scheduled or 0,
|
||||||
|
"with_source_id": m.with_source_id or 0,
|
||||||
|
"earliest_match": m.earliest_match.isoformat() if m.earliest_match else None,
|
||||||
|
"latest_match": m.latest_match.isoformat() if m.latest_match else None,
|
||||||
|
},
|
||||||
|
"stats": {
|
||||||
|
"rows": stats_rows,
|
||||||
|
"fields": {
|
||||||
|
"xg": {"count": s.xg or 0, "pct": pct(s.xg or 0)},
|
||||||
|
"shots": {"count": s.shots or 0, "pct": pct(s.shots or 0)},
|
||||||
|
"possession": {"count": s.possession or 0, "pct": pct(s.possession or 0)},
|
||||||
|
"corners": {"count": s.corners or 0, "pct": pct(s.corners or 0)},
|
||||||
|
"fouls": {"count": s.fouls or 0, "pct": pct(s.fouls or 0)},
|
||||||
|
"big_chances": {"count": s.big_chances or 0, "pct": pct(s.big_chances or 0)},
|
||||||
|
"cards": {"count": s.cards or 0, "pct": pct(s.cards or 0)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"standings": {
|
||||||
|
"rows": st.rows or 0,
|
||||||
|
"latest_retrieved": st.latest_retrieved.isoformat() if st.latest_retrieved else None,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
# 整体健康信号
|
||||||
|
total_finished = sum(l["matches"]["finished"] for l in out_leagues)
|
||||||
|
total_stats = sum(l["stats"]["rows"] for l in out_leagues)
|
||||||
|
stats_coverage = round(total_stats / total_finished * 100, 1) if total_finished else 0.0
|
||||||
|
issues: list[str] = []
|
||||||
|
for l in out_leagues:
|
||||||
|
if l["matches"]["finished"] == 0:
|
||||||
|
issues.append(f"{l['name']}: 无已完赛比赛,请先运行「比赛数据」采集")
|
||||||
|
elif l["stats"]["rows"] == 0:
|
||||||
|
issues.append(f"{l['name']}: 已完赛 {l['matches']['finished']} 场但无统计回填,请运行「统计回填」采集")
|
||||||
|
elif stats_coverage < 80:
|
||||||
|
issues.append(f"{l['name']}: 统计覆盖率仅 {stats_coverage}%,建议增量回填")
|
||||||
|
if l["standings"]["rows"] == 0:
|
||||||
|
issues.append(f"{l['name']}: 无积分榜数据,请运行「积分榜」采集")
|
||||||
|
if not issues:
|
||||||
|
issues.append("各联赛数据完整度良好")
|
||||||
|
|
||||||
|
return {
|
||||||
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"leagues": out_leagues,
|
||||||
|
"totals": {
|
||||||
|
"finished_matches": total_finished,
|
||||||
|
"stats_rows": total_stats,
|
||||||
|
"stats_coverage_pct": stats_coverage,
|
||||||
|
},
|
||||||
|
"issues": issues,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# ── 数据质量检查 API ────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/data-quality")
|
||||||
|
async def data_quality_checks(db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""数据质量检查结果(只读)。"""
|
||||||
|
from src.db.models import IngestFailure, DataQualityCheck
|
||||||
|
from sqlalchemy import func
|
||||||
|
|
||||||
|
# 最近的失败记录
|
||||||
|
failures = (
|
||||||
|
await db.execute(
|
||||||
|
select(IngestFailure)
|
||||||
|
.where(IngestFailure.status.in_(["pending", "retrying"]))
|
||||||
|
.order_by(IngestFailure.created_at.desc())
|
||||||
|
.limit(20)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
|
||||||
|
# 最近的质量检查
|
||||||
|
checks = (
|
||||||
|
await db.execute(
|
||||||
|
select(DataQualityCheck)
|
||||||
|
.order_by(DataQualityCheck.checked_at.desc())
|
||||||
|
.limit(20)
|
||||||
|
)
|
||||||
|
).scalars().all()
|
||||||
|
|
||||||
|
return {
|
||||||
|
"failures": [
|
||||||
|
{
|
||||||
|
"id": f.id,
|
||||||
|
"source": f.source_system,
|
||||||
|
"entity_type": f.entity_type,
|
||||||
|
"source_record_id": f.source_record_id,
|
||||||
|
"error_type": f.error_type,
|
||||||
|
"error_detail": f.error_detail,
|
||||||
|
"retry_count": f.retry_count,
|
||||||
|
"status": f.status,
|
||||||
|
"created_at": f.created_at.isoformat() if f.created_at else None,
|
||||||
|
}
|
||||||
|
for f in failures
|
||||||
|
],
|
||||||
|
"checks": [
|
||||||
|
{
|
||||||
|
"id": c.id,
|
||||||
|
"check_name": c.check_name,
|
||||||
|
"entity_type": c.entity_type,
|
||||||
|
"passed": c.passed,
|
||||||
|
"severity": c.severity,
|
||||||
|
"detail": c.detail,
|
||||||
|
"checked_at": c.checked_at.isoformat() if c.checked_at else None,
|
||||||
|
}
|
||||||
|
for c in checks
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/data-quality/run")
|
||||||
|
async def run_data_quality_check(db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""手动触发一次数据质量检查。"""
|
||||||
|
from src.db.models import DataQualityCheck, Match, MatchStats, Standing, League
|
||||||
|
from sqlalchemy import func
|
||||||
|
|
||||||
|
checks = []
|
||||||
|
|
||||||
|
# 检查1: 已完赛但无统计的比赛
|
||||||
|
finished_no_stats = (
|
||||||
|
await db.execute(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(Match)
|
||||||
|
.outerjoin(MatchStats, Match.id == MatchStats.match_id)
|
||||||
|
.where(Match.match_status == "finished")
|
||||||
|
.where(MatchStats.id.is_(None))
|
||||||
|
)
|
||||||
|
).scalar() or 0
|
||||||
|
|
||||||
|
checks.append(DataQualityCheck(
|
||||||
|
check_name="finished_without_stats",
|
||||||
|
entity_type="match",
|
||||||
|
actual_value=float(finished_no_stats),
|
||||||
|
passed=finished_no_stats == 0,
|
||||||
|
severity="warning" if finished_no_stats > 0 else "info",
|
||||||
|
detail={"message": f"{finished_no_stats} 场已完赛比赛缺少统计数据"},
|
||||||
|
))
|
||||||
|
|
||||||
|
# 检查2: 积分榜缺失的联赛
|
||||||
|
leagues_without_standings = (
|
||||||
|
await db.execute(
|
||||||
|
select(func.count())
|
||||||
|
.select_from(League)
|
||||||
|
.outerjoin(Standing, League.id == Standing.league_id)
|
||||||
|
.where(Standing.id.is_(None))
|
||||||
|
)
|
||||||
|
).scalar() or 0
|
||||||
|
|
||||||
|
checks.append(DataQualityCheck(
|
||||||
|
check_name="league_without_standings",
|
||||||
|
entity_type="league",
|
||||||
|
actual_value=float(leagues_without_standings),
|
||||||
|
passed=leagues_without_standings == 0,
|
||||||
|
severity="warning" if leagues_without_standings > 0 else "info",
|
||||||
|
detail={"message": f"{leagues_without_standings} 个联赛缺少积分榜"},
|
||||||
|
))
|
||||||
|
|
||||||
|
for c in checks:
|
||||||
|
db.add(c)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
return {"ok": True, "checks": [{"name": c.check_name, "passed": c.passed} for c in checks]}
|
||||||
|
|||||||
@@ -1,60 +0,0 @@
|
|||||||
"""后台管理:球队别名管理(只读列表 + 添加别名)。
|
|
||||||
|
|
||||||
归一名(teams.name)是球队唯一键;别名(team_aliases)是同一球队的不同写法
|
|
||||||
(大小写/译名/缩写)到归一后 teams.id 的映射。入库时 normalize(name) 依次查
|
|
||||||
teams.name 与 team_aliases,命中即复用,避免重复 Team。
|
|
||||||
|
|
||||||
不自动合并历史重复队;需显式添加别名(或先 SQL/再经由此接口)。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
|
||||||
from sqlalchemy import desc, select
|
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
|
||||||
from src.api.schemas import TeamAliasIn, TeamAliasOut
|
|
||||||
from src.db.base import AsyncSession, get_db_read
|
|
||||||
from src.db.models import Team, TeamAlias
|
|
||||||
from src.db.repositories import TeamRepository
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin)])
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/teams/aliases", response_model=list[TeamAliasOut])
|
|
||||||
async def list_team_aliases(db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""列出所有球队别名(最新在前)。"""
|
|
||||||
rows = (await db.execute(select(TeamAlias).order_by(desc(TeamAlias.created_at)).limit(200))).scalars().all()
|
|
||||||
return [
|
|
||||||
TeamAliasOut(
|
|
||||||
alias_normalized=r.alias_normalized,
|
|
||||||
team_id=r.team_id,
|
|
||||||
original_alias=r.original_alias,
|
|
||||||
)
|
|
||||||
for r in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/teams/aliases", response_model=TeamAliasOut, status_code=201)
|
|
||||||
async def add_team_alias(req: TeamAliasIn, db: AsyncSession = Depends(get_db_read)):
|
|
||||||
"""为已有 Team 添加别名(幂等:重复添加会更新指向)。
|
|
||||||
|
|
||||||
不自动合并历史重复队。若需合并 A→B:先为 A 的归一名添加别名指向 B,
|
|
||||||
再人工确认 A 是否仍有独立引用。
|
|
||||||
"""
|
|
||||||
# 校验目标 Team 存在
|
|
||||||
team = await db.get(Team, req.team_id)
|
|
||||||
if team is None:
|
|
||||||
raise HTTPException(404, f"目标 Team 不存在: id={req.team_id}")
|
|
||||||
|
|
||||||
repo = TeamRepository(db)
|
|
||||||
row = await repo.add_alias(req.alias, req.team_id)
|
|
||||||
logger.info("添加 Team 别名: %s -> team_id=%s", req.alias, req.team_id)
|
|
||||||
return TeamAliasOut(
|
|
||||||
alias_normalized=row.alias_normalized,
|
|
||||||
team_id=row.team_id,
|
|
||||||
original_alias=row.original_alias,
|
|
||||||
)
|
|
||||||
+60
-57
@@ -10,17 +10,17 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
import uuid
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
from src.api.deps import require_admin
|
from src.api.deps import require_admin
|
||||||
from src.api.schemas import IngestBzzoiroRequest, IngestBzzoiroResponse
|
from src.api.schemas import IngestBzzoiroRequest
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS
|
from src.data.config import BZZOIRO_LEAGUE_IDS
|
||||||
from src.data.bzzoiro_standings import ingest_bzzoiro_standings
|
from src.data.bzzoiro import ingest_bzzoiro_event_stats, ingest_bzzoiro_standings
|
||||||
from src.data.bzzoiro_stats import ingest_bzzoiro_event_stats
|
|
||||||
from src.data.sources import get_source
|
from src.data.sources import get_source
|
||||||
|
from src.db.models import IngestJob
|
||||||
from src.db.unit_of_work import get_uow
|
from src.db.unit_of_work import get_uow
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -33,6 +33,29 @@ _background_tasks: set[asyncio.Task] = set()
|
|||||||
VALID_TASKS = {"events", "standings", "stats", "all"}
|
VALID_TASKS = {"events", "standings", "stats", "all"}
|
||||||
|
|
||||||
|
|
||||||
|
def _utcnow() -> datetime:
|
||||||
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
async def _update_job(job_id: str, **fields) -> None:
|
||||||
|
"""更新采集任务状态(尽力而为:失败只记日志,绝不拖垮采集主流程)。
|
||||||
|
|
||||||
|
job 记录是任务级可观测性,IngestFailure 死信是行级失败记录,
|
||||||
|
两者独立工作 —— 本函数抛错不应影响采集结果,故整体吞异常。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
async with get_uow() as session:
|
||||||
|
stmt = select(IngestJob).where(IngestJob.id == job_id)
|
||||||
|
job = (await session.execute(stmt)).scalar_one_or_none()
|
||||||
|
if job is None:
|
||||||
|
logger.warning("ingest job %s 不存在,跳过状态更新(%s)", job_id, fields)
|
||||||
|
return
|
||||||
|
for k, v in fields.items():
|
||||||
|
setattr(job, k, v)
|
||||||
|
except Exception:
|
||||||
|
logger.warning("ingest job %s 状态更新失败(不影响采集): %s", job_id, fields, exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
def _spawn(coro) -> None:
|
def _spawn(coro) -> None:
|
||||||
"""启动后台采集任务;异常已在任务内记录到系统日志。"""
|
"""启动后台采集任务;异常已在任务内记录到系统日志。"""
|
||||||
task = asyncio.create_task(coro)
|
task = asyncio.create_task(coro)
|
||||||
@@ -40,58 +63,52 @@ def _spawn(coro) -> None:
|
|||||||
task.add_done_callback(_background_tasks.discard)
|
task.add_done_callback(_background_tasks.discard)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ingest/bzzoiro", response_model=IngestBzzoiroResponse, dependencies=[Depends(require_admin)])
|
@router.post("/ingest/bzzoiro", dependencies=[Depends(require_admin)])
|
||||||
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
async def ingest_bzzoiro_route(req: IngestBzzoiroRequest):
|
||||||
"""触发 bzzoiro 采集(events / standings / stats / all)。
|
"""触发 bzzoiro 采集(events / standings / stats / all)。
|
||||||
|
|
||||||
启动后台任务前写入 ingest_jobs(pending),响应返回 job_id 供前端轮询。
|
启动后台任务前先创建 ingest_jobs 记录(pending),响应返回 job_id,
|
||||||
兼容原 message 字段(仍返回)。
|
供 admin 通过 GET /api/v1/admin/ingest/jobs/{job_id} 轮询状态。
|
||||||
"""
|
"""
|
||||||
if req.task not in VALID_TASKS:
|
if req.task not in VALID_TASKS:
|
||||||
raise HTTPException(status_code=422, detail=f"未知任务类型: {req.task}(可选: {', '.join(sorted(VALID_TASKS))})")
|
raise HTTPException(status_code=422, detail=f"未知任务类型: {req.task}(可选: {', '.join(sorted(VALID_TASKS))})")
|
||||||
leagues = req.leagues or list(BZZOIRO_LEAGUE_IDS.keys())
|
leagues = req.leagues or list(BZZOIRO_LEAGUE_IDS.keys())
|
||||||
task_label = {"events": "比赛数据", "standings": "积分榜", "stats": "统计回填", "all": "全量(比赛+积分榜+统计)"}[req.task]
|
task_label = {"events": "比赛数据", "standings": "积分榜", "stats": "统计回填", "all": "全量(比赛+积分榜+统计)"}[req.task]
|
||||||
|
|
||||||
job_id = await _create_ingest_job(req.task, leagues, req)
|
# 任务级可观测性:先落 pending 记录,后台 _run_bzzoiro 接管状态流转
|
||||||
_spawn(_run_bzzoiro(job_id, req.task, leagues, req))
|
job = IngestJob(
|
||||||
|
task=req.task,
|
||||||
return IngestBzzoiroResponse(
|
|
||||||
ok=True,
|
|
||||||
job_id=job_id,
|
|
||||||
message=f"采集任务已启动(后台执行,任务: {task_label}),请到「数据采集」页跟踪进度",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _create_ingest_job(task: str, leagues: list[str], req: IngestBzzoiroRequest) -> str:
|
|
||||||
"""写入一条 ingest_jobs(pending),返回 job_id。"""
|
|
||||||
from src.db.models import IngestJob
|
|
||||||
|
|
||||||
job_id = str(uuid.uuid4())
|
|
||||||
params={
|
params={
|
||||||
"leagues": leagues,
|
"leagues": leagues,
|
||||||
"date_from": req.date_from,
|
"date_from": req.date_from,
|
||||||
"date_to": req.date_to,
|
"date_to": req.date_to,
|
||||||
"status": req.status,
|
"status": req.status,
|
||||||
"task": task,
|
|
||||||
"limit": req.limit,
|
"limit": req.limit,
|
||||||
"season": req.season,
|
"season": req.season,
|
||||||
}
|
},
|
||||||
|
status="pending",
|
||||||
|
)
|
||||||
async with get_uow() as session:
|
async with get_uow() as session:
|
||||||
job = IngestJob(id=job_id, task=task, params=params, status="pending")
|
|
||||||
session.add(job)
|
session.add(job)
|
||||||
logger.info("ingest_jobs 创建: job=%s task=%s leagues=%s", job_id, task, leagues)
|
|
||||||
return job_id
|
_spawn(_run_bzzoiro(job.id, req.task, leagues, req))
|
||||||
|
return {
|
||||||
|
"ok": True,
|
||||||
|
"job_id": job.id,
|
||||||
|
"message": f"采集任务已启动(后台执行,任务: {task_label}),可用 job_id 查询状态或在「系统日志」查看进度",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None:
|
async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBzzoiroRequest) -> None:
|
||||||
"""后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。
|
"""后台执行 bzzoiro 采集:上游限速时单次可能耗时数分钟,必须脱离请求生命周期。
|
||||||
|
|
||||||
状态流转: pending → running → (success|failed)。
|
同时维护 ingest_jobs 状态(pending → running → success/failed):
|
||||||
|
- result 存各子任务统计摘要(errors 截断到前 10 条)
|
||||||
|
- 状态更新经 _update_job 尽力而为,失败不影响采集本身
|
||||||
|
- 单条管线抓取失败的行级记录仍由 bzzoiro 死信逻辑(IngestFailure)负责
|
||||||
"""
|
"""
|
||||||
from src.db.models import IngestJob
|
await _update_job(job_id, status="running", started_at=_utcnow())
|
||||||
|
summary: dict = {}
|
||||||
await _update_job(job_id, status="running", started_at=datetime.now(timezone.utc))
|
|
||||||
result: dict = {}
|
|
||||||
try:
|
try:
|
||||||
if task in ("events", "all"):
|
if task in ("events", "all"):
|
||||||
statuses = [req.status] if req.status else ["finished", "scheduled"]
|
statuses = [req.status] if req.status else ["finished", "scheduled"]
|
||||||
@@ -118,7 +135,7 @@ async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBz
|
|||||||
)
|
)
|
||||||
if merged["errors"]:
|
if merged["errors"]:
|
||||||
logger.warning("bzzoiro 比赛采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3])
|
logger.warning("bzzoiro 比赛采集错误 %d 条: %s", len(merged["errors"]), merged["errors"][:3])
|
||||||
result["events"] = merged
|
summary["events"] = {**merged, "errors": merged["errors"][:10]}
|
||||||
|
|
||||||
if task in ("standings", "all"):
|
if task in ("standings", "all"):
|
||||||
async with get_uow() as session:
|
async with get_uow() as session:
|
||||||
@@ -127,7 +144,9 @@ async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBz
|
|||||||
logger.warning("bzzoiro 积分榜采集部分失败: %s", r["errors"][:3])
|
logger.warning("bzzoiro 积分榜采集部分失败: %s", r["errors"][:3])
|
||||||
else:
|
else:
|
||||||
logger.info("bzzoiro 积分榜采集完成: upsert %d 条", r["total_upserted"])
|
logger.info("bzzoiro 积分榜采集完成: upsert %d 条", r["total_upserted"])
|
||||||
result["standings"] = r
|
summary["standings"] = {k: r.get(k) for k in ("total_upserted", "errors") if k in r}
|
||||||
|
if isinstance(summary["standings"].get("errors"), list):
|
||||||
|
summary["standings"]["errors"] = summary["standings"]["errors"][:10]
|
||||||
|
|
||||||
if task in ("stats", "all"):
|
if task in ("stats", "all"):
|
||||||
async with get_uow() as session:
|
async with get_uow() as session:
|
||||||
@@ -136,28 +155,12 @@ async def _run_bzzoiro(job_id: str, task: str, leagues: list[str], req: IngestBz
|
|||||||
)
|
)
|
||||||
if r["errors"]:
|
if r["errors"]:
|
||||||
logger.warning("bzzoiro 统计回填错误 %d 条: %s", len(r["errors"]), r["errors"][:3])
|
logger.warning("bzzoiro 统计回填错误 %d 条: %s", len(r["errors"]), r["errors"][:3])
|
||||||
result["stats"] = r
|
summary["stats"] = {k: r.get(k) for k in ("total_inserted", "total_updated", "errors") if k in r}
|
||||||
|
if isinstance(summary["stats"].get("errors"), list):
|
||||||
await _update_job(job_id, status="success", result=result, finished_at=datetime.now(timezone.utc))
|
summary["stats"]["errors"] = summary["stats"]["errors"][:10]
|
||||||
logger.info("ingest_jobs 完成: job=%s task=%s", job_id, task)
|
except Exception as exc:
|
||||||
except Exception as e:
|
|
||||||
logger.exception("bzzoiro 采集任务失败(task=%s)", task)
|
logger.exception("bzzoiro 采集任务失败(task=%s)", task)
|
||||||
await _update_job(
|
await _update_job(job_id, status="failed", finished_at=_utcnow(), error=repr(exc))
|
||||||
job_id, status="failed", error=str(e), finished_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _update_job(job_id: str, **fields) -> None:
|
|
||||||
"""更新 ingest_jobs 单行;失败仅记日志,绝不抛异常(避免干扰采集主流程)。"""
|
|
||||||
from src.db.models import IngestJob
|
|
||||||
|
|
||||||
try:
|
|
||||||
async with get_uow() as session:
|
|
||||||
job = await session.get(IngestJob, job_id)
|
|
||||||
if job is None:
|
|
||||||
logger.warning("ingest_jobs 更新失败: job=%s 不存在", job_id)
|
|
||||||
return
|
return
|
||||||
for k, v in fields.items():
|
|
||||||
setattr(job, k, v)
|
await _update_job(job_id, status="success", finished_at=_utcnow(), result=summary)
|
||||||
except Exception:
|
|
||||||
logger.warning("ingest_jobs 更新异常: job=%s fields=%s", job_id, list(fields.keys()))
|
|
||||||
|
|||||||
@@ -5,11 +5,11 @@ from datetime import datetime, timedelta
|
|||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
from sqlalchemy import func, or_, select
|
from sqlalchemy import func, or_, select
|
||||||
from sqlalchemy.orm import load_only, selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from src.api.schemas import MatchListOut, MatchOut, PredictionOut
|
from src.api.schemas import MatchListOut, MatchOut, PredictionOut
|
||||||
from src.db.base import AsyncSession, get_db_read
|
from src.db.base import AsyncSession, get_db_read
|
||||||
from src.db.models import League, Match, MatchStats, Prediction, Standing, Team
|
from src.db.models import League, Match, Prediction, Standing
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/v1", tags=["data"])
|
router = APIRouter(prefix="/api/v1", tags=["data"])
|
||||||
|
|
||||||
@@ -49,20 +49,8 @@ async def list_matches(
|
|||||||
limit: int = Query(50, ge=1, le=100),
|
limit: int = Query(50, ge=1, le=100),
|
||||||
db: AsyncSession = Depends(get_db_read),
|
db: AsyncSession = Depends(get_db_read),
|
||||||
):
|
):
|
||||||
"""比赛列表(游标分页)。
|
"""比赛列表(游标分页)。"""
|
||||||
|
q = select(Match).options(selectinload(Match.league), selectinload(Match.home_team), selectinload(Match.away_team))
|
||||||
加载策略(列表 vs 详情):
|
|
||||||
- 列表:仅 selectinload 序列化需要的 3 个关系 + stats,且用 load_only 限定列
|
|
||||||
(League.code / Team.name,name_zh / MatchStats.home_xg,away_xg),避免传输全列;
|
|
||||||
同时一次性加载 stats 消除 N+1(m.stats.home_xg 此前触发懒加载)。
|
|
||||||
- 详情(/matches/{id}):保持完整 options(league/teams/stats 全列 + 最近预测)。
|
|
||||||
"""
|
|
||||||
q = select(Match).options(
|
|
||||||
selectinload(Match.league).load_only(League.code),
|
|
||||||
selectinload(Match.home_team).load_only(Team.name, Team.name_zh),
|
|
||||||
selectinload(Match.away_team).load_only(Team.name, Team.name_zh),
|
|
||||||
selectinload(Match.stats).load_only(MatchStats.home_xg, MatchStats.away_xg),
|
|
||||||
)
|
|
||||||
|
|
||||||
if cursor:
|
if cursor:
|
||||||
try:
|
try:
|
||||||
@@ -286,9 +274,7 @@ async def list_standings(
|
|||||||
):
|
):
|
||||||
"""联赛积分榜(只读)。按联赛分组,每张榜按 position 排序。
|
"""联赛积分榜(只读)。按联赛分组,每张榜按 position 排序。
|
||||||
|
|
||||||
P0-02: standings 为追加快照,公开接口取每队 available_at 最新快照
|
season 为空时返回每个联赛最新采集到的赛季榜单(适合前端"查看最新积分榜")。
|
||||||
(league_id, season, team_id 上按 available_at 取最新)。
|
|
||||||
season 为空时返回每个联赛最新采集到的赛季榜单。
|
|
||||||
"""
|
"""
|
||||||
# 取每个联赛最新赛季(当 season 为空时)
|
# 取每个联赛最新赛季(当 season 为空时)
|
||||||
latest_seasons: dict[int, str] = {}
|
latest_seasons: dict[int, str] = {}
|
||||||
@@ -301,25 +287,9 @@ async def list_standings(
|
|||||||
).all()
|
).all()
|
||||||
latest_seasons = {r.league_id: r.latest for r in rows}
|
latest_seasons = {r.league_id: r.latest for r in rows}
|
||||||
|
|
||||||
# P0-02: 子查询取每队最新 available_at 快照,再 JOIN 回主表拿完整行 + League
|
|
||||||
latest_per_team = (
|
|
||||||
select(
|
|
||||||
Standing.league_id, Standing.season, Standing.team_id,
|
|
||||||
func.max(Standing.available_at).label("max_available"),
|
|
||||||
)
|
|
||||||
.group_by(Standing.league_id, Standing.season, Standing.team_id)
|
|
||||||
.subquery("latest_per_team")
|
|
||||||
)
|
|
||||||
q = (
|
q = (
|
||||||
select(Standing, League)
|
select(Standing, League)
|
||||||
.join(League, League.id == Standing.league_id)
|
.join(League, League.id == Standing.league_id)
|
||||||
.join(
|
|
||||||
latest_per_team,
|
|
||||||
(Standing.league_id == latest_per_team.c.league_id)
|
|
||||||
& (Standing.season == latest_per_team.c.season)
|
|
||||||
& (Standing.team_id == latest_per_team.c.team_id)
|
|
||||||
& (Standing.available_at == latest_per_team.c.max_available),
|
|
||||||
)
|
|
||||||
.order_by(League.name.asc(), Standing.position.asc())
|
.order_by(League.name.asc(), Standing.position.asc())
|
||||||
)
|
)
|
||||||
if league:
|
if league:
|
||||||
|
|||||||
@@ -64,8 +64,10 @@ async def predict(req: PredictRequest, request: Request):
|
|||||||
raise HTTPException(500, "预测失败,请查看服务器日志")
|
raise HTTPException(500, "预测失败,请查看服务器日志")
|
||||||
|
|
||||||
# D2: 三种模式统一返回 PredictResult —— 字段映射单一化,无 dict 分支。
|
# D2: 三种模式统一返回 PredictResult —— 字段映射单一化,无 dict 分支。
|
||||||
# P3-2:baseline 已在服务层(predict_baseline)落库并回填真实 prediction_id,
|
# 仅 baseline 的 prediction_id 需要在此落库补齐(服务层不落库)。
|
||||||
# 路由层不再需要特殊的 _persist_baseline,与 single/multi 路径统一。
|
if req.mode == "baseline":
|
||||||
|
prediction_id = await _persist_baseline(req.match_id, result)
|
||||||
|
else:
|
||||||
prediction_id = result.prediction_id
|
prediction_id = result.prediction_id
|
||||||
|
|
||||||
# 3. 结果映射(无 DB 访问)
|
# 3. 结果映射(无 DB 访问)
|
||||||
@@ -99,6 +101,36 @@ async def predict(req: PredictRequest, request: Request):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _persist_baseline(match_id: int, baseline: PredictResult) -> int:
|
||||||
|
"""将基线预测结果写入 prediction 表,复用 upsert 语义。"""
|
||||||
|
from src.db.unit_of_work import get_uow
|
||||||
|
from src.llm.predict import _upsert_prediction
|
||||||
|
|
||||||
|
async with get_uow() as session:
|
||||||
|
pred = await _upsert_prediction(
|
||||||
|
session,
|
||||||
|
match_id=match_id,
|
||||||
|
provider_name="baseline",
|
||||||
|
model="baseline",
|
||||||
|
mode="baseline",
|
||||||
|
run_type="live", # baseline 是 live 预测的变体,符合 ck_run_type_enum
|
||||||
|
values={
|
||||||
|
"prompt_version": baseline.prompt_version,
|
||||||
|
"prompt_tokens": baseline.prompt_tokens or 0,
|
||||||
|
"completion_tokens": baseline.completion_tokens or 0,
|
||||||
|
"latency_ms": baseline.latency_ms or 0,
|
||||||
|
"pred_home_goals": baseline.pred_home_goals,
|
||||||
|
"pred_away_goals": baseline.pred_away_goals,
|
||||||
|
"pred_1x2": baseline.pred_1x2,
|
||||||
|
"subjective_confidence": baseline.subjective_confidence,
|
||||||
|
"reasoning": baseline.reasoning,
|
||||||
|
"raw_response": baseline.raw,
|
||||||
|
"status": "success",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return pred.id
|
||||||
|
|
||||||
|
|
||||||
@router.get("/predictions", response_model=list[PredictionOut], dependencies=[Depends(require_admin)])
|
@router.get("/predictions", response_model=list[PredictionOut], dependencies=[Depends(require_admin)])
|
||||||
async def list_predictions(
|
async def list_predictions(
|
||||||
match_id: int | None = None,
|
match_id: int | None = None,
|
||||||
|
|||||||
@@ -10,8 +10,7 @@ from sqlalchemy import select, delete
|
|||||||
from src.api.deps import require_admin
|
from src.api.deps import require_admin
|
||||||
from src.api.schemas import ScheduleIn, ScheduleUpdate, ScheduleOut
|
from src.api.schemas import ScheduleIn, ScheduleUpdate, ScheduleOut
|
||||||
from src.core.scheduler import scheduler
|
from src.core.scheduler import scheduler
|
||||||
from src.data.bzzoiro_standings import ingest_bzzoiro_standings
|
from src.data.bzzoiro import ingest_bzzoiro_event_stats, ingest_bzzoiro_standings
|
||||||
from src.data.bzzoiro_stats import ingest_bzzoiro_event_stats
|
|
||||||
from src.data.sources import get_source
|
from src.data.sources import get_source
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS
|
from src.data.config import BZZOIRO_LEAGUE_IDS
|
||||||
from src.db.base import AsyncSession, get_db_read
|
from src.db.base import AsyncSession, get_db_read
|
||||||
@@ -183,3 +182,49 @@ async def retry_ingest_failure(failure_id: int, db: AsyncSession = Depends(get_d
|
|||||||
|
|
||||||
# 触发重试(简化版:仅标记状态,实际重试逻辑由调度器处理)
|
# 触发重试(简化版:仅标记状态,实际重试逻辑由调度器处理)
|
||||||
return {"ok": True, "message": f"已标记重试 (第 {failure.retry_count} 次)"}
|
return {"ok": True, "message": f"已标记重试 (第 {failure.retry_count} 次)"}
|
||||||
|
|
||||||
|
|
||||||
|
# ── 采集任务状态(ingest_jobs) ──────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_job(j) -> dict:
|
||||||
|
"""ingest_jobs 行 → 响应 dict(时间统一 isoformat)。"""
|
||||||
|
return {
|
||||||
|
"id": j.id,
|
||||||
|
"task": j.task,
|
||||||
|
"params": j.params,
|
||||||
|
"status": j.status,
|
||||||
|
"result": j.result,
|
||||||
|
"error": j.error,
|
||||||
|
"created_at": j.created_at.isoformat() if j.created_at else None,
|
||||||
|
"started_at": j.started_at.isoformat() if j.started_at else None,
|
||||||
|
"finished_at": j.finished_at.isoformat() if j.finished_at else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/ingest/jobs")
|
||||||
|
async def list_ingest_jobs(
|
||||||
|
limit: int = 20,
|
||||||
|
status: str | None = None,
|
||||||
|
db: AsyncSession = Depends(get_db_read),
|
||||||
|
):
|
||||||
|
"""列出采集任务状态(最新在前),可按 status 过滤。"""
|
||||||
|
from src.db.models import IngestJob
|
||||||
|
|
||||||
|
stmt = select(IngestJob).order_by(IngestJob.created_at.desc()).limit(max(1, min(limit, 100)))
|
||||||
|
if status:
|
||||||
|
stmt = stmt.where(IngestJob.status == status)
|
||||||
|
rows = (await db.execute(stmt)).scalars().all()
|
||||||
|
return [_serialize_job(j) for j in rows]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/ingest/jobs/{job_id}")
|
||||||
|
async def get_ingest_job(job_id: str, db: AsyncSession = Depends(get_db_read)):
|
||||||
|
"""查询单次采集任务的状态与结果(供前端提交后轮询)。"""
|
||||||
|
from src.db.models import IngestJob
|
||||||
|
|
||||||
|
stmt = select(IngestJob).where(IngestJob.id == job_id)
|
||||||
|
job = (await db.execute(stmt)).scalar_one_or_none()
|
||||||
|
if job is None:
|
||||||
|
raise HTTPException(404, "采集任务不存在")
|
||||||
|
return _serialize_job(job)
|
||||||
|
|||||||
+12
-33
@@ -7,6 +7,13 @@ from typing import Any
|
|||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class LeagueOut(BaseModel):
|
||||||
|
id: int
|
||||||
|
code: str
|
||||||
|
name: str
|
||||||
|
country: str | None
|
||||||
|
|
||||||
|
|
||||||
class MatchOut(BaseModel):
|
class MatchOut(BaseModel):
|
||||||
id: int
|
id: int
|
||||||
league_code: str | None
|
league_code: str | None
|
||||||
@@ -110,39 +117,11 @@ class IngestBzzoiroRequest(BaseModel):
|
|||||||
season: str | None = Field(None, description="standings 赛季,如 '2026-2027';空 = 当前赛季")
|
season: str | None = Field(None, description="standings 赛季,如 '2026-2027';空 = 当前赛季")
|
||||||
|
|
||||||
|
|
||||||
class TeamAliasIn(BaseModel):
|
class IngestResponse(BaseModel):
|
||||||
"""POST /api/v1/admin/teams/aliases 请求体:为已有 Team 添加别名。"""
|
leagues: dict
|
||||||
|
total_inserted: int
|
||||||
alias: str = Field(..., min_length=1, max_length=120, description="球队别名(原始写法)")
|
total_updated: int
|
||||||
team_id: int = Field(..., gt=0, description="归一后的目标 teams.id")
|
errors: list[str] = []
|
||||||
|
|
||||||
|
|
||||||
class TeamAliasOut(BaseModel):
|
|
||||||
alias_normalized: str
|
|
||||||
team_id: int
|
|
||||||
original_alias: str
|
|
||||||
|
|
||||||
|
|
||||||
class IngestBzzoiroResponse(BaseModel):
|
|
||||||
"""POST /api/v1/ingest/bzzoiro 响应:兼容原 message 字段,新增 job_id 供轮询。"""
|
|
||||||
|
|
||||||
ok: bool = True
|
|
||||||
job_id: str = Field(..., description="采集任务 ID(GET /api/v1/admin/ingest/jobs/{job_id} 轮询)")
|
|
||||||
message: str = ""
|
|
||||||
|
|
||||||
|
|
||||||
class IngestJobOut(BaseModel):
|
|
||||||
"""采集任务状态详情。"""
|
|
||||||
|
|
||||||
id: str
|
|
||||||
task: str
|
|
||||||
params: dict
|
|
||||||
status: str # pending | running | success | failed
|
|
||||||
result: dict | None = None
|
|
||||||
error: str | None = None
|
|
||||||
created_at: datetime | None = None
|
|
||||||
started_at: datetime | None = None
|
|
||||||
finished_at: datetime | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class ScheduleIn(BaseModel):
|
class ScheduleIn(BaseModel):
|
||||||
|
|||||||
@@ -11,10 +11,6 @@ class Settings(BaseSettings):
|
|||||||
# --- app ---
|
# --- app ---
|
||||||
APP_ENV: str = "development"
|
APP_ENV: str = "development"
|
||||||
LOG_LEVEL: str = "INFO"
|
LOG_LEVEL: str = "INFO"
|
||||||
# P3-3:多 worker 时应用内限流与 KeyRing 各自独立计数(配额放大 N 倍)。
|
|
||||||
# 设为 True 时若以多 worker 启动 uvicorn 则拒绝启动,避免静默配额漂移。
|
|
||||||
# 仅在你已前置 Nginx/网关做全局限流、确认不需要此守护时留空/False。
|
|
||||||
STRICT_SINGLE_WORKER: bool = False
|
|
||||||
# 生产环境强制要求管理鉴权配置,即使 APP_ENV=production 也生效。
|
# 生产环境强制要求管理鉴权配置,即使 APP_ENV=production 也生效。
|
||||||
# True 时若 auth_configured() 为 False 则拒绝(503),development 保持 fail-open。
|
# True 时若 auth_configured() 为 False 则拒绝(503),development 保持 fail-open。
|
||||||
REQUIRE_ADMIN_AUTH: bool = False
|
REQUIRE_ADMIN_AUTH: bool = False
|
||||||
@@ -32,12 +28,6 @@ class Settings(BaseSettings):
|
|||||||
LLM_SPECIALIST_MODEL: str = ""
|
LLM_SPECIALIST_MODEL: str = ""
|
||||||
LLM_AGGREGATOR_MODEL: str = ""
|
LLM_AGGREGATOR_MODEL: str = ""
|
||||||
|
|
||||||
# ── 预测缓存 ──
|
|
||||||
# 预测响应缓存后端:空(默认)=进程内 LRU+TTL 字典;填 redis://host:port/db 启用 Redis。
|
|
||||||
# Redis 失败自动降级内存缓存并 warning,不中断预测;不强制依赖 redis 包。
|
|
||||||
# TTL 固定 300s(5 分钟),键格式与内存后端一致(含 prompt 模板 hash)。
|
|
||||||
PREDICT_CACHE_URL: str = ""
|
|
||||||
|
|
||||||
# --- data sources ---
|
# --- data sources ---
|
||||||
BZZOIRO_KEY: str = ""
|
BZZOIRO_KEY: str = ""
|
||||||
BZZOIRO_BASE: str = "https://sports.bzzoiro.com/api/v2"
|
BZZOIRO_BASE: str = "https://sports.bzzoiro.com/api/v2"
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ _MIN_SECRET_KEY_LEN = 16
|
|||||||
_WEAK_DB_PATTERNS = ("football:football@", "admin:admin@", "password@", "123456@")
|
_WEAK_DB_PATTERNS = ("football:football@", "admin:admin@", "password@", "123456@")
|
||||||
|
|
||||||
|
|
||||||
|
class SecurityCheckError(Exception):
|
||||||
|
"""生产环境安全校验失败。"""
|
||||||
|
|
||||||
|
|
||||||
async def _auth_configured() -> bool:
|
async def _auth_configured() -> bool:
|
||||||
"""运行时鉴权是否已配置(含数据库密码哈希/.env 明文/API Key)。"""
|
"""运行时鉴权是否已配置(含数据库密码哈希/.env 明文/API Key)。"""
|
||||||
if await get_admin_password_hash():
|
if await get_admin_password_hash():
|
||||||
|
|||||||
+779
-52
@@ -1,66 +1,793 @@
|
|||||||
"""Bzzoiro 数据源:抓取 + 入库(单一数据源)—— 聚合门面。
|
"""Bzzoiro 数据源:抓取 + 入库(单一数据源)。
|
||||||
|
|
||||||
实现按管线拆分(单文件 → 多模块),本模块只做再导出,保持不变量:
|
三条管线:
|
||||||
1. sources._load_sources() 仍从本模块导入 BzzoiroSource(注册表入口不变);
|
1. events — 比赛日程/比分(/events/),含 source_event_id 血缘
|
||||||
2. 测试与脚本对 `bz.<名称>` 的 monkeypatch 语义不变 —— 抓取函数 / REQUEST_INTERVAL
|
2. standings— 联赛积分榜快照(/leagues/{id}/standings/)
|
||||||
仍经本门面解析可替换;Bronze 写入助手(_write_raw_event/_write_lineage/
|
3. stats — 已完赛比赛详细统计回填(/events/{id}/stats/)
|
||||||
_safe_write_ingest_failure)已改为管线模块直接 import pipeline_write,
|
|
||||||
测试需 patch `src.data.pipeline_write.*` 源模块。
|
|
||||||
|
|
||||||
三条管线(各自模块):
|
|
||||||
1. events — 比赛日程/比分(/events/),含 source_event_id 血缘 → bzzoiro_events.py
|
|
||||||
2. standings— 联赛积分榜快照(/leagues/{id}/standings/) → bzzoiro_standings.py
|
|
||||||
3. stats — 已完赛比赛详细统计回填(/events/{id}/stats/) → bzzoiro_stats.py
|
|
||||||
|
|
||||||
共享基础:HTTP 抓取(多 key 轮换)与字段转换 → bzzoiro_common.py;
|
|
||||||
Bronze 基础设施(RawEvent/IngestFailure/DataLineage)→ pipeline_write.py
|
|
||||||
(各管线模块直接 import pipeline_write,不再经本门面转发)。
|
|
||||||
|
|
||||||
D4(工程债): Team/League/Match 的查找/创建经 Repository 层(src/db/repositories.py),
|
D4(工程债): Team/League/Match 的查找/创建经 Repository 层(src/db/repositories.py),
|
||||||
各管线不直接控制事务(commit/rollback 由调用方 UnitOfWork 控制,只 flush)。
|
本模块不直接控制事务(commit/rollback 由调用方 UnitOfWork 控制,这里只 flush)。
|
||||||
Standing/RawEvent/Lineage 等管线内私有读写仍不强行 Repository 化。
|
Standing/RawEvent/Lineage 等管线内私有读写仍在本模块内实现,不强行 Repository 化。
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
# ── 配置常量(原文件即从 config 再导出,维持 bz.REQUEST_INTERVAL 等引用) ──
|
import asyncio
|
||||||
from src.data.config import ( # noqa: F401
|
import logging
|
||||||
BZZOIRO_LEAGUE_IDS,
|
import random
|
||||||
LEAGUE_COUNTRIES,
|
from collections.abc import Iterable
|
||||||
LEAGUE_NAMES,
|
from datetime import datetime, timedelta, timezone
|
||||||
REQUEST_INTERVAL,
|
|
||||||
)
|
|
||||||
from src.data.key_ring import _mask # noqa: F401 (R1 测试引用 bz._mask)
|
|
||||||
from src.data.normalize import normalize_bzzoiro # noqa: F401
|
|
||||||
|
|
||||||
# ── 共享原语:HTTP 抓取 + 宽松字段转换 ──
|
from sqlalchemy import select
|
||||||
# NOTE: 管线模块同时从 bzzoiro_common 直接 import _fetch_json_async 等(经本处也转发)。
|
|
||||||
from src.data.bzzoiro_common import ( # noqa: F401
|
import httpx
|
||||||
_fetch_json_async,
|
|
||||||
_match_key,
|
from src.core.runtime_config import get_runtime_value
|
||||||
_to_date,
|
from src.core.http_client import get_client
|
||||||
_to_float_or_none,
|
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||||
_to_int_or_none,
|
from src.data.key_ring import _mask, get_key_ring
|
||||||
|
from src.data.normalize import normalize_bzzoiro
|
||||||
|
from src.data.team_names_zh import zh_name
|
||||||
|
from src.data.sources import register
|
||||||
|
from src.db.models import Match, MatchStats, Standing, Team, RawEvent, IngestFailure, DataLineage
|
||||||
|
from src.db.repositories import LeagueRepository, MatchRepository, TeamRepository
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_date(value):
|
||||||
|
"""把 datetime / date / str 统一成 `date`。"""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
if hasattr(value, "date") and callable(value.date):
|
||||||
|
return value.date()
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _to_int_or_none(value) -> int | None:
|
||||||
|
"""宽松转 int(用于上游 ID 解析,失败返回 None 不抛错)。"""
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(str(value).strip())
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]:
|
||||||
|
"""比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。
|
||||||
|
|
||||||
|
统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类
|
||||||
|
隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配,
|
||||||
|
导致所有比赛被判为不存在而重复插入。
|
||||||
|
"""
|
||||||
|
d = _to_date(match_date)
|
||||||
|
return (home_team_id, away_team_id, d.isoformat() if d is not None else "")
|
||||||
|
|
||||||
|
|
||||||
|
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||||
|
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。
|
||||||
|
|
||||||
|
多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。
|
||||||
|
"""
|
||||||
|
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
||||||
|
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
||||||
|
ring = get_key_ring(base, raw_keys)
|
||||||
|
|
||||||
|
url = f"{base}/{path.lstrip('/')}"
|
||||||
|
key = ring.get()
|
||||||
|
if not key:
|
||||||
|
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||||
|
|
||||||
|
last_exc: Exception | None = None
|
||||||
|
for attempt in range(max_retries):
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Token {key}",
|
||||||
|
"Accept": "application/json",
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
client = get_client()
|
||||||
|
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
|
||||||
|
resp = await asyncio.wait_for(
|
||||||
|
client.get(
|
||||||
|
url, headers=headers, params=params,
|
||||||
|
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
|
||||||
|
),
|
||||||
|
timeout=60.0,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp.json()
|
||||||
|
except Exception as e:
|
||||||
|
last_exc = e
|
||||||
|
status = getattr(getattr(e, "response", None), "status_code", None)
|
||||||
|
if status == 429:
|
||||||
|
# 限流:标记当前 key 冷却,切换到下一个
|
||||||
|
new_key = ring.report_rate_limited(key)
|
||||||
|
if new_key and new_key != key:
|
||||||
|
logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _mask(key), _mask(new_key))
|
||||||
|
key = new_key
|
||||||
|
continue # 立即重试,不等待
|
||||||
|
# 单 key 或全部冷却:等待最早恢复的 key
|
||||||
|
wait = ring.wait_if_all_blocked()
|
||||||
|
if wait > 0:
|
||||||
|
logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait)
|
||||||
|
await asyncio.sleep(min(wait, 30.0))
|
||||||
|
else:
|
||||||
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||||
|
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
key = ring.get() or key
|
||||||
|
continue
|
||||||
|
if 500 <= (status or 0) < 600:
|
||||||
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||||
|
logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
continue
|
||||||
|
# 网络错误(连接失败/超时)也退避重试
|
||||||
|
if isinstance(e, (TimeoutError, ConnectionError, OSError)):
|
||||||
|
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
||||||
|
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
continue
|
||||||
|
raise
|
||||||
|
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_bzzoiro_events(
|
||||||
|
league_code: str,
|
||||||
|
*,
|
||||||
|
status: str = "finished",
|
||||||
|
date_from: str | None = None,
|
||||||
|
date_to: str | None = None,
|
||||||
|
limit: int = 200,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""抓取 bzzoiro 原始事件(纯异步,无需 run_in_executor)。"""
|
||||||
|
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
||||||
|
if league_id is None:
|
||||||
|
raise ValueError(f"未知联赛代码: {league_code}")
|
||||||
|
|
||||||
|
rows: list[dict] = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
params: dict = {
|
||||||
|
"league_id": league_id,
|
||||||
|
"status": status,
|
||||||
|
"limit": limit,
|
||||||
|
"offset": offset,
|
||||||
|
}
|
||||||
|
if date_from:
|
||||||
|
params["date_from"] = str(date_from)[:10]
|
||||||
|
if date_to:
|
||||||
|
params["date_to"] = str(date_to)[:10]
|
||||||
|
payload = await _fetch_json_async("/events/", params)
|
||||||
|
batch = payload.get("results") or []
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
rows.extend(batch)
|
||||||
|
total = payload.get("total")
|
||||||
|
offset += limit
|
||||||
|
if total is not None and offset >= total:
|
||||||
|
break
|
||||||
|
if len(batch) < limit:
|
||||||
|
break
|
||||||
|
await asyncio.sleep(REQUEST_INTERVAL)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
@register
|
||||||
|
class BzzoiroSource:
|
||||||
|
"""bzzoiro 数据源(实现 DataSource 协议)。"""
|
||||||
|
|
||||||
|
name = "bzzoiro"
|
||||||
|
|
||||||
|
async def ingest(
|
||||||
|
self,
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
leagues: Iterable[str],
|
||||||
|
date_from: str | None = None,
|
||||||
|
date_to: str | None = None,
|
||||||
|
status: str = "finished",
|
||||||
|
) -> dict:
|
||||||
|
"""采集 bzzoiro → 入库。返回统计。
|
||||||
|
|
||||||
|
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
||||||
|
"""
|
||||||
|
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||||
|
|
||||||
|
for code in leagues:
|
||||||
|
league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
|
||||||
|
try:
|
||||||
|
raw_events = await fetch_bzzoiro_events(code, status=status, date_from=date_from, date_to=date_to)
|
||||||
|
except Exception as e:
|
||||||
|
# 单联赛抓取失败隔离:记录错误后继续其余联赛,不拖垮整批
|
||||||
|
logger.exception("bzzoiro fetch failed for %s", code)
|
||||||
|
league_r["errors"].append(f"fetch failed: {e}")
|
||||||
|
await _safe_write_ingest_failure(
|
||||||
|
db,
|
||||||
|
entity_type="events",
|
||||||
|
source_record_id=None,
|
||||||
|
error=e,
|
||||||
|
raw_payload={"league": code, "status": status, "date_from": date_from, "date_to": date_to},
|
||||||
|
)
|
||||||
|
result["leagues"][code] = league_r
|
||||||
|
continue
|
||||||
|
|
||||||
|
# D4: 联赛查找/创建经 LeagueRepository(事务仍由调用方 UoW 提交)
|
||||||
|
league = await LeagueRepository(db).get_or_create(
|
||||||
|
code, LEAGUE_NAMES.get(code, code), LEAGUE_COUNTRIES.get(code)
|
||||||
|
)
|
||||||
|
team_r = TeamRepository(db)
|
||||||
|
match_r = MatchRepository(db)
|
||||||
|
|
||||||
|
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
||||||
|
team_name_to_id: dict[str, int] = {}
|
||||||
|
existing_matches: dict[tuple[int, int, str], Match] = {} # 完整对象,避免重复查询
|
||||||
|
# (NormalizedMatch, 原始 event) 成对保存:后续写 source_event_id 时
|
||||||
|
# 必须用配对的那条 event,不能依赖外层循环变量残留值。
|
||||||
|
normalized_matches: list[tuple] = []
|
||||||
|
|
||||||
|
if raw_events:
|
||||||
|
# 一次遍历: 收集球队名 + 规范化
|
||||||
|
all_team_names = set()
|
||||||
|
for raw in raw_events:
|
||||||
|
nm = normalize_bzzoiro(raw, code)
|
||||||
|
if nm is not None:
|
||||||
|
try:
|
||||||
|
nm.validate()
|
||||||
|
except Exception as e:
|
||||||
|
# P1-3: 统一使用 warning,不追加到 errors(仅运行时错误入 errors)
|
||||||
|
logger.warning("normalize skip: %s", e)
|
||||||
|
continue
|
||||||
|
normalized_matches.append((nm, raw))
|
||||||
|
all_team_names.add(nm.home_team)
|
||||||
|
all_team_names.add(nm.away_team)
|
||||||
|
|
||||||
|
if all_team_names:
|
||||||
|
team_name_to_id = {
|
||||||
|
name: t.id
|
||||||
|
for name, t in (await team_r.get_all_by_names(list(all_team_names))).items()
|
||||||
|
}
|
||||||
|
|
||||||
|
# P1-2: 按需加载,只加载 raw_events 涉及日期范围的比赛(加 30 天缓冲)
|
||||||
|
# 避免加载联赛全部历史比赛到内存(多赛季采集时内存溢出)
|
||||||
|
if normalized_matches:
|
||||||
|
# normalized_matches 存的是 (nm, raw) 元组,遍历需解包
|
||||||
|
dates = [nm.date for nm, _raw in normalized_matches if nm.date is not None]
|
||||||
|
if dates:
|
||||||
|
min_dt = min(dates) - timedelta(days=30)
|
||||||
|
max_dt = max(dates) + timedelta(days=30)
|
||||||
|
matches_in_range = await match_r.find_by_league_and_date_range(
|
||||||
|
league.id, min_dt, max_dt
|
||||||
|
)
|
||||||
|
existing_matches = {
|
||||||
|
_match_key(m.home_team_id, m.away_team_id, m.match_date_date): m
|
||||||
|
for m in matches_in_range
|
||||||
|
}
|
||||||
|
# else: existing_matches 保持空 dict(全量新比赛)
|
||||||
|
|
||||||
|
# D1: Bronze 层批次信息(每联赛每批次一个 batch_id;seen 防同批重复写入)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
bronze_batch_id = f"bzzoiro-events-{code}-{now:%Y%m%d%H%M%S}"
|
||||||
|
bronze_written: set[str] = set()
|
||||||
|
|
||||||
|
for nm, raw in normalized_matches:
|
||||||
|
# D1: RawEvent 幂等键(上游 id 或合成键),插入/变更更新共用
|
||||||
|
record_id = _events_record_id(code, nm, raw)
|
||||||
|
|
||||||
|
# 球队: 内存查找 + 按需创建(D4: 经 TeamRepository)
|
||||||
|
home_team_id = team_name_to_id.get(nm.home_team)
|
||||||
|
if home_team_id is None:
|
||||||
|
home = await team_r.get_or_create(nm.home_team, name_zh=zh_name(nm.home_team))
|
||||||
|
home_team_id = home.id
|
||||||
|
team_name_to_id[nm.home_team] = home_team_id
|
||||||
|
|
||||||
|
away_team_id = team_name_to_id.get(nm.away_team)
|
||||||
|
if away_team_id is None:
|
||||||
|
away = await team_r.get_or_create(nm.away_team, name_zh=zh_name(nm.away_team))
|
||||||
|
away_team_id = away.id
|
||||||
|
team_name_to_id[nm.away_team] = away_team_id
|
||||||
|
|
||||||
|
# 查找已有比赛: 内存查找
|
||||||
|
match_key = _match_key(home_team_id, away_team_id, nm.date)
|
||||||
|
existing_match = existing_matches.get(match_key)
|
||||||
|
|
||||||
|
if existing_match is None:
|
||||||
|
m = Match(
|
||||||
|
league_id=league.id,
|
||||||
|
season=nm.season_label or None,
|
||||||
|
home_team_id=home_team_id,
|
||||||
|
away_team_id=away_team_id,
|
||||||
|
match_date=nm.date,
|
||||||
|
match_date_date=_to_date(nm.date),
|
||||||
|
match_status=nm.match_status,
|
||||||
|
home_goals=nm.home_goals,
|
||||||
|
away_goals=nm.away_goals,
|
||||||
|
home_ht_goals=nm.home_ht_goals,
|
||||||
|
away_ht_goals=nm.away_ht_goals,
|
||||||
|
match_stage=nm.match_stage,
|
||||||
|
source_event_id=_to_int_or_none(raw.get("id")),
|
||||||
|
)
|
||||||
|
db.add(m)
|
||||||
|
await db.flush()
|
||||||
|
existing_matches[match_key] = m # 防止同批重复
|
||||||
|
# 统计字段不在 /events/ 载荷中(单独由 stats 管线回填),
|
||||||
|
# 此处不再创建 MatchStats。
|
||||||
|
league_r["inserted"] += 1
|
||||||
|
# D1: 成功插入 → 补写 Bronze 层(原始载荷 + 血缘)
|
||||||
|
if record_id not in bronze_written:
|
||||||
|
bronze_written.add(record_id)
|
||||||
|
await _write_events_bronze(
|
||||||
|
db,
|
||||||
|
source_record_id=record_id,
|
||||||
|
raw_payload=raw,
|
||||||
|
target_match_id=m.id,
|
||||||
|
league_code=code,
|
||||||
|
match_status=nm.match_status,
|
||||||
|
batch_id=bronze_batch_id,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# 已有比赛: 直接从内存获取对象更新(无需再查询)
|
||||||
|
changed = False
|
||||||
|
if existing_match.match_status != nm.match_status and nm.match_status == "finished":
|
||||||
|
existing_match.match_status = nm.match_status
|
||||||
|
changed = True
|
||||||
|
if existing_match.home_goals is None and nm.home_goals is not None:
|
||||||
|
existing_match.home_goals = nm.home_goals
|
||||||
|
existing_match.away_goals = nm.away_goals
|
||||||
|
existing_match.home_ht_goals = nm.home_ht_goals
|
||||||
|
existing_match.away_ht_goals = nm.away_ht_goals
|
||||||
|
changed = True
|
||||||
|
if existing_match.match_stage is None and nm.match_stage:
|
||||||
|
existing_match.match_stage = nm.match_stage
|
||||||
|
changed = True
|
||||||
|
if existing_match.source_event_id is None:
|
||||||
|
eid = _to_int_or_none(raw.get("id"))
|
||||||
|
if eid is not None:
|
||||||
|
existing_match.source_event_id = eid
|
||||||
|
changed = True
|
||||||
|
if changed:
|
||||||
|
league_r["updated"] += 1
|
||||||
|
# D1: 变更更新 → 补写血缘(RawEvent 幂等键不变,重复采集自动跳过)
|
||||||
|
if record_id not in bronze_written:
|
||||||
|
bronze_written.add(record_id)
|
||||||
|
await _write_events_bronze(
|
||||||
|
db,
|
||||||
|
source_record_id=record_id,
|
||||||
|
raw_payload=raw,
|
||||||
|
target_match_id=existing_match.id,
|
||||||
|
league_code=code,
|
||||||
|
match_status=nm.match_status,
|
||||||
|
batch_id=bronze_batch_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── events 管线:BzzoiroSource(注册表入口)+ 抓取/入库 ──
|
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
||||||
from src.data.bzzoiro_events import ( # noqa: F401
|
result["leagues"][code] = league_r
|
||||||
BzzoiroSource,
|
result["total_inserted"] += league_r["inserted"]
|
||||||
_events_record_id,
|
result["total_updated"] += league_r["updated"]
|
||||||
_write_events_bronze,
|
return result
|
||||||
fetch_bzzoiro_events,
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 管线基础设施:RawEvent / IngestFailure / DataLineage
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None:
|
||||||
|
"""写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。"""
|
||||||
|
from sqlalchemy import select as _select
|
||||||
|
stmt = _select(RawEvent).where(
|
||||||
|
RawEvent.source_system == source_system,
|
||||||
|
RawEvent.source_record_id == source_record_id,
|
||||||
|
)
|
||||||
|
existing = (await db.execute(stmt)).scalar_one_or_none()
|
||||||
|
if existing is None:
|
||||||
|
db.add(RawEvent(
|
||||||
|
source_system=source_system,
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
raw_payload=raw_payload,
|
||||||
|
ingest_batch_id=batch_id,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None:
|
||||||
|
"""写入采集失败死信。"""
|
||||||
|
db.add(IngestFailure(
|
||||||
|
source_system=source_system,
|
||||||
|
entity_type=entity_type,
|
||||||
|
source_record_id=source_record_id,
|
||||||
|
error_type=error_type,
|
||||||
|
error_detail=error_detail,
|
||||||
|
raw_payload=raw_payload,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
async def _safe_write_ingest_failure(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
entity_type: str,
|
||||||
|
source_record_id: str | None,
|
||||||
|
error: Exception,
|
||||||
|
raw_payload: dict | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""抓取失败时尽力写入死信表(失败不影响主流程)。
|
||||||
|
|
||||||
|
死信是「可观测性」基础设施,与 RawEvent/Lineage 同级:写入失败只记
|
||||||
|
warning,绝不能让原始抓取错误之外的新异常打断采集循环。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
await _write_ingest_failure(
|
||||||
|
db, "bzzoiro", entity_type, source_record_id,
|
||||||
|
"fetch_error", str(error), raw_payload,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"写入 ingest_failures 死信失败(entity=%s, record=%s): %s",
|
||||||
|
entity_type, source_record_id, error, exc_info=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── standings 管线 ──
|
|
||||||
from src.data.bzzoiro_standings import ( # noqa: F401
|
async def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
|
||||||
_season_label_from_dates,
|
"""写入 ETL 血缘追踪。"""
|
||||||
_write_standings_bronze,
|
db.add(DataLineage(
|
||||||
fetch_bzzoiro_standings,
|
source_system=source_system,
|
||||||
ingest_bzzoiro_standings,
|
source_record_id=source_record_id,
|
||||||
|
target_table=target_table,
|
||||||
|
target_id=target_id,
|
||||||
|
transform_name=transform_name,
|
||||||
|
transform_detail=transform_detail,
|
||||||
|
batch_id=batch_id,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def _events_record_id(league_code: str, nm, raw: dict) -> str:
|
||||||
|
"""events 载荷的 RawEvent 幂等键。
|
||||||
|
|
||||||
|
优先用上游 event id;缺失时用 (league:home:away:date) 合成稳定键 ——
|
||||||
|
取 normalize 后的队名与天级日期(与 _match_key 同口径),不依赖 DB 自增 id,
|
||||||
|
保证同一来源比赛重复采集时命中同一条 RawEvent,不产生重复原始载荷。
|
||||||
|
"""
|
||||||
|
eid = _to_int_or_none(raw.get("id"))
|
||||||
|
if eid is not None:
|
||||||
|
return str(eid)
|
||||||
|
d = _to_date(nm.date)
|
||||||
|
date_part = d.isoformat() if d is not None else "na"
|
||||||
|
return f"{league_code}:{nm.home_team}:{nm.away_team}:{date_part}"
|
||||||
|
|
||||||
|
|
||||||
|
async def _write_events_bronze(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
source_record_id: str,
|
||||||
|
raw_payload: dict,
|
||||||
|
target_match_id: int | None,
|
||||||
|
league_code: str,
|
||||||
|
match_status: str | None,
|
||||||
|
batch_id: str,
|
||||||
|
) -> None:
|
||||||
|
"""events 成功插入/更新单场比赛后的 Bronze 层补写:RawEvent(幂等) + DataLineage。
|
||||||
|
|
||||||
|
D1(工程债):此前只有 stats 回填写 RawEvent/Lineage,events 管线作为比赛
|
||||||
|
主数据的唯一入口反而不留溯源记录。幂等性由 _write_raw_event 的
|
||||||
|
source_record_id 查重保证;best-effort:基础设施写入失败只记 warning,
|
||||||
|
绝不拖垮采集主流程(与 _safe_write_ingest_failure 同级约束)。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
await _write_raw_event(db, "bzzoiro", source_record_id, raw_payload, batch_id)
|
||||||
|
await _write_lineage(
|
||||||
|
db, "bzzoiro", source_record_id,
|
||||||
|
"matches", target_match_id, "events_ingest",
|
||||||
|
{"league": league_code, "match_status": match_status},
|
||||||
|
batch_id,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
logger.warning(
|
||||||
|
"events Bronze 写入失败(record=%s, match=%s),不影响采集主流程",
|
||||||
|
source_record_id, target_match_id, exc_info=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── stats 回填管线 ──
|
|
||||||
from src.data.bzzoiro_stats import ( # noqa: F401
|
# ============================================================
|
||||||
_pick,
|
# 积分榜管线:/leagues/{id}/standings/ → standings 表
|
||||||
_stats_from_payload,
|
# ============================================================
|
||||||
ingest_bzzoiro_event_stats,
|
|
||||||
|
async def fetch_bzzoiro_standings(league_code: str, season: str | None = None) -> dict:
|
||||||
|
"""抓取联赛积分榜(纯抓取,不入库)。season 为 None 时取当前赛季。"""
|
||||||
|
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
||||||
|
if league_id is None:
|
||||||
|
raise ValueError(f"未知联赛代码: {league_code}")
|
||||||
|
params: dict = {}
|
||||||
|
if season:
|
||||||
|
params["season"] = season
|
||||||
|
return await _fetch_json_async(f"/leagues/{league_id}/standings/", params)
|
||||||
|
|
||||||
|
|
||||||
|
def _season_label_from_dates(start_date, end_date) -> str:
|
||||||
|
"""从赛季起止日期推导赛季标签(与 derive_season_label 语义一致)。"""
|
||||||
|
try:
|
||||||
|
if isinstance(start_date, str):
|
||||||
|
start = datetime.fromisoformat(start_date[:10])
|
||||||
|
else:
|
||||||
|
start = start_date
|
||||||
|
y = start.year
|
||||||
|
return f"{y}-{y + 1}" if start.month >= 8 else f"{y - 1}-{y}"
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return "?"
|
||||||
|
|
||||||
|
|
||||||
|
async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str | None = None) -> dict:
|
||||||
|
"""采集积分榜 → upsert standings 表。
|
||||||
|
|
||||||
|
season 为 None 时采集当前赛季(bzzoiro 默认返回 is_current 赛季)。
|
||||||
|
球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。
|
||||||
|
"""
|
||||||
|
from src.data.team_names import normalize as normalize_name
|
||||||
|
|
||||||
|
result: dict = {"leagues": {}, "total_upserted": 0, "errors": []}
|
||||||
|
for code in leagues:
|
||||||
|
league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0, "errors": []}
|
||||||
|
try:
|
||||||
|
payload = await fetch_bzzoiro_standings(code, season=season)
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception("bzzoiro standings fetch failed for %s", code)
|
||||||
|
league_r["errors"].append(str(e))
|
||||||
|
await _safe_write_ingest_failure(
|
||||||
|
db,
|
||||||
|
entity_type="standings",
|
||||||
|
source_record_id=None,
|
||||||
|
error=e,
|
||||||
|
raw_payload={"league": code, "season": season},
|
||||||
)
|
)
|
||||||
|
result["leagues"][code] = league_r
|
||||||
|
result["errors"].append(f"{code}: {e}")
|
||||||
|
continue
|
||||||
|
|
||||||
|
rows = payload.get("standings") or []
|
||||||
|
if not rows:
|
||||||
|
result["leagues"][code] = {"error": "无积分榜数据(赛季未开始或未提供)"}
|
||||||
|
result["errors"].append(f"{code}: 无积分榜数据")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 联赛(get-or-create,D4: 经 LeagueRepository)
|
||||||
|
league = await LeagueRepository(db).get_or_create(
|
||||||
|
code, LEAGUE_NAMES.get(code, code), LEAGUE_COUNTRIES.get(code)
|
||||||
|
)
|
||||||
|
team_r = TeamRepository(db)
|
||||||
|
|
||||||
|
# 赛季标签:优先用返回的 season 对象推导
|
||||||
|
season_obj = payload.get("season") or {}
|
||||||
|
season_label = _season_label_from_dates(
|
||||||
|
season_obj.get("start_date"), season_obj.get("end_date")
|
||||||
|
)
|
||||||
|
if season_label == "?":
|
||||||
|
season_label = season or ""
|
||||||
|
|
||||||
|
# 批量预载球队(与 events 管线使用同一 normalize 规则,保证 Team 匹配)
|
||||||
|
names = {normalize_name(str(r.get("team_name", ""))) for r in rows}
|
||||||
|
names.discard("")
|
||||||
|
team_map: dict[str, Team] = await team_r.get_all_by_names(list(names))
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
for r in rows:
|
||||||
|
team_name = normalize_name(str(r.get("team_name", "")))
|
||||||
|
if not team_name:
|
||||||
|
continue
|
||||||
|
team = team_map.get(team_name)
|
||||||
|
if team is None:
|
||||||
|
team = await team_r.get_or_create(team_name, name_zh=zh_name(team_name))
|
||||||
|
team_map[team_name] = team
|
||||||
|
league_r["teams_created"] += 1
|
||||||
|
|
||||||
|
zone = r.get("zone") or {}
|
||||||
|
values = dict(
|
||||||
|
position=_to_int_or_none(r.get("position")) or 0,
|
||||||
|
played=_to_int_or_none(r.get("played")) or 0,
|
||||||
|
won=_to_int_or_none(r.get("won")) or 0,
|
||||||
|
drawn=_to_int_or_none(r.get("drawn")) or 0,
|
||||||
|
lost=_to_int_or_none(r.get("lost")) or 0,
|
||||||
|
goals_for=_to_int_or_none(r.get("gf")) or 0,
|
||||||
|
goals_against=_to_int_or_none(r.get("ga")) or 0,
|
||||||
|
goal_diff=_to_int_or_none(r.get("gd")) or 0,
|
||||||
|
points=_to_int_or_none(r.get("pts")) or 0,
|
||||||
|
xg_for=_to_float_or_none(r.get("xgf")),
|
||||||
|
xg_against=_to_float_or_none(r.get("xga")),
|
||||||
|
form=r.get("form") or None,
|
||||||
|
zone=zone.get("label") or zone.get("key") or None,
|
||||||
|
updated_at=now,
|
||||||
|
retrieved_at=now,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 同一联赛同一赛季只保留最新快照:按 (league, season, team) upsert
|
||||||
|
stmt = select(Standing).where(
|
||||||
|
Standing.league_id == league.id,
|
||||||
|
Standing.season == season_label,
|
||||||
|
Standing.team_id == team.id,
|
||||||
|
)
|
||||||
|
standing = (await db.execute(stmt)).scalar_one_or_none()
|
||||||
|
if standing is None:
|
||||||
|
standing = Standing(
|
||||||
|
league_id=league.id, season=season_label, team_id=team.id, **values
|
||||||
|
)
|
||||||
|
db.add(standing)
|
||||||
|
else:
|
||||||
|
for k, v in values.items():
|
||||||
|
setattr(standing, k, v)
|
||||||
|
league_r["upserted"] += 1
|
||||||
|
|
||||||
|
league_r["rows"] = len(rows)
|
||||||
|
result["leagues"][code] = league_r
|
||||||
|
result["total_upserted"] += league_r["upserted"]
|
||||||
|
logger.info(
|
||||||
|
"bzzoiro standings 采集完成: %s 赛季 %s, upsert %d/%d",
|
||||||
|
code, season_label, league_r["upserted"], league_r["rows"],
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 统计回填管线:/events/{id}/stats/ → match_stats 表
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
# bzzoiro stats 字段 → MatchStats 字段映射(stats.home / stats.away 下)
|
||||||
|
_STATS_FIELD_MAP = {
|
||||||
|
"xg": ("home_xg", "away_xg"), # 回退 expected_goals
|
||||||
|
"ball_possession": ("home_possession", None), # 只取主队值,客队=100-home
|
||||||
|
"total_shots": ("home_shots", "away_shots"),
|
||||||
|
"shots_on_target": ("home_shots_on_target", "away_shots_on_target"),
|
||||||
|
"corner_kicks": ("home_corners", "away_corners"),
|
||||||
|
"yellow_cards": ("home_yellow_cards", "away_yellow_cards"),
|
||||||
|
"red_cards": ("home_red_cards", "away_red_cards"),
|
||||||
|
"big_chances": ("home_big_chances", "away_big_chances"),
|
||||||
|
"fouls": ("home_fouls", "away_fouls"),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _pick(d: dict, *keys):
|
||||||
|
"""按优先级取第一个非空字段值。"""
|
||||||
|
for k in keys:
|
||||||
|
v = d.get(k)
|
||||||
|
if v is not None:
|
||||||
|
return v
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _stats_from_payload(payload: dict) -> dict:
|
||||||
|
"""把 /events/{id}/stats/ 响应映射成 MatchStats 字段 dict。
|
||||||
|
|
||||||
|
响应结构: {"event_id": ..., "stats": {"home": {...}, "away": {...}}}
|
||||||
|
"""
|
||||||
|
stats = (payload or {}).get("stats") or {}
|
||||||
|
home = stats.get("home") or {}
|
||||||
|
away = stats.get("away") or {}
|
||||||
|
out: dict = {}
|
||||||
|
|
||||||
|
xg_h = _pick(home, "xg", "expected_goals")
|
||||||
|
xg_a = _pick(away, "xg", "expected_goals")
|
||||||
|
if xg_h is not None:
|
||||||
|
out["home_xg"] = _to_float_or_none(xg_h)
|
||||||
|
if xg_a is not None:
|
||||||
|
out["away_xg"] = _to_float_or_none(xg_a)
|
||||||
|
|
||||||
|
poss = home.get("ball_possession")
|
||||||
|
if poss is not None:
|
||||||
|
p = _to_float_or_none(poss)
|
||||||
|
if p is not None:
|
||||||
|
out["home_possession"] = p
|
||||||
|
|
||||||
|
for src, (h_fld, a_fld) in _STATS_FIELD_MAP.items():
|
||||||
|
if src in ("xg", "ball_possession"):
|
||||||
|
continue # 已处理
|
||||||
|
hv = home.get(src)
|
||||||
|
av = away.get(src)
|
||||||
|
if hv is not None and h_fld:
|
||||||
|
out[h_fld] = _to_int_or_none(hv)
|
||||||
|
if av is not None and a_fld:
|
||||||
|
out[a_fld] = _to_int_or_none(av)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _to_float_or_none(value) -> float | None:
|
||||||
|
if value is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(str(value).strip())
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
async def ingest_bzzoiro_event_stats(
|
||||||
|
db,
|
||||||
|
*,
|
||||||
|
leagues: Iterable[str],
|
||||||
|
limit: int = 100,
|
||||||
|
only_missing: bool = True,
|
||||||
|
) -> dict:
|
||||||
|
"""回填已完赛比赛的详细统计(逐场调 /events/{id}/stats/)。
|
||||||
|
|
||||||
|
筛选条件: match_status=finished 且 source_event_id 非空。
|
||||||
|
only_missing=True 时跳过已有统计的比赛(增量);False 则全量刷新。
|
||||||
|
limit 控制单次最多处理的比赛数(上游限速 1.2s/请求,大批量需分次触发)。
|
||||||
|
"""
|
||||||
|
result: dict = {"fetched": 0, "created": 0, "updated": 0, "skipped": 0, "errors": []}
|
||||||
|
|
||||||
|
league_ids = [BZZOIRO_LEAGUE_IDS[c] for c in leagues if c in BZZOIRO_LEAGUE_IDS]
|
||||||
|
if not league_ids:
|
||||||
|
result["errors"].append("无有效联赛代码")
|
||||||
|
return result
|
||||||
|
|
||||||
|
# D4: 候选比赛查询经 MatchRepository(含 stats 预加载,筛选/排序/limit 语义不变)
|
||||||
|
matches = await MatchRepository(db).find_finished_with_stats(
|
||||||
|
league_ids, limit=limit * 3 if only_missing else limit
|
||||||
|
)
|
||||||
|
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
processed = 0
|
||||||
|
for m in matches:
|
||||||
|
if processed >= limit:
|
||||||
|
break
|
||||||
|
if only_missing and m.stats is not None and m.stats.home_shots is not None:
|
||||||
|
result["skipped"] += 1
|
||||||
|
continue
|
||||||
|
processed += 1
|
||||||
|
try:
|
||||||
|
payload = await _fetch_json_async(f"/events/{m.source_event_id}/stats/")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("stats fetch failed match=%s event=%s: %s", m.id, m.source_event_id, e)
|
||||||
|
result["errors"].append(f"match {m.id}: {e}")
|
||||||
|
await _safe_write_ingest_failure(
|
||||||
|
db,
|
||||||
|
entity_type="match_stats",
|
||||||
|
source_record_id=str(m.source_event_id),
|
||||||
|
error=e,
|
||||||
|
raw_payload={"match_id": m.id},
|
||||||
|
)
|
||||||
|
await asyncio.sleep(REQUEST_INTERVAL)
|
||||||
|
continue
|
||||||
|
|
||||||
|
result["fetched"] += 1
|
||||||
|
fields = _stats_from_payload(payload)
|
||||||
|
if not fields:
|
||||||
|
result["skipped"] += 1
|
||||||
|
await asyncio.sleep(REQUEST_INTERVAL)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if m.stats is None:
|
||||||
|
available_at = m.match_date + timedelta(hours=2) if m.match_date else now
|
||||||
|
m.stats = MatchStats(
|
||||||
|
match_id=m.id,
|
||||||
|
source="bzzoiro",
|
||||||
|
source_record_id=str(m.source_event_id),
|
||||||
|
retrieved_at=now,
|
||||||
|
available_at=available_at,
|
||||||
|
)
|
||||||
|
db.add(m.stats)
|
||||||
|
result["created"] += 1
|
||||||
|
else:
|
||||||
|
result["updated"] += 1
|
||||||
|
if m.stats.source is None:
|
||||||
|
m.stats.source = "bzzoiro"
|
||||||
|
m.stats.source_record_id = str(m.source_event_id)
|
||||||
|
if m.stats.retrieved_at is None:
|
||||||
|
m.stats.retrieved_at = now
|
||||||
|
if m.stats.available_at is None and m.match_date:
|
||||||
|
m.stats.available_at = m.match_date + timedelta(hours=2)
|
||||||
|
|
||||||
|
for fld, v in fields.items():
|
||||||
|
if hasattr(m.stats, fld):
|
||||||
|
setattr(m.stats, fld, v)
|
||||||
|
|
||||||
|
# 管线基础设施:写入 RawEvent + DataLineage
|
||||||
|
batch_id = f"bzzoiro-stats-{m.source_event_id}-{now.strftime('%Y%m%d%H%M%S')}"
|
||||||
|
try:
|
||||||
|
await _write_raw_event(db, "bzzoiro", str(m.source_event_id), payload, batch_id)
|
||||||
|
await _write_lineage(db, "bzzoiro", str(m.source_event_id), "match_stats", m.stats.id if m.stats else None, "stats_backfill", {"match_id": m.id}, batch_id)
|
||||||
|
except Exception:
|
||||||
|
pass # 基础设施写入失败不影响主流程
|
||||||
|
|
||||||
|
await asyncio.sleep(REQUEST_INTERVAL)
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
"bzzoiro stats 回填完成: 抓取 %d, 新建 %d, 更新 %d, 跳过 %d, 错误 %d",
|
||||||
|
result["fetched"], result["created"], result["updated"],
|
||||||
|
result["skipped"], len(result["errors"]),
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|||||||
@@ -1,123 +0,0 @@
|
|||||||
"""bzzoiro 管线共享原语:HTTP 抓取(多 key 轮换)与宽松字段转换。
|
|
||||||
|
|
||||||
从 bzzoiro.py 拆出(单文件 → 多模块):仅放无业务语义的共享基础,
|
|
||||||
三条管线(events/standings/stats)与聚合门面见 bzzoiro.py。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
import random
|
|
||||||
|
|
||||||
import httpx
|
|
||||||
|
|
||||||
from src.core.http_client import get_client
|
|
||||||
from src.core.runtime_config import get_runtime_value
|
|
||||||
from src.data.key_ring import _mask, get_key_ring
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _to_date(value):
|
|
||||||
"""把 datetime / date / str 统一成 `date`。"""
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
if hasattr(value, "date") and callable(value.date):
|
|
||||||
return value.date()
|
|
||||||
return value
|
|
||||||
|
|
||||||
|
|
||||||
def _to_int_or_none(value) -> int | None:
|
|
||||||
"""宽松转 int(用于上游 ID 解析,失败返回 None 不抛错)。"""
|
|
||||||
if value is None:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return int(str(value).strip())
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _to_float_or_none(value) -> float | None:
|
|
||||||
try:
|
|
||||||
return float(str(value).strip())
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, int, str]:
|
|
||||||
"""比赛去重键:(主队, 客队, 天级日期 ISO 字符串)。
|
|
||||||
|
|
||||||
统一在这里构造,避免"预加载时用 str(date)、写入时用 isoformat()"这类
|
|
||||||
隐式格式依赖 —— 两者当前恰好相等,但一旦有人改动其一就会静默失配,
|
|
||||||
导致所有比赛被判为不存在而重复插入。
|
|
||||||
"""
|
|
||||||
d = _to_date(match_date)
|
|
||||||
return (home_team_id, away_team_id, d.isoformat() if d is not None else "")
|
|
||||||
|
|
||||||
|
|
||||||
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
|
||||||
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。
|
|
||||||
|
|
||||||
多 key 轮换:遇到 429 自动切换到下一个 key;全部 key 冷却时等待最早恢复。
|
|
||||||
"""
|
|
||||||
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
|
|
||||||
raw_keys = await get_runtime_value("BZZOIRO_KEY")
|
|
||||||
ring = get_key_ring(base, raw_keys)
|
|
||||||
|
|
||||||
url = f"{base}/{path.lstrip('/')}"
|
|
||||||
key = ring.get()
|
|
||||||
if not key:
|
|
||||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
|
||||||
|
|
||||||
last_exc: Exception | None = None
|
|
||||||
for attempt in range(max_retries):
|
|
||||||
headers = {
|
|
||||||
"Authorization": f"Token {key}",
|
|
||||||
"Accept": "application/json",
|
|
||||||
}
|
|
||||||
try:
|
|
||||||
client = get_client()
|
|
||||||
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
|
|
||||||
resp = await asyncio.wait_for(
|
|
||||||
client.get(
|
|
||||||
url, headers=headers, params=params,
|
|
||||||
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
|
|
||||||
),
|
|
||||||
timeout=60.0,
|
|
||||||
)
|
|
||||||
resp.raise_for_status()
|
|
||||||
return resp.json()
|
|
||||||
except Exception as e:
|
|
||||||
last_exc = e
|
|
||||||
status = getattr(getattr(e, "response", None), "status_code", None)
|
|
||||||
if status == 429:
|
|
||||||
# 限流:标记当前 key 冷却,切换到下一个
|
|
||||||
new_key = ring.report_rate_limited(key)
|
|
||||||
if new_key and new_key != key:
|
|
||||||
logger.info("bzzoiro 429 → 切换 key: %s → %s,立即重试", _mask(key), _mask(new_key))
|
|
||||||
key = new_key
|
|
||||||
continue # 立即重试,不等待
|
|
||||||
# 单 key 或全部冷却:等待最早恢复的 key
|
|
||||||
wait = ring.wait_if_all_blocked()
|
|
||||||
if wait > 0:
|
|
||||||
logger.warning("bzzoiro 全部 key 冷却,等待 %.1fs 后重试", wait)
|
|
||||||
await asyncio.sleep(min(wait, 30.0))
|
|
||||||
else:
|
|
||||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
||||||
logger.warning("bzzoiro 429, retry %d in %.1fs", attempt + 1, delay)
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
key = ring.get() or key
|
|
||||||
continue
|
|
||||||
if 500 <= (status or 0) < 600:
|
|
||||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
||||||
logger.warning("bzzoiro %d, retry %d in %.1fs", status, attempt + 1, delay)
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
continue
|
|
||||||
# 网络错误(连接失败/超时)也退避重试
|
|
||||||
if isinstance(e, (TimeoutError, ConnectionError, OSError)):
|
|
||||||
delay = min(2 ** attempt, 16) + random.uniform(0, 1)
|
|
||||||
logger.warning("bzzoiro network error, retry %d in %.1fs: %s", attempt + 1, delay, e)
|
|
||||||
await asyncio.sleep(delay)
|
|
||||||
continue
|
|
||||||
raise
|
|
||||||
raise RuntimeError(f"bzzoiro request failed after {max_retries} attempts: {last_exc}")
|
|
||||||
@@ -1,327 +0,0 @@
|
|||||||
"""bzzoiro events 管线:比赛日程/比分抓取(/events/)与入库(matches 表)。
|
|
||||||
|
|
||||||
从 bzzoiro.py 拆出。比赛主数据唯一入口;Team/League/Match 查找/创建经
|
|
||||||
Repository 层,事务由调用方 UnitOfWork 控制(分批事务约定不变)。
|
|
||||||
|
|
||||||
可替换协作者(抓取函数 / Bronze 写入助手 / REQUEST_INTERVAL)在运行期
|
|
||||||
经聚合门面 src.data.bzzoiro 解析 —— 与拆分前的单文件 monkeypatch 语义一致。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
from collections.abc import Iterable
|
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
|
|
||||||
from src.data.bzzoiro_common import _match_key, _to_date, _to_int_or_none
|
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES
|
|
||||||
from src.data.normalize import normalize_bzzoiro
|
|
||||||
from src.data.pipeline_write import _safe_write_ingest_failure, _write_lineage, _write_raw_event
|
|
||||||
from src.data.sources import register
|
|
||||||
from src.data.team_names_zh import zh_name
|
|
||||||
from src.db.models import Match
|
|
||||||
from src.db.repositories import LeagueRepository, MatchRepository, TeamRepository
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_bzzoiro_events(
|
|
||||||
league_code: str,
|
|
||||||
*,
|
|
||||||
status: str = "finished",
|
|
||||||
date_from: str | None = None,
|
|
||||||
date_to: str | None = None,
|
|
||||||
limit: int = 200,
|
|
||||||
) -> list[dict]:
|
|
||||||
"""抓取 bzzoiro 原始事件(纯异步,无需 run_in_executor)。"""
|
|
||||||
from src.data import bzzoiro as bz
|
|
||||||
|
|
||||||
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
|
||||||
if league_id is None:
|
|
||||||
raise ValueError(f"未知联赛代码: {league_code}")
|
|
||||||
|
|
||||||
rows: list[dict] = []
|
|
||||||
offset = 0
|
|
||||||
while True:
|
|
||||||
params: dict = {
|
|
||||||
"league_id": league_id,
|
|
||||||
"status": status,
|
|
||||||
"limit": limit,
|
|
||||||
"offset": offset,
|
|
||||||
}
|
|
||||||
if date_from:
|
|
||||||
params["date_from"] = str(date_from)[:10]
|
|
||||||
if date_to:
|
|
||||||
params["date_to"] = str(date_to)[:10]
|
|
||||||
payload = await bz._fetch_json_async("/events/", params)
|
|
||||||
batch = payload.get("results") or []
|
|
||||||
if not batch:
|
|
||||||
break
|
|
||||||
rows.extend(batch)
|
|
||||||
total = payload.get("total")
|
|
||||||
offset += limit
|
|
||||||
if total is not None and offset >= total:
|
|
||||||
break
|
|
||||||
if len(batch) < limit:
|
|
||||||
break
|
|
||||||
await asyncio.sleep(bz.REQUEST_INTERVAL)
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
@register
|
|
||||||
class BzzoiroSource:
|
|
||||||
"""bzzoiro 数据源(实现 DataSource 协议)。"""
|
|
||||||
|
|
||||||
name = "bzzoiro"
|
|
||||||
|
|
||||||
async def ingest(
|
|
||||||
self,
|
|
||||||
db,
|
|
||||||
*,
|
|
||||||
leagues: Iterable[str],
|
|
||||||
date_from: str | None = None,
|
|
||||||
date_to: str | None = None,
|
|
||||||
status: str = "finished",
|
|
||||||
) -> dict:
|
|
||||||
"""采集 bzzoiro → 入库。返回统计。
|
|
||||||
|
|
||||||
注意: 本方法不控制事务(commit/rollback),由调用方通过 UnitOfWork 控制。
|
|
||||||
"""
|
|
||||||
from src.data import bzzoiro as bz
|
|
||||||
|
|
||||||
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
|
||||||
|
|
||||||
for code in leagues:
|
|
||||||
league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
|
|
||||||
try:
|
|
||||||
raw_events = await bz.fetch_bzzoiro_events(code, status=status, date_from=date_from, date_to=date_to)
|
|
||||||
except Exception as e:
|
|
||||||
# 单联赛抓取失败隔离:记录错误后继续其余联赛,不拖垮整批
|
|
||||||
logger.exception("bzzoiro fetch failed for %s", code)
|
|
||||||
league_r["errors"].append(f"fetch failed: {e}")
|
|
||||||
await _safe_write_ingest_failure(
|
|
||||||
db,
|
|
||||||
entity_type="events",
|
|
||||||
source_record_id=None,
|
|
||||||
error=e,
|
|
||||||
raw_payload={"league": code, "status": status, "date_from": date_from, "date_to": date_to},
|
|
||||||
)
|
|
||||||
result["leagues"][code] = league_r
|
|
||||||
continue
|
|
||||||
|
|
||||||
# D4: 联赛查找/创建经 LeagueRepository(事务仍由调用方 UoW 提交)
|
|
||||||
league = await LeagueRepository(db).get_or_create(
|
|
||||||
code, LEAGUE_NAMES.get(code, code), LEAGUE_COUNTRIES.get(code)
|
|
||||||
)
|
|
||||||
team_r = TeamRepository(db)
|
|
||||||
match_r = MatchRepository(db)
|
|
||||||
|
|
||||||
# === 批量优化: 预加载球队和已有比赛到内存 ===
|
|
||||||
team_name_to_id: dict[str, int] = {}
|
|
||||||
existing_matches: dict[tuple[int, int, str], Match] = {} # 完整对象,避免重复查询
|
|
||||||
# (NormalizedMatch, 原始 event) 成对保存:后续写 source_event_id 时
|
|
||||||
# 必须用配对的那条 event,不能依赖外层循环变量残留值。
|
|
||||||
normalized_matches: list[tuple] = []
|
|
||||||
|
|
||||||
if raw_events:
|
|
||||||
# 一次遍历: 收集球队名 + 规范化
|
|
||||||
all_team_names = set()
|
|
||||||
for raw in raw_events:
|
|
||||||
nm = normalize_bzzoiro(raw, code)
|
|
||||||
if nm is not None:
|
|
||||||
try:
|
|
||||||
nm.validate()
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
normalized_matches.append((nm, raw))
|
|
||||||
all_team_names.add(nm.home_team)
|
|
||||||
all_team_names.add(nm.away_team)
|
|
||||||
|
|
||||||
if all_team_names:
|
|
||||||
team_name_to_id = {
|
|
||||||
name: t.id
|
|
||||||
for name, t in (await team_r.get_all_by_names(list(all_team_names))).items()
|
|
||||||
}
|
|
||||||
|
|
||||||
# P1-2: 按需加载,只加载 raw_events 涉及日期范围的比赛(加 30 天缓冲)
|
|
||||||
# 避免加载联赛全部历史比赛到内存(多赛季采集时内存溢出)
|
|
||||||
if normalized_matches:
|
|
||||||
# normalized_matches 存的是 (nm, raw) 元组,遍历需解包
|
|
||||||
dates = [nm.date for nm, _raw in normalized_matches if nm.date is not None]
|
|
||||||
if dates:
|
|
||||||
min_dt = min(dates) - timedelta(days=30)
|
|
||||||
max_dt = max(dates) + timedelta(days=30)
|
|
||||||
matches_in_range = await match_r.find_by_league_and_date_range(
|
|
||||||
league.id, min_dt, max_dt
|
|
||||||
)
|
|
||||||
existing_matches = {
|
|
||||||
_match_key(m.home_team_id, m.away_team_id, m.match_date_date): m
|
|
||||||
for m in matches_in_range
|
|
||||||
}
|
|
||||||
# else: existing_matches 保持空 dict(全量新比赛)
|
|
||||||
|
|
||||||
# D1: Bronze 层批次信息(每联赛每批次一个 batch_id;seen 防同批重复写入)
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
bronze_batch_id = f"bzzoiro-events-{code}-{now:%Y%m%d%H%M%S}"
|
|
||||||
bronze_written: set[str] = set()
|
|
||||||
|
|
||||||
for nm, raw in normalized_matches:
|
|
||||||
# D1: RawEvent 幂等键(上游 id 或合成键),插入/变更更新共用
|
|
||||||
record_id = _events_record_id(code, nm, raw)
|
|
||||||
|
|
||||||
# 球队: 内存查找 + 按需创建(D4: 经 TeamRepository)
|
|
||||||
home_team_id = team_name_to_id.get(nm.home_team)
|
|
||||||
if home_team_id is None:
|
|
||||||
home = await team_r.get_or_create(nm.home_team, name_zh=zh_name(nm.home_team))
|
|
||||||
home_team_id = home.id
|
|
||||||
team_name_to_id[nm.home_team] = home_team_id
|
|
||||||
|
|
||||||
away_team_id = team_name_to_id.get(nm.away_team)
|
|
||||||
if away_team_id is None:
|
|
||||||
away = await team_r.get_or_create(nm.away_team, name_zh=zh_name(nm.away_team))
|
|
||||||
away_team_id = away.id
|
|
||||||
team_name_to_id[nm.away_team] = away_team_id
|
|
||||||
|
|
||||||
# 查找已有比赛:优先按 upstream event_id 定位(命中即唯一),
|
|
||||||
# 否则回退自然键(联赛+主客+天级日期)内存查找。
|
|
||||||
# source_event_id 上有 partial unique 索引保障 upstream 唯一。
|
|
||||||
eid = _to_int_or_none(raw.get("id"))
|
|
||||||
existing_match = None
|
|
||||||
if eid is not None:
|
|
||||||
existing_match = await match_r.find_by_source_event_id(eid)
|
|
||||||
if existing_match is None:
|
|
||||||
match_key = _match_key(home_team_id, away_team_id, nm.date)
|
|
||||||
existing_match = existing_matches.get(match_key)
|
|
||||||
|
|
||||||
if existing_match is None:
|
|
||||||
m = Match(
|
|
||||||
league_id=league.id,
|
|
||||||
season=nm.season_label or None,
|
|
||||||
home_team_id=home_team_id,
|
|
||||||
away_team_id=away_team_id,
|
|
||||||
match_date=nm.date,
|
|
||||||
match_date_date=_to_date(nm.date),
|
|
||||||
match_status=nm.match_status,
|
|
||||||
score_status=nm.score_status,
|
|
||||||
home_goals=nm.home_goals,
|
|
||||||
away_goals=nm.away_goals,
|
|
||||||
home_ht_goals=nm.home_ht_goals,
|
|
||||||
away_ht_goals=nm.away_ht_goals,
|
|
||||||
match_stage=nm.match_stage,
|
|
||||||
source_event_id=_to_int_or_none(raw.get("id")),
|
|
||||||
)
|
|
||||||
db.add(m)
|
|
||||||
await db.flush()
|
|
||||||
existing_matches[match_key] = m # 防止同批重复
|
|
||||||
# 统计字段不在 /events/ 载荷中(单独由 stats 管线回填),
|
|
||||||
# 此处不再创建 MatchStats。
|
|
||||||
league_r["inserted"] += 1
|
|
||||||
# D1: 成功插入 → 补写 Bronze 层(原始载荷 + 血缘)
|
|
||||||
if record_id not in bronze_written:
|
|
||||||
bronze_written.add(record_id)
|
|
||||||
await _write_events_bronze(
|
|
||||||
db,
|
|
||||||
source_record_id=record_id,
|
|
||||||
raw_payload=raw,
|
|
||||||
target_match_id=m.id,
|
|
||||||
league_code=code,
|
|
||||||
match_status=nm.match_status,
|
|
||||||
batch_id=bronze_batch_id,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# 已有比赛: 直接从内存获取对象更新(无需再查询)
|
|
||||||
changed = False
|
|
||||||
if existing_match.match_status != nm.match_status and nm.match_status == "finished":
|
|
||||||
existing_match.match_status = nm.match_status
|
|
||||||
changed = True
|
|
||||||
if existing_match.home_goals is None and nm.home_goals is not None:
|
|
||||||
existing_match.home_goals = nm.home_goals
|
|
||||||
existing_match.away_goals = nm.away_goals
|
|
||||||
existing_match.home_ht_goals = nm.home_ht_goals
|
|
||||||
existing_match.away_ht_goals = nm.away_ht_goals
|
|
||||||
# 比分由缺变有 → 标记 known
|
|
||||||
existing_match.score_status = "known"
|
|
||||||
changed = True
|
|
||||||
elif (
|
|
||||||
nm.match_status == "finished"
|
|
||||||
and nm.home_goals is None
|
|
||||||
and existing_match.score_status == "unknown"
|
|
||||||
):
|
|
||||||
# 确认完赛仍缺分 → 标记 missing(不伪造 0:0)
|
|
||||||
existing_match.score_status = "missing"
|
|
||||||
changed = True
|
|
||||||
if existing_match.match_stage is None and nm.match_stage:
|
|
||||||
existing_match.match_stage = nm.match_stage
|
|
||||||
changed = True
|
|
||||||
if existing_match.source_event_id is None:
|
|
||||||
eid = _to_int_or_none(raw.get("id"))
|
|
||||||
if eid is not None:
|
|
||||||
existing_match.source_event_id = eid
|
|
||||||
changed = True
|
|
||||||
if changed:
|
|
||||||
league_r["updated"] += 1
|
|
||||||
# D1: 变更更新 → 补写血缘(RawEvent 幂等键不变,重复采集自动跳过)
|
|
||||||
if record_id not in bronze_written:
|
|
||||||
bronze_written.add(record_id)
|
|
||||||
await _write_events_bronze(
|
|
||||||
db,
|
|
||||||
source_record_id=record_id,
|
|
||||||
raw_payload=raw,
|
|
||||||
target_match_id=existing_match.id,
|
|
||||||
league_code=code,
|
|
||||||
match_status=nm.match_status,
|
|
||||||
batch_id=bronze_batch_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
# 注意: 不在此处 commit,由调用方 UnitOfWork 控制事务
|
|
||||||
result["leagues"][code] = league_r
|
|
||||||
result["total_inserted"] += league_r["inserted"]
|
|
||||||
result["total_updated"] += league_r["updated"]
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _events_record_id(league_code: str, nm, raw: dict) -> str:
|
|
||||||
"""events 载荷的 RawEvent 幂等键。
|
|
||||||
|
|
||||||
优先用上游 event id;缺失时用 (league:home:away:date) 合成稳定键 ——
|
|
||||||
取 normalize 后的队名与天级日期(与 _match_key 同口径),不依赖 DB 自增 id,
|
|
||||||
保证同一来源比赛重复采集时命中同一条 RawEvent,不产生重复原始载荷。
|
|
||||||
"""
|
|
||||||
eid = _to_int_or_none(raw.get("id"))
|
|
||||||
if eid is not None:
|
|
||||||
return str(eid)
|
|
||||||
d = _to_date(nm.date)
|
|
||||||
date_part = d.isoformat() if d is not None else "na"
|
|
||||||
return f"{league_code}:{nm.home_team}:{nm.away_team}:{date_part}"
|
|
||||||
|
|
||||||
|
|
||||||
async def _write_events_bronze(
|
|
||||||
db,
|
|
||||||
*,
|
|
||||||
source_record_id: str,
|
|
||||||
raw_payload: dict,
|
|
||||||
target_match_id: int | None,
|
|
||||||
league_code: str,
|
|
||||||
match_status: str | None,
|
|
||||||
batch_id: str,
|
|
||||||
) -> None:
|
|
||||||
"""events 成功插入/更新单场比赛后的 Bronze 层补写:RawEvent(幂等) + DataLineage。
|
|
||||||
|
|
||||||
D1(工程债):此前只有 stats 回填写 RawEvent/Lineage,events 管线作为比赛
|
|
||||||
主数据的唯一入口反而不留溯源记录。幂等性由 _write_raw_event 的
|
|
||||||
source_record_id 查重保证;best-effort:基础设施写入失败只记 warning,
|
|
||||||
绝不拖垮采集主流程(与 _safe_write_ingest_failure 同级约束)。
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
await _write_raw_event(db, "bzzoiro", source_record_id, raw_payload, batch_id)
|
|
||||||
await _write_lineage(
|
|
||||||
db, "bzzoiro", source_record_id,
|
|
||||||
"matches", target_match_id, "events_ingest",
|
|
||||||
{"league": league_code, "match_status": match_status},
|
|
||||||
batch_id,
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
logger.warning(
|
|
||||||
"events Bronze 写入失败(record=%s, match=%s),不影响采集主流程",
|
|
||||||
source_record_id, target_match_id, exc_info=True,
|
|
||||||
)
|
|
||||||
@@ -1,207 +0,0 @@
|
|||||||
"""bzzoiro standings 管线:联赛积分榜快照(/leagues/{id}/standings/)→ standings 表。
|
|
||||||
|
|
||||||
从 bzzoiro.py 拆出。同一联赛同一赛季只保留最新快照(按 (league, season, team)
|
|
||||||
upsert);球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。
|
|
||||||
|
|
||||||
可替换协作者(抓取函数 / Bronze 写入助手)在运行期经聚合门面
|
|
||||||
src.data.bzzoiro 解析 —— 与拆分前的单文件 monkeypatch 语义一致。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from collections.abc import Iterable
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from src.data.bzzoiro_common import _to_float_or_none, _to_int_or_none
|
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES
|
|
||||||
from src.data.pipeline_write import _safe_write_ingest_failure, _write_lineage, _write_raw_event
|
|
||||||
from src.data.team_names_zh import zh_name
|
|
||||||
from src.db.models import Standing, Team
|
|
||||||
from src.db.repositories import LeagueRepository, TeamRepository
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
async def fetch_bzzoiro_standings(league_code: str, season: str | None = None) -> dict:
|
|
||||||
"""抓取联赛积分榜(纯抓取,不入库)。season 为 None 时取当前赛季。"""
|
|
||||||
from src.data import bzzoiro as bz
|
|
||||||
|
|
||||||
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
|
||||||
if league_id is None:
|
|
||||||
raise ValueError(f"未知联赛代码: {league_code}")
|
|
||||||
params: dict = {}
|
|
||||||
if season:
|
|
||||||
params["season"] = season
|
|
||||||
|
|
||||||
return await bz._fetch_json_async(f"/leagues/{league_id}/standings/", params)
|
|
||||||
|
|
||||||
|
|
||||||
def _season_label_from_dates(start_date, end_date) -> str:
|
|
||||||
"""从赛季起止日期推导赛季标签(与 derive_season_label 语义一致)。"""
|
|
||||||
try:
|
|
||||||
if isinstance(start_date, str):
|
|
||||||
start = datetime.fromisoformat(start_date[:10])
|
|
||||||
else:
|
|
||||||
start = start_date
|
|
||||||
if start is None:
|
|
||||||
return "?"
|
|
||||||
y = start.year
|
|
||||||
return f"{y}-{y + 1}" if start.month >= 8 else f"{y - 1}-{y}"
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return "?"
|
|
||||||
|
|
||||||
|
|
||||||
async def ingest_bzzoiro_standings(db, *, leagues: Iterable[str], season: str | None = None) -> dict:
|
|
||||||
"""采集积分榜 → upsert standings 表。
|
|
||||||
|
|
||||||
season 为 None 时采集当前赛季(bzzoiro 默认返回 is_current 赛季)。
|
|
||||||
球队名与 events 管线使用同一 normalize 规则,保证 Team 匹配。
|
|
||||||
"""
|
|
||||||
from src.data.team_names import normalize as normalize_name
|
|
||||||
|
|
||||||
from src.data import bzzoiro as bz
|
|
||||||
|
|
||||||
result: dict = {"leagues": {}, "total_upserted": 0, "errors": []}
|
|
||||||
for code in leagues:
|
|
||||||
league_r: dict = {"upserted": 0, "teams_created": 0, "rows": 0, "errors": []}
|
|
||||||
try:
|
|
||||||
payload = await bz.fetch_bzzoiro_standings(code, season=season)
|
|
||||||
except Exception as e:
|
|
||||||
logger.exception("bzzoiro standings fetch failed for %s", code)
|
|
||||||
league_r["errors"].append(str(e))
|
|
||||||
await _safe_write_ingest_failure(
|
|
||||||
db,
|
|
||||||
entity_type="standings",
|
|
||||||
source_record_id=None,
|
|
||||||
error=e,
|
|
||||||
raw_payload={"league": code, "season": season},
|
|
||||||
)
|
|
||||||
result["leagues"][code] = league_r
|
|
||||||
result["errors"].append(f"{code}: {e}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
rows = payload.get("standings") or []
|
|
||||||
if not rows:
|
|
||||||
result["leagues"][code] = {"error": "无积分榜数据(赛季未开始或未提供)"}
|
|
||||||
result["errors"].append(f"{code}: 无积分榜数据")
|
|
||||||
continue
|
|
||||||
|
|
||||||
# 联赛(get-or-create,D4: 经 LeagueRepository)
|
|
||||||
league = await LeagueRepository(db).get_or_create(
|
|
||||||
code, LEAGUE_NAMES.get(code, code), LEAGUE_COUNTRIES.get(code)
|
|
||||||
)
|
|
||||||
team_r = TeamRepository(db)
|
|
||||||
|
|
||||||
# 赛季标签:优先用返回的 season 对象推导
|
|
||||||
season_obj = payload.get("season") or {}
|
|
||||||
season_label = _season_label_from_dates(
|
|
||||||
season_obj.get("start_date"), season_obj.get("end_date")
|
|
||||||
)
|
|
||||||
if season_label == "?":
|
|
||||||
season_label = season or ""
|
|
||||||
|
|
||||||
# 批量预载球队(与 events 管线使用同一 normalize 规则,保证 Team 匹配)
|
|
||||||
names = {normalize_name(str(r.get("team_name", ""))) for r in rows}
|
|
||||||
names.discard("")
|
|
||||||
team_map: dict[str, Team] = await team_r.get_all_by_names(list(names))
|
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
for r in rows:
|
|
||||||
team_name = normalize_name(str(r.get("team_name", "")))
|
|
||||||
if not team_name:
|
|
||||||
continue
|
|
||||||
team = team_map.get(team_name)
|
|
||||||
if team is None:
|
|
||||||
team = await team_r.get_or_create(team_name, name_zh=zh_name(team_name))
|
|
||||||
team_map[team_name] = team
|
|
||||||
league_r["teams_created"] += 1
|
|
||||||
|
|
||||||
zone = r.get("zone") or {}
|
|
||||||
values = dict(
|
|
||||||
position=_to_int_or_none(r.get("position")) or 0,
|
|
||||||
played=_to_int_or_none(r.get("played")) or 0,
|
|
||||||
won=_to_int_or_none(r.get("won")) or 0,
|
|
||||||
drawn=_to_int_or_none(r.get("drawn")) or 0,
|
|
||||||
lost=_to_int_or_none(r.get("lost")) or 0,
|
|
||||||
goals_for=_to_int_or_none(r.get("gf")) or 0,
|
|
||||||
goals_against=_to_int_or_none(r.get("ga")) or 0,
|
|
||||||
goal_diff=_to_int_or_none(r.get("gd")) or 0,
|
|
||||||
points=_to_int_or_none(r.get("pts")) or 0,
|
|
||||||
xg_for=_to_float_or_none(r.get("xgf")),
|
|
||||||
xg_against=_to_float_or_none(r.get("xga")),
|
|
||||||
form=r.get("form") or None,
|
|
||||||
zone=zone.get("label") or zone.get("key") or None,
|
|
||||||
updated_at=now,
|
|
||||||
retrieved_at=now,
|
|
||||||
)
|
|
||||||
|
|
||||||
# P0-02: 追加快照——每次采集 INSERT 新行(available_at=now),
|
|
||||||
# ON CONFLICT (league, season, team, available_at) DO NOTHING。
|
|
||||||
standing = Standing(
|
|
||||||
league_id=league.id, season=season_label, team_id=team.id, available_at=now, **values
|
|
||||||
)
|
|
||||||
db.add(standing)
|
|
||||||
league_r["upserted"] += 1
|
|
||||||
|
|
||||||
league_r["rows"] = len(rows)
|
|
||||||
|
|
||||||
# D1(对称 events/stats 管线): 联赛成功 upsert → 补写 Bronze 层。
|
|
||||||
# 幂等键 standings:{league}:{season}:积分榜是联赛级快照,一次成功
|
|
||||||
# 采集写一条 RawEvent(整份原始载荷)+ 一条血缘。season 用实际入库的
|
|
||||||
# 标签(由载荷推导,与 Standing.season 同口径),不依赖调用方传参,
|
|
||||||
# 保证不同调用方(season=None 或显式传参)对同一赛季命中同一条 RawEvent。
|
|
||||||
if league_r["upserted"] > 0:
|
|
||||||
bronze_batch_id = f"bzzoiro-standings-{code}-{now:%Y%m%d%H%M%S}"
|
|
||||||
await _write_standings_bronze(
|
|
||||||
db,
|
|
||||||
source_record_id=f"standings:{code}:{season_label}",
|
|
||||||
raw_payload=payload,
|
|
||||||
league_id=league.id,
|
|
||||||
league_code=code,
|
|
||||||
season_label=season_label,
|
|
||||||
rows_upserted=league_r["upserted"],
|
|
||||||
batch_id=bronze_batch_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
result["leagues"][code] = league_r
|
|
||||||
result["total_upserted"] += league_r["upserted"]
|
|
||||||
logger.info(
|
|
||||||
"bzzoiro standings 采集完成: %s 赛季 %s, upsert %d/%d",
|
|
||||||
code, season_label, league_r["upserted"], league_r["rows"],
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
async def _write_standings_bronze(
|
|
||||||
db,
|
|
||||||
*,
|
|
||||||
source_record_id: str,
|
|
||||||
raw_payload: dict,
|
|
||||||
league_id: int | None,
|
|
||||||
league_code: str,
|
|
||||||
season_label: str,
|
|
||||||
rows_upserted: int,
|
|
||||||
batch_id: str,
|
|
||||||
) -> None:
|
|
||||||
"""standings 成功 upsert 一个联赛后的 Bronze 层补写:RawEvent(幂等) + DataLineage。
|
|
||||||
|
|
||||||
与 _write_events_bronze 同级约束:幂等性由 _write_raw_event 的
|
|
||||||
source_record_id 查重保证(积分榜是联赛级快照,同联赛同赛季重复采集
|
|
||||||
命中同一条 RawEvent);best-effort:基础设施写入失败只记 warning,
|
|
||||||
绝不拖垮采集主流程。
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
await _write_raw_event(db, "bzzoiro", source_record_id, raw_payload, batch_id)
|
|
||||||
await _write_lineage(
|
|
||||||
db, "bzzoiro", source_record_id,
|
|
||||||
"standings", league_id, "standings_ingest",
|
|
||||||
{"league": league_code, "season": season_label, "rows_upserted": rows_upserted},
|
|
||||||
batch_id,
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
logger.warning(
|
|
||||||
"standings Bronze 写入失败(record=%s, league=%s),不影响采集主流程",
|
|
||||||
source_record_id, league_code, exc_info=True,
|
|
||||||
)
|
|
||||||
@@ -1,181 +0,0 @@
|
|||||||
"""bzzoiro stats 回填管线:已完赛比赛详细统计(/events/{id}/stats/)→ match_stats 表。
|
|
||||||
|
|
||||||
从 bzzoiro.py 拆出。上游限速(REQUEST_INTERVAL 秒/请求),大批量回填需分次触发;
|
|
||||||
只 add/flush 不 commit,事务由调用方 UnitOfWork 控制。
|
|
||||||
|
|
||||||
可替换协作者(_fetch_json_async / Bronze 写入助手 / REQUEST_INTERVAL)在运行期
|
|
||||||
经聚合门面 src.data.bzzoiro 解析 —— 与拆分前的单文件 monkeypatch 语义一致。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
from collections.abc import Iterable
|
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
|
|
||||||
from src.data.bzzoiro_common import _to_float_or_none, _to_int_or_none
|
|
||||||
from src.data.config import BZZOIRO_LEAGUE_IDS
|
|
||||||
from src.data.pipeline_write import _safe_write_ingest_failure, _write_lineage, _write_raw_event
|
|
||||||
from src.db.models import MatchStats
|
|
||||||
from src.db.repositories import MatchRepository
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
# bzzoiro stats 字段 → MatchStats 字段映射(stats.home / stats.away 下)
|
|
||||||
_STATS_FIELD_MAP = {
|
|
||||||
"xg": ("home_xg", "away_xg"), # 回退 expected_goals
|
|
||||||
"ball_possession": ("home_possession", None), # 只取主队值,客队=100-home
|
|
||||||
"total_shots": ("home_shots", "away_shots"),
|
|
||||||
"shots_on_target": ("home_shots_on_target", "away_shots_on_target"),
|
|
||||||
"corner_kicks": ("home_corners", "away_corners"),
|
|
||||||
"yellow_cards": ("home_yellow_cards", "away_yellow_cards"),
|
|
||||||
"red_cards": ("home_red_cards", "away_red_cards"),
|
|
||||||
"big_chances": ("home_big_chances", "away_big_chances"),
|
|
||||||
"fouls": ("home_fouls", "away_fouls"),
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _pick(d: dict, *keys):
|
|
||||||
"""按优先级取第一个非空字段值。"""
|
|
||||||
for k in keys:
|
|
||||||
v = d.get(k)
|
|
||||||
if v is not None:
|
|
||||||
return v
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _stats_from_payload(payload: dict) -> dict:
|
|
||||||
"""把 /events/{id}/stats/ 响应映射成 MatchStats 字段 dict。
|
|
||||||
|
|
||||||
响应结构: {"event_id": ..., "stats": {"home": {...}, "away": {...}}}
|
|
||||||
"""
|
|
||||||
stats = (payload or {}).get("stats") or {}
|
|
||||||
home = stats.get("home") or {}
|
|
||||||
away = stats.get("away") or {}
|
|
||||||
out: dict = {}
|
|
||||||
|
|
||||||
xg_h = _pick(home, "xg", "expected_goals")
|
|
||||||
xg_a = _pick(away, "xg", "expected_goals")
|
|
||||||
if xg_h is not None:
|
|
||||||
out["home_xg"] = _to_float_or_none(xg_h)
|
|
||||||
if xg_a is not None:
|
|
||||||
out["away_xg"] = _to_float_or_none(xg_a)
|
|
||||||
|
|
||||||
poss = home.get("ball_possession")
|
|
||||||
if poss is not None:
|
|
||||||
p = _to_float_or_none(poss)
|
|
||||||
if p is not None:
|
|
||||||
out["home_possession"] = p
|
|
||||||
|
|
||||||
for src, (h_fld, a_fld) in _STATS_FIELD_MAP.items():
|
|
||||||
if src in ("xg", "ball_possession"):
|
|
||||||
continue # 已处理
|
|
||||||
hv = home.get(src)
|
|
||||||
av = away.get(src)
|
|
||||||
if hv is not None and h_fld:
|
|
||||||
out[h_fld] = _to_int_or_none(hv)
|
|
||||||
if av is not None and a_fld:
|
|
||||||
out[a_fld] = _to_int_or_none(av)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
async def ingest_bzzoiro_event_stats(
|
|
||||||
db,
|
|
||||||
*,
|
|
||||||
leagues: Iterable[str],
|
|
||||||
limit: int = 100,
|
|
||||||
only_missing: bool = True,
|
|
||||||
) -> dict:
|
|
||||||
"""回填已完赛比赛的详细统计(逐场调 /events/{id}/stats/)。
|
|
||||||
|
|
||||||
筛选条件: match_status=finished 且 source_event_id 非空。
|
|
||||||
only_missing=True 时跳过已有统计的比赛(增量);False 则全量刷新。
|
|
||||||
limit 控制单次最多处理的比赛数(上游限速约 1.2s/请求,大批量需分次触发)。
|
|
||||||
"""
|
|
||||||
from src.data import bzzoiro as bz
|
|
||||||
|
|
||||||
result: dict = {"fetched": 0, "created": 0, "updated": 0, "skipped": 0, "errors": []}
|
|
||||||
|
|
||||||
league_ids = [BZZOIRO_LEAGUE_IDS[c] for c in leagues if c in BZZOIRO_LEAGUE_IDS]
|
|
||||||
if not league_ids:
|
|
||||||
result["errors"].append("无有效联赛代码")
|
|
||||||
return result
|
|
||||||
|
|
||||||
# D4: 候选比赛查询经 MatchRepository(含 stats 预加载,筛选/排序/limit 语义不变)
|
|
||||||
matches = await MatchRepository(db).find_finished_with_stats(
|
|
||||||
league_ids, limit=limit * 3 if only_missing else limit
|
|
||||||
)
|
|
||||||
|
|
||||||
now = datetime.now(timezone.utc)
|
|
||||||
processed = 0
|
|
||||||
for m in matches:
|
|
||||||
if processed >= limit:
|
|
||||||
break
|
|
||||||
if only_missing and m.stats is not None and m.stats.home_shots is not None:
|
|
||||||
result["skipped"] += 1
|
|
||||||
continue
|
|
||||||
processed += 1
|
|
||||||
try:
|
|
||||||
payload = await bz._fetch_json_async(f"/events/{m.source_event_id}/stats/")
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("stats fetch failed match=%s event=%s: %s", m.id, m.source_event_id, e)
|
|
||||||
result["errors"].append(f"match {m.id}: {e}")
|
|
||||||
await _safe_write_ingest_failure(
|
|
||||||
db,
|
|
||||||
entity_type="match_stats",
|
|
||||||
source_record_id=str(m.source_event_id),
|
|
||||||
error=e,
|
|
||||||
raw_payload={"match_id": m.id},
|
|
||||||
)
|
|
||||||
await asyncio.sleep(bz.REQUEST_INTERVAL)
|
|
||||||
continue
|
|
||||||
|
|
||||||
result["fetched"] += 1
|
|
||||||
fields = _stats_from_payload(payload)
|
|
||||||
if not fields:
|
|
||||||
result["skipped"] += 1
|
|
||||||
await asyncio.sleep(bz.REQUEST_INTERVAL)
|
|
||||||
continue
|
|
||||||
|
|
||||||
if m.stats is None:
|
|
||||||
available_at = m.match_date + timedelta(hours=2) if m.match_date else now
|
|
||||||
m.stats = MatchStats(
|
|
||||||
match_id=m.id,
|
|
||||||
source="bzzoiro",
|
|
||||||
source_record_id=str(m.source_event_id),
|
|
||||||
retrieved_at=now,
|
|
||||||
available_at=available_at,
|
|
||||||
)
|
|
||||||
db.add(m.stats)
|
|
||||||
result["created"] += 1
|
|
||||||
else:
|
|
||||||
result["updated"] += 1
|
|
||||||
if m.stats.source is None:
|
|
||||||
m.stats.source = "bzzoiro"
|
|
||||||
m.stats.source_record_id = str(m.source_event_id)
|
|
||||||
if m.stats.retrieved_at is None:
|
|
||||||
m.stats.retrieved_at = now
|
|
||||||
if m.stats.available_at is None and m.match_date:
|
|
||||||
m.stats.available_at = m.match_date + timedelta(hours=2)
|
|
||||||
|
|
||||||
for fld, v in fields.items():
|
|
||||||
if hasattr(m.stats, fld):
|
|
||||||
setattr(m.stats, fld, v)
|
|
||||||
|
|
||||||
# 管线基础设施:写入 RawEvent + DataLineage
|
|
||||||
batch_id = f"bzzoiro-stats-{m.source_event_id}-{now.strftime('%Y%m%d%H%M%S')}"
|
|
||||||
try:
|
|
||||||
await _write_raw_event(db, "bzzoiro", str(m.source_event_id), payload, batch_id)
|
|
||||||
await _write_lineage(db, "bzzoiro", str(m.source_event_id), "match_stats", m.stats.id if m.stats else None, "stats_backfill", {"match_id": m.id}, batch_id)
|
|
||||||
except Exception:
|
|
||||||
pass # 基础设施写入失败不影响主流程
|
|
||||||
|
|
||||||
await asyncio.sleep(bz.REQUEST_INTERVAL)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"bzzoiro stats 回填完成: 抓取 %d, 新建 %d, 更新 %d, 跳过 %d, 错误 %d",
|
|
||||||
result["fetched"], result["created"], result["updated"],
|
|
||||||
result["skipped"], len(result["errors"]),
|
|
||||||
)
|
|
||||||
return result
|
|
||||||
@@ -35,8 +35,6 @@ class NormalizedMatch:
|
|||||||
home_team: str
|
home_team: str
|
||||||
away_team: str
|
away_team: str
|
||||||
match_status: str = "finished"
|
match_status: str = "finished"
|
||||||
# P0-01:比分可信度。known=可靠比分;missing=完赛缺分;unknown=待定。
|
|
||||||
score_status: str = "unknown"
|
|
||||||
home_goals: int | None = None
|
home_goals: int | None = None
|
||||||
away_goals: int | None = None
|
away_goals: int | None = None
|
||||||
season_label: str = ""
|
season_label: str = ""
|
||||||
@@ -219,11 +217,5 @@ def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
|
|||||||
m.away_red_cards = _to_int(raw.get("away_red_cards", raw.get("red_cards_away")))
|
m.away_red_cards = _to_int(raw.get("away_red_cards", raw.get("red_cards_away")))
|
||||||
|
|
||||||
if m.match_status == "finished" and m.home_goals is None:
|
if m.match_status == "finished" and m.home_goals is None:
|
||||||
# P0-01: 完赛缺分不再静默降级为 scheduled(那会丢失「已完赛」事实);
|
m.match_status = "scheduled"
|
||||||
# 保留 status=finished,score_status=missing,goals=NULL(禁止伪造 0:0)。
|
|
||||||
m.score_status = "missing"
|
|
||||||
elif m.home_goals is not None and m.away_goals is not None:
|
|
||||||
m.score_status = "known"
|
|
||||||
else:
|
|
||||||
m.score_status = "unknown"
|
|
||||||
return m
|
return m
|
||||||
|
|||||||
@@ -1,80 +0,0 @@
|
|||||||
"""管线基础设施写入助手:RawEvent(Bronze 原始载荷)/ IngestFailure(死信)/ DataLineage(血缘)。
|
|
||||||
|
|
||||||
从 bzzoiro.py 拆出。约定(与拆分前一致):
|
|
||||||
- 只 add 不 commit —— 事务由调用方 UnitOfWork 控制,分批事务约定不变;
|
|
||||||
- 死信与 Bronze 写入同为 best-effort:失败只记 warning,绝不拖垮采集主流程。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
from src.db.models import DataLineage, IngestFailure, RawEvent
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
async def _write_raw_event(db, source_system: str, source_record_id: str, raw_payload: dict, batch_id: str | None = None) -> None:
|
|
||||||
"""写入 Bronze 层原始事件(幂等:同 source_record_id 跳过)。"""
|
|
||||||
from sqlalchemy import select as _select
|
|
||||||
stmt = _select(RawEvent).where(
|
|
||||||
RawEvent.source_system == source_system,
|
|
||||||
RawEvent.source_record_id == source_record_id,
|
|
||||||
)
|
|
||||||
existing = (await db.execute(stmt)).scalar_one_or_none()
|
|
||||||
if existing is None:
|
|
||||||
db.add(RawEvent(
|
|
||||||
source_system=source_system,
|
|
||||||
source_record_id=source_record_id,
|
|
||||||
raw_payload=raw_payload,
|
|
||||||
ingest_batch_id=batch_id,
|
|
||||||
))
|
|
||||||
|
|
||||||
|
|
||||||
async def _write_ingest_failure(db, source_system: str, entity_type: str, source_record_id: str | None, error_type: str, error_detail: str | None, raw_payload: dict | None = None) -> None:
|
|
||||||
"""写入采集失败死信。"""
|
|
||||||
db.add(IngestFailure(
|
|
||||||
source_system=source_system,
|
|
||||||
entity_type=entity_type,
|
|
||||||
source_record_id=source_record_id,
|
|
||||||
error_type=error_type,
|
|
||||||
error_detail=error_detail,
|
|
||||||
raw_payload=raw_payload,
|
|
||||||
))
|
|
||||||
|
|
||||||
|
|
||||||
async def _safe_write_ingest_failure(
|
|
||||||
db,
|
|
||||||
*,
|
|
||||||
entity_type: str,
|
|
||||||
source_record_id: str | None,
|
|
||||||
error: Exception,
|
|
||||||
raw_payload: dict | None = None,
|
|
||||||
) -> None:
|
|
||||||
"""抓取失败时尽力写入死信表(失败不影响主流程)。
|
|
||||||
|
|
||||||
死信是「可观测性」基础设施,与 RawEvent/Lineage 同级:写入失败只记
|
|
||||||
warning,绝不能让原始抓取错误之外的新异常打断采集循环。
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
await _write_ingest_failure(
|
|
||||||
db, "bzzoiro", entity_type, source_record_id,
|
|
||||||
"fetch_error", str(error), raw_payload,
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
logger.warning(
|
|
||||||
"写入 ingest_failures 死信失败(entity=%s, record=%s): %s",
|
|
||||||
entity_type, source_record_id, error, exc_info=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def _write_lineage(db, source_system: str, source_record_id: str, target_table: str, target_id: int | None, transform_name: str, transform_detail: dict | None = None, batch_id: str | None = None) -> None:
|
|
||||||
"""写入 ETL 血缘追踪。"""
|
|
||||||
db.add(DataLineage(
|
|
||||||
source_system=source_system,
|
|
||||||
source_record_id=source_record_id,
|
|
||||||
target_table=target_table,
|
|
||||||
target_id=target_id,
|
|
||||||
transform_name=transform_name,
|
|
||||||
transform_detail=transform_detail,
|
|
||||||
batch_id=batch_id,
|
|
||||||
))
|
|
||||||
@@ -68,6 +68,25 @@ async def short_read():
|
|||||||
yield session
|
yield session
|
||||||
|
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def short_write():
|
||||||
|
"""短生命周期 write session: 提交后立即释放。
|
||||||
|
|
||||||
|
用法:
|
||||||
|
async with short_write() as session:
|
||||||
|
session.add(pred)
|
||||||
|
await session.commit()
|
||||||
|
# session 已关闭,连接已释放
|
||||||
|
"""
|
||||||
|
async with AsyncSessionLocal() as session:
|
||||||
|
try:
|
||||||
|
yield session
|
||||||
|
await session.commit()
|
||||||
|
except Exception:
|
||||||
|
await session.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
async def init_db() -> None:
|
async def init_db() -> None:
|
||||||
"""验证数据库连接(不建表)。
|
"""验证数据库连接(不建表)。
|
||||||
|
|
||||||
|
|||||||
+47
-78
@@ -1,9 +1,12 @@
|
|||||||
"""ORM 模型: leagues / teams / matches / match_stats / standings / predictions。
|
"""ORM 模型(13 张表)。
|
||||||
|
|
||||||
数据源统一为 bzzoiro(单一数据源),伤停(injuries)与 Understat 已移除。
|
业务表: leagues / teams / matches / match_stats / standings / predictions
|
||||||
|
治理与配置表: raw_events / ingest_failures / data_quality_checks / data_lineage /
|
||||||
|
app_settings / schedules / ingest_jobs(完整说明见 docs/05-data.md)。
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import uuid
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
@@ -19,7 +22,6 @@ from sqlalchemy import (
|
|||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
UniqueConstraint,
|
UniqueConstraint,
|
||||||
column,
|
|
||||||
func,
|
func,
|
||||||
)
|
)
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
@@ -57,22 +59,6 @@ class Team(Base):
|
|||||||
away_matches: Mapped[list["Match"]] = relationship(foreign_keys="Match.away_team_id", back_populates="away_team")
|
away_matches: Mapped[list["Match"]] = relationship(foreign_keys="Match.away_team_id", back_populates="away_team")
|
||||||
|
|
||||||
|
|
||||||
class TeamAlias(Base):
|
|
||||||
"""球队别名:同一球队的不同写法(大小写/译名/缩写)映射到归一后的 teams.id。
|
|
||||||
|
|
||||||
入库流程(get_or_create):normalize(name) → 查 teams.name → 查 team_aliases
|
|
||||||
→ 都没有再 insert 新 Team。别名不自动合并历史重复队,需显式添加。
|
|
||||||
alias_normalized 为 normalize(别名)后的稳定幂等键,用作 PK 避免重复插入。
|
|
||||||
"""
|
|
||||||
__tablename__ = "team_aliases"
|
|
||||||
|
|
||||||
# normalize(别名)后的值,稳定幂等,用作主键
|
|
||||||
alias_normalized: Mapped[str] = mapped_column(String(120), primary_key=True)
|
|
||||||
team_id: Mapped[int] = mapped_column(ForeignKey("teams.id", ondelete="CASCADE"), nullable=False)
|
|
||||||
original_alias: Mapped[str] = mapped_column(String(120), nullable=False) # 原始写法(保留供参考)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
|
||||||
|
|
||||||
|
|
||||||
class Match(Base):
|
class Match(Base):
|
||||||
__tablename__ = "matches"
|
__tablename__ = "matches"
|
||||||
|
|
||||||
@@ -89,9 +75,6 @@ class Match(Base):
|
|||||||
index=True,
|
index=True,
|
||||||
)
|
)
|
||||||
match_status: Mapped[str] = mapped_column(String(20), default="scheduled")
|
match_status: Mapped[str] = mapped_column(String(20), default="scheduled")
|
||||||
# P0-01:比分可信度标记。known=有可靠比分;missing=完赛但缺分(保留 NULL 不伪造 0:0);
|
|
||||||
# unknown=待定(无比分且未确认完赛)。禁止把缺分写成 0:0。
|
|
||||||
score_status: Mapped[str] = mapped_column(String(20), server_default="unknown", nullable=False)
|
|
||||||
home_goals: Mapped[int | None] = mapped_column(Integer)
|
home_goals: Mapped[int | None] = mapped_column(Integer)
|
||||||
away_goals: Mapped[int | None] = mapped_column(Integer)
|
away_goals: Mapped[int | None] = mapped_column(Integer)
|
||||||
home_ht_goals: Mapped[int | None] = mapped_column(Integer)
|
home_ht_goals: Mapped[int | None] = mapped_column(Integer)
|
||||||
@@ -131,19 +114,10 @@ class Match(Base):
|
|||||||
"match_date_date",
|
"match_date_date",
|
||||||
unique=True,
|
unique=True,
|
||||||
),
|
),
|
||||||
# P0-01:比分可信度约束(替代原 ck_matches_finished_has_score):
|
# DB-5: 数据库级约束 — 已完赛比赛必须有比分
|
||||||
# - score_status=known → 必须有比分(非 NULL)
|
|
||||||
# - score_status=missing → 必须 NULL(完赛缺分,禁止伪造 0:0)
|
|
||||||
# - score_status=unknown → 必须 NULL
|
|
||||||
CheckConstraint(
|
CheckConstraint(
|
||||||
"score_status IN ('known', 'missing', 'unknown')",
|
"match_status <> 'finished' OR (home_goals IS NOT NULL AND away_goals IS NOT NULL)",
|
||||||
name="ck_matches_score_status_enum",
|
name="ck_matches_finished_has_score",
|
||||||
),
|
|
||||||
CheckConstraint(
|
|
||||||
"match_status <> 'finished'"
|
|
||||||
" OR (score_status = 'known' AND home_goals IS NOT NULL AND away_goals IS NOT NULL)"
|
|
||||||
" OR (score_status IN ('missing', 'unknown') AND home_goals IS NULL AND away_goals IS NULL)",
|
|
||||||
name="ck_matches_score_integrity",
|
|
||||||
),
|
),
|
||||||
CheckConstraint(
|
CheckConstraint(
|
||||||
"match_status IN ('finished', 'scheduled', 'in_play', 'paused', 'postponed', 'cancelled', 'suspended')",
|
"match_status IN ('finished', 'scheduled', 'in_play', 'paused', 'postponed', 'cancelled', 'suspended')",
|
||||||
@@ -205,11 +179,8 @@ class MatchStats(Base):
|
|||||||
class Standing(Base):
|
class Standing(Base):
|
||||||
"""联赛积分榜快照(bzzoiro /leagues/{id}/standings/)。
|
"""联赛积分榜快照(bzzoiro /leagues/{id}/standings/)。
|
||||||
|
|
||||||
P0-02: 改为追加快照(append-only)。每次采集 INSERT 新行,available_at=now;
|
同一联赛同一赛季只保留最新快照:重新采集时按 (league_id, season, team_id)
|
||||||
查询取 available_at<=cutoff 的每队最新快照(DISTINCT ON team_id ORDER available_at DESC)。
|
upsert。zone 来自 bzzoiro 分区(如 champions_league / europa_league / relegation)。
|
||||||
回测时可还原任意历史时刻的榜单,不再只是"最新快照、忽略 cutoff"。
|
|
||||||
同一 (league_id, season, team_id, available_at) 唯一,ON CONFLICT DO NOTHING。
|
|
||||||
zone 来自 bzzoiro 分区(如 champions_league / europa_league / relegation)。
|
|
||||||
"""
|
"""
|
||||||
__tablename__ = "standings"
|
__tablename__ = "standings"
|
||||||
|
|
||||||
@@ -232,17 +203,13 @@ class Standing(Base):
|
|||||||
zone: Mapped[str | None] = mapped_column(String(50)) # champions_league / relegation 等
|
zone: Mapped[str | None] = mapped_column(String(50)) # champions_league / relegation 等
|
||||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow, onupdate=_utcnow)
|
||||||
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
||||||
# P0-02: 快照可用时间(采集时间),唯一键组成部分 + cutoff 过滤依据
|
|
||||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=_utcnow)
|
|
||||||
|
|
||||||
league: Mapped[League] = relationship()
|
league: Mapped[League] = relationship()
|
||||||
team: Mapped[Team] = relationship(lazy="selectin")
|
team: Mapped[Team] = relationship(lazy="selectin")
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
# P0-02: (league, season, team, available_at) 唯一,支持追加快照 + ON CONFLICT DO NOTHING
|
UniqueConstraint("league_id", "season", "team_id", name="uq_standings_league_season_team"),
|
||||||
UniqueConstraint("league_id", "season", "team_id", "available_at", name="uq_standings_league_season_team_available"),
|
|
||||||
Index("ix_standings_league_season_pos", "league_id", "season", "position"),
|
Index("ix_standings_league_season_pos", "league_id", "season", "position"),
|
||||||
Index("ix_standings_league_season_team_available", "league_id", "season", "team_id", "available_at"),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -290,13 +257,14 @@ class Prediction(Base):
|
|||||||
match: Mapped[Match] = relationship(back_populates="predictions")
|
match: Mapped[Match] = relationship(back_populates="predictions")
|
||||||
|
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
# P0-03: 幂等指纹——input_hash 非空时唯一(同指纹→返回已有行,不 UPDATE/INSERT);
|
# Fix: 唯一约束增加 mode + run_type,允许 live 与 backtest 共存
|
||||||
# 兼容旧数据 NULL input_hash(不强制回填)。
|
# 防止回测覆盖未结算的实盘预测(后续 settle 会污染评估数据)
|
||||||
Index(
|
UniqueConstraint(
|
||||||
"ix_predictions_input_hash_unique", "input_hash", unique=True,
|
"match_id", "provider", "model", "mode", "run_type",
|
||||||
postgresql_where=column("input_hash").isnot(None),
|
name="uq_predictions_match_provider_model_mode_run_type",
|
||||||
),
|
),
|
||||||
Index("ix_predictions_match", "match_id"),
|
Index("ix_predictions_match", "match_id"),
|
||||||
|
Index("ix_predictions_provider_model", "provider", "model"),
|
||||||
# 数据截止时间过滤查询用(按 prediction_cutoff_at 取「赛前已生成」的预测)
|
# 数据截止时间过滤查询用(按 prediction_cutoff_at 取「赛前已生成」的预测)
|
||||||
Index("ix_predictions_cutoff_at", "prediction_cutoff_at"),
|
Index("ix_predictions_cutoff_at", "prediction_cutoff_at"),
|
||||||
# 数据库级约束:最后一道防线
|
# 数据库级约束:最后一道防线
|
||||||
@@ -359,35 +327,6 @@ class RawEvent(Base):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class IngestJob(Base):
|
|
||||||
"""采集任务状态:跟踪每次触后台采集任务的执行进度与结果。
|
|
||||||
|
|
||||||
POST /api/v1/ingest/bzzoiro 触发时写入(pending→running→success/failed),
|
|
||||||
前端 Collection 页据此轮询到终态,替代此前"30 秒后盲标完成"的模拟。
|
|
||||||
分批 get_uow / BzzoiroSource / IngestFailure / Bronze/Lineage 均不受影响
|
|
||||||
(本表仅作状态追踪,不介入采集事务)。
|
|
||||||
"""
|
|
||||||
__tablename__ = "ingest_jobs"
|
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(36), primary_key=True) # uuid4
|
|
||||||
task: Mapped[str] = mapped_column(String(20), nullable=False)
|
|
||||||
params: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict)
|
|
||||||
status: Mapped[str] = mapped_column(String(20), nullable=False, server_default="pending")
|
|
||||||
result: Mapped[dict | None] = mapped_column(JSONB)
|
|
||||||
error: Mapped[str | None] = mapped_column(Text)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
||||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
||||||
|
|
||||||
__table_args__ = (
|
|
||||||
Index("ix_ingest_job_status_created", "status", "created_at"),
|
|
||||||
CheckConstraint(
|
|
||||||
"status IN ('pending', 'running', 'success', 'failed')",
|
|
||||||
name="ck_ingest_job_status",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IngestFailure(Base):
|
class IngestFailure(Base):
|
||||||
"""采集失败死信:记录失败原因、重试次数与下次重试时间。
|
"""采集失败死信:记录失败原因、重试次数与下次重试时间。
|
||||||
|
|
||||||
@@ -468,3 +407,33 @@ class DataLineage(Base):
|
|||||||
Index("ix_lineage_target", "target_table", "target_id"),
|
Index("ix_lineage_target", "target_table", "target_id"),
|
||||||
Index("ix_lineage_batch", "batch_id"),
|
Index("ix_lineage_batch", "batch_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class IngestJob(Base):
|
||||||
|
"""采集任务状态跟踪:解决 POST /ingest/bzzoiro fire-and-forget 不可观测问题。
|
||||||
|
|
||||||
|
生命周期: pending(路由创建) → running(后台开始) → success / failed(终态)。
|
||||||
|
result 存各任务统计摘要(如 inserted/updated/errors 截断);error 存失败原因。
|
||||||
|
与 IngestFailure 死信相互独立:死信记录单条管线抓取失败(行级),
|
||||||
|
job 记录整次任务执行结果(任务级),两者可同时存在。
|
||||||
|
job 状态更新是「尽力而为」:更新失败只记日志,不影响采集主流程。
|
||||||
|
"""
|
||||||
|
__tablename__ = "ingest_jobs"
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||||
|
task: Mapped[str] = mapped_column(String(20), nullable=False) # events / standings / stats / all
|
||||||
|
params: Mapped[dict] = mapped_column(JSONB, nullable=False, default=dict) # leagues/日期等请求参数
|
||||||
|
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending")
|
||||||
|
result: Mapped[dict | None] = mapped_column(JSONB) # 成功时的统计摘要
|
||||||
|
error: Mapped[str | None] = mapped_column(Text) # 失败原因(repr)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_utcnow)
|
||||||
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('pending','running','success','failed')",
|
||||||
|
name="ck_ingest_jobs_status",
|
||||||
|
),
|
||||||
|
Index("ix_ingest_jobs_created", "created_at"),
|
||||||
|
)
|
||||||
|
|||||||
+18
-69
@@ -5,10 +5,6 @@ Repository 只负责查询,不负责事务提交。
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
@@ -80,20 +76,6 @@ class MatchRepository:
|
|||||||
)
|
)
|
||||||
return (await self._session.execute(stmt)).scalars().all()
|
return (await self._session.execute(stmt)).scalars().all()
|
||||||
|
|
||||||
async def find_by_source_event_id(self, source_event_id: int) -> Match | None:
|
|
||||||
"""按上游 event id 查找比赛(唯一命中,用于 upsert 优先路径)。
|
|
||||||
|
|
||||||
source_event_id 上有 partial unique 索引(WHERE IS NOT NULL),
|
|
||||||
同联赛同主客同天(自然键)与上游 event_id 共同保障同一场比赛
|
|
||||||
重复采集时 upsert 而非插入重复行。
|
|
||||||
"""
|
|
||||||
stmt = (
|
|
||||||
select(Match)
|
|
||||||
.options(selectinload(Match.stats))
|
|
||||||
.where(Match.source_event_id == source_event_id)
|
|
||||||
)
|
|
||||||
return (await self._session.execute(stmt)).scalar_one_or_none()
|
|
||||||
|
|
||||||
async def find_finished_with_stats(self, league_ids: list[int], *, limit: int) -> list[Match]:
|
async def find_finished_with_stats(self, league_ids: list[int], *, limit: int) -> list[Match]:
|
||||||
"""已完赛且有上游 event id 的比赛(按日期倒序),供统计回填逐场拉取。
|
"""已完赛且有上游 event id 的比赛(按日期倒序),供统计回填逐场拉取。
|
||||||
|
|
||||||
@@ -126,61 +108,14 @@ class TeamRepository:
|
|||||||
return (await self._session.execute(stmt)).scalar_one_or_none()
|
return (await self._session.execute(stmt)).scalar_one_or_none()
|
||||||
|
|
||||||
async def get_or_create(self, name: str, *, name_zh: str | None = None) -> Team:
|
async def get_or_create(self, name: str, *, name_zh: str | None = None) -> Team:
|
||||||
"""按名获取球队,不存在则创建(name_zh 供 bzzoiro 管线写中文名)。
|
"""按名获取球队,不存在则创建(name_zh 供 bzzoiro 管线写中文名)。"""
|
||||||
|
team = await self.get_by_name(name)
|
||||||
归一化咽喉 + 别名查找,三步定位:
|
if team is None:
|
||||||
1) normalize(name) → 查 teams.name
|
team = Team(name=name, name_zh=name_zh)
|
||||||
2) 查 team_aliases(以 normalize(name) 为幂等键)→ 复用已映射的 teams.id
|
|
||||||
3) 都没有 → insert 新 Team(归一名)
|
|
||||||
创建新 Team 时 info 打出原始名与归一后的规范名,便于排查重名。
|
|
||||||
不自动合并历史重复队;需显式添加别名。
|
|
||||||
"""
|
|
||||||
from src.data.team_names import normalize as normalize_name
|
|
||||||
from src.db.models import TeamAlias
|
|
||||||
|
|
||||||
normalized = normalize_name(name) or name.strip()
|
|
||||||
|
|
||||||
# 1) 归一名直查 teams
|
|
||||||
team = await self.get_by_name(normalized)
|
|
||||||
if team is not None:
|
|
||||||
return team
|
|
||||||
|
|
||||||
# 2) 别名查找:normalize(别名) 作为幂等键,命中即复用已有 Team
|
|
||||||
alias = await self._session.get(TeamAlias, normalized)
|
|
||||||
if alias is not None:
|
|
||||||
team = await self._session.get(Team, alias.team_id)
|
|
||||||
if team is not None:
|
|
||||||
logger.info("Team 别名命中: %s -> %s(已有 id=%s)", name, normalized, team.id)
|
|
||||||
return team
|
|
||||||
|
|
||||||
# 3) 新建 Team(归一名)
|
|
||||||
logger.info("创建新 Team: %s -> %s", name, normalized)
|
|
||||||
team = Team(name=normalized, name_zh=name_zh)
|
|
||||||
self._session.add(team)
|
self._session.add(team)
|
||||||
await self._session.flush()
|
await self._session.flush()
|
||||||
return team
|
return team
|
||||||
|
|
||||||
async def add_alias(self, alias: str, team_id: int) -> TeamAlias:
|
|
||||||
"""为已有 Team 添加别名。
|
|
||||||
|
|
||||||
幂等:以 normalize(alias) 为 PK,重复添加同一别名会 upsert。
|
|
||||||
不自动合并历史重复队,仅建立别名映射。
|
|
||||||
"""
|
|
||||||
from src.data.team_names import normalize as normalize_name
|
|
||||||
from src.db.models import TeamAlias
|
|
||||||
|
|
||||||
normalized = normalize_name(alias) or alias.strip()
|
|
||||||
existing = await self._session.get(TeamAlias, normalized)
|
|
||||||
if existing is not None:
|
|
||||||
existing.team_id = team_id # 允许重新指向
|
|
||||||
existing.original_alias = alias
|
|
||||||
await self._session.flush()
|
|
||||||
return existing
|
|
||||||
row = TeamAlias(alias_normalized=normalized, team_id=team_id, original_alias=alias)
|
|
||||||
self._session.add(row)
|
|
||||||
await self._session.flush()
|
|
||||||
return row
|
|
||||||
|
|
||||||
async def get_all_by_names(self, names: list[str]) -> dict[str, Team]:
|
async def get_all_by_names(self, names: list[str]) -> dict[str, Team]:
|
||||||
"""批量获取球队,返回 name → Team 映射。"""
|
"""批量获取球队,返回 name → Team 映射。"""
|
||||||
if not names:
|
if not names:
|
||||||
@@ -215,3 +150,17 @@ class LeagueRepository:
|
|||||||
async def add(self, league: League) -> None:
|
async def add(self, league: League) -> None:
|
||||||
self._session.add(league)
|
self._session.add(league)
|
||||||
await self._session.flush()
|
await self._session.flush()
|
||||||
|
|
||||||
|
|
||||||
|
class PredictionRepository:
|
||||||
|
"""预测记录数据访问。"""
|
||||||
|
|
||||||
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
|
self._session = session
|
||||||
|
|
||||||
|
async def get_by_id(self, prediction_id: int) -> Prediction | None:
|
||||||
|
return await self._session.get(Prediction, prediction_id)
|
||||||
|
|
||||||
|
async def add(self, prediction: Prediction) -> None:
|
||||||
|
self._session.add(prediction)
|
||||||
|
await self._session.flush()
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from src.core.config import settings
|
|||||||
from src.db.base import AsyncSessionLocal
|
from src.db.base import AsyncSessionLocal
|
||||||
from src.db.models import Match, Prediction
|
from src.db.models import Match, Prediction
|
||||||
from src.db.unit_of_work import get_uow
|
from src.db.unit_of_work import get_uow
|
||||||
from src.llm.predict import PredictResult, _insert_or_find_by_fingerprint
|
from src.llm.predict import PredictResult, _upsert_prediction
|
||||||
from src.llm.agents.base import AgentReport, AgentSpec, load_agent_prompt
|
from src.llm.agents.base import AgentReport, AgentSpec, load_agent_prompt
|
||||||
from src.llm.context_builder import (
|
from src.llm.context_builder import (
|
||||||
MatchHeader,
|
MatchHeader,
|
||||||
@@ -285,13 +285,10 @@ async def predict_match_multi(
|
|||||||
|
|
||||||
latency_ms = int((time.perf_counter() - start) * 1000)
|
latency_ms = int((time.perf_counter() - start) * 1000)
|
||||||
|
|
||||||
# P0-03: 指纹输入——终裁报告 hash 作 context_hash,专家列表作 agent_ids
|
# 3.5 计算输入 hash(基于终裁报告)
|
||||||
reports_json = _reports_to_json(reports)
|
input_hash = hashlib.sha256(
|
||||||
context_hash = hashlib.sha256(reports_json.encode("utf-8")).hexdigest()
|
_reports_to_json(reports).encode("utf-8")
|
||||||
agent_ids = sorted([r.agent for r in reports]) if reports else []
|
).hexdigest()
|
||||||
# 终裁模板 hash(规范:复用 prompt 版本 + 终裁 system prompt)
|
|
||||||
prompt_hash = hashlib.sha256(f"multi_{version}".encode("utf-8")).hexdigest()
|
|
||||||
system_prompt_hash = hashlib.sha256(AGGREGATOR_SYSTEM.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
# 4. 存库(使用 UnitOfWork)
|
# 4. 存库(使用 UnitOfWork)
|
||||||
async with get_uow() as session:
|
async with get_uow() as session:
|
||||||
@@ -318,20 +315,15 @@ async def predict_match_multi(
|
|||||||
pred_status = "degraded"
|
pred_status = "degraded"
|
||||||
model_name = aggregator_model
|
model_name = aggregator_model
|
||||||
|
|
||||||
pred = await _insert_or_find_by_fingerprint(
|
pred = await _upsert_prediction(
|
||||||
session,
|
session,
|
||||||
|
match_id=match_id,
|
||||||
|
provider_name=settings.LLM_PROVIDER,
|
||||||
|
model=model_name,
|
||||||
|
mode="multi",
|
||||||
|
run_type="backtest" if backtest else "live",
|
||||||
values={
|
values={
|
||||||
"match_id": match_id,
|
|
||||||
"provider": settings.LLM_PROVIDER,
|
|
||||||
"model": model_name,
|
|
||||||
"mode": "multi",
|
|
||||||
"run_type": "backtest" if backtest else "live",
|
|
||||||
"prompt_version": f"multi_{version}",
|
"prompt_version": f"multi_{version}",
|
||||||
"prompt_hash": prompt_hash,
|
|
||||||
"system_prompt_hash": system_prompt_hash,
|
|
||||||
"temperature": 0.2,
|
|
||||||
"context_hash": context_hash,
|
|
||||||
"agent_ids": agent_ids,
|
|
||||||
"prompt_tokens": sum(r.prompt_tokens or 0 for r in reports) + agg_prompt_tokens,
|
"prompt_tokens": sum(r.prompt_tokens or 0 for r in reports) + agg_prompt_tokens,
|
||||||
"completion_tokens": sum(r.completion_tokens or 0 for r in reports) + agg_completion_tokens,
|
"completion_tokens": sum(r.completion_tokens or 0 for r in reports) + agg_completion_tokens,
|
||||||
"latency_ms": latency_ms,
|
"latency_ms": latency_ms,
|
||||||
@@ -349,6 +341,7 @@ async def predict_match_multi(
|
|||||||
"match_kickoff_at": match_kickoff_at,
|
"match_kickoff_at": match_kickoff_at,
|
||||||
"prediction_cutoff_at": prediction_cutoff_at,
|
"prediction_cutoff_at": prediction_cutoff_at,
|
||||||
"prediction_created_at": now,
|
"prediction_created_at": now,
|
||||||
|
"input_hash": input_hash,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -130,7 +130,6 @@ async def _get_historical_matches(
|
|||||||
selectinload(Match.away_team),
|
selectinload(Match.away_team),
|
||||||
)
|
)
|
||||||
.where(Match.match_status == "finished")
|
.where(Match.match_status == "finished")
|
||||||
.where(Match.score_status == "known")
|
|
||||||
.where(Match.home_goals.is_not(None))
|
.where(Match.home_goals.is_not(None))
|
||||||
.where(Match.away_goals.is_not(None))
|
.where(Match.away_goals.is_not(None))
|
||||||
)
|
)
|
||||||
|
|||||||
+12
-51
@@ -5,15 +5,14 @@
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import case, func, select
|
from sqlalchemy import case, func, select
|
||||||
|
|
||||||
from src.db.base import AsyncSession, AsyncSessionLocal
|
from src.db.base import AsyncSession, AsyncSessionLocal
|
||||||
from src.db.models import Match
|
from src.db.models import Match
|
||||||
from src.llm.predict import PredictResult, _insert_or_find_by_fingerprint
|
from src.llm.predict import PredictResult
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -59,23 +58,19 @@ async def predict_baseline(
|
|||||||
|
|
||||||
返回 PredictResult(D2 统一结果类型):
|
返回 PredictResult(D2 统一结果类型):
|
||||||
provider=model="baseline", 不调用 LLM,latency_ms≈0。
|
provider=model="baseline", 不调用 LLM,latency_ms≈0。
|
||||||
|
prediction_id 为占位 0 —— baseline 不在服务层落库,
|
||||||
P3-2:baseline 落库下沉到服务层 —— 直接在服务层完成落库并回填真实
|
由路由层 _persist_baseline 落库后取得真实 id。
|
||||||
prediction_id,路由层不再需要特殊的 _persist_baseline,与 single/multi
|
|
||||||
路径统一(result.prediction_id 即可用)。对外 JSON 不变。
|
|
||||||
"""
|
"""
|
||||||
from src.db.unit_of_work import get_uow
|
|
||||||
|
|
||||||
async with AsyncSessionLocal() as db:
|
async with AsyncSessionLocal() as db:
|
||||||
match = await db.get(Match, match_id)
|
match = await db.get(Match, match_id)
|
||||||
if match is None:
|
if match is None:
|
||||||
raise ValueError(f"match {match_id} not found")
|
raise ValueError(f"match {match_id} not found")
|
||||||
|
|
||||||
before = None
|
before = None
|
||||||
if backtest and match.match_date:
|
if backtest and match.match_dt:
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
|
||||||
before = match.match_date - timedelta(days=1)
|
before = match.match_dt - timedelta(days=1)
|
||||||
elif cutoff_at is not None:
|
elif cutoff_at is not None:
|
||||||
before = cutoff_at
|
before = cutoff_at
|
||||||
|
|
||||||
@@ -98,45 +93,8 @@ async def predict_baseline(
|
|||||||
else:
|
else:
|
||||||
pred_1x2 = "X"
|
pred_1x2 = "X"
|
||||||
|
|
||||||
# P0-03: 基线指纹——基于主客场场均进球数据(context_hash) + 截止时间
|
|
||||||
context_hash = hashlib.sha256(
|
|
||||||
f"{home_avg:.4f}:{away_avg:.4f}:{before.isoformat() if before else 'none'}".encode("utf-8")
|
|
||||||
).hexdigest()
|
|
||||||
|
|
||||||
values = {
|
|
||||||
"match_id": match_id,
|
|
||||||
"provider": "baseline",
|
|
||||||
"model": "baseline",
|
|
||||||
"mode": "baseline",
|
|
||||||
"run_type": "live",
|
|
||||||
"prompt_version": "baseline_v1",
|
|
||||||
"prompt_hash": hashlib.sha256(b"baseline_v1").hexdigest(),
|
|
||||||
"system_prompt_hash": hashlib.sha256(b"baseline").hexdigest(),
|
|
||||||
"temperature": 0.0,
|
|
||||||
"context_hash": context_hash,
|
|
||||||
"agent_ids": [],
|
|
||||||
"prompt_tokens": 0,
|
|
||||||
"completion_tokens": 0,
|
|
||||||
"latency_ms": 0,
|
|
||||||
"pred_home_goals": float(pred_home),
|
|
||||||
"pred_away_goals": float(pred_away),
|
|
||||||
"pred_1x2": pred_1x2,
|
|
||||||
"subjective_confidence": 0.5,
|
|
||||||
"reasoning": (
|
|
||||||
f"基线估计(非投注建议): 主队主场场均进球 {home_avg:.2f} → 预测 {pred_home}; "
|
|
||||||
f"客队客场场均进球 {away_avg:.2f} → 预测 {pred_away}。"
|
|
||||||
),
|
|
||||||
"raw_response": {"home_avg": round(home_avg, 2), "away_avg": round(away_avg, 2)},
|
|
||||||
"status": "success",
|
|
||||||
}
|
|
||||||
|
|
||||||
# P0-03:服务层幂等插入,回填真实 prediction_id(与 single/multi 统一)。
|
|
||||||
async with get_uow() as session:
|
|
||||||
pred = await _insert_or_find_by_fingerprint(session, values=values)
|
|
||||||
prediction_id = pred.id
|
|
||||||
|
|
||||||
return PredictResult(
|
return PredictResult(
|
||||||
prediction_id=prediction_id,
|
prediction_id=0, # 占位:真实 id 由路由层 _persist_baseline 落库后返回
|
||||||
provider="baseline",
|
provider="baseline",
|
||||||
model="baseline",
|
model="baseline",
|
||||||
prompt_version="baseline_v1",
|
prompt_version="baseline_v1",
|
||||||
@@ -147,11 +105,14 @@ async def predict_baseline(
|
|||||||
alt_pred_away_goals=None,
|
alt_pred_away_goals=None,
|
||||||
pred_1x2=pred_1x2,
|
pred_1x2=pred_1x2,
|
||||||
subjective_confidence=0.5,
|
subjective_confidence=0.5,
|
||||||
reasoning=values["reasoning"],
|
reasoning=(
|
||||||
|
f"基线估计(非投注建议): 主队主场场均进球 {home_avg:.2f} → 预测 {pred_home}; "
|
||||||
|
f"客队客场场均进球 {away_avg:.2f} → 预测 {pred_away}。"
|
||||||
|
),
|
||||||
context="", # baseline 不构建 LLM 上下文
|
context="", # baseline 不构建 LLM 上下文
|
||||||
status="success",
|
status="success",
|
||||||
latency_ms=0,
|
latency_ms=0,
|
||||||
prompt_tokens=0,
|
prompt_tokens=0,
|
||||||
completion_tokens=0,
|
completion_tokens=0,
|
||||||
raw=values["raw_response"],
|
raw={"home_avg": round(home_avg, 2), "away_avg": round(away_avg, 2)},
|
||||||
)
|
)
|
||||||
|
|||||||
+543
-28
@@ -1,9 +1,9 @@
|
|||||||
"""上下文构建器:数据切片 + 拼接(聚合门面)。
|
"""上下文构建器:数据切片 + 拼接。
|
||||||
|
|
||||||
实现按 slice 拆分(单文件 → slices 包),本模块只做再导出:
|
架构:
|
||||||
- 切片函数: 每个领域 agent 一个数据切片 → src/llm/slices/{form,h2h,stats,home_away,standings}.py
|
- match_header: 比赛基础信息(对阵双方/联赛/时间)
|
||||||
- 共享类型/头信息/查询助手 → src/llm/slices/common.py
|
- 切片函数: 每个领域 agent 一个数据切片(h2h / form / standings / stats)
|
||||||
- build_context: 单 agent 路径,拼接全部切片(行为与旧版一致) → src/llm/slices/aggregate.py
|
- build_context: 单 agent 路径,拼接全部切片(行为与旧版一致)
|
||||||
|
|
||||||
multi-agent 路径由 agents/orchestrator.py 调用切片函数,每个专家只拿自己的切片。
|
multi-agent 路径由 agents/orchestrator.py 调用切片函数,每个专家只拿自己的切片。
|
||||||
|
|
||||||
@@ -11,33 +11,548 @@ multi-agent 路径由 agents/orchestrator.py 调用切片函数,每个专家只
|
|||||||
build_context 创建一个共享 session 并传给所有切片函数,
|
build_context 创建一个共享 session 并传给所有切片函数,
|
||||||
避免每个切片独立创建 session —— 回测 20 场并发时,
|
避免每个切片独立创建 session —— 回测 20 场并发时,
|
||||||
5 个切片 × 20 场 = 100 个连接会耗尽连接池(pool_size=15)。
|
5 个切片 × 20 场 = 100 个连接会耗尽连接池(pool_size=15)。
|
||||||
|
|
||||||
消费方(路由/orchestrator/tests)仍从本模块 import,签名与拆分前完全一致。
|
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
# ── 共享类型与基础(判空/赛果/统计可用性) ──
|
import logging
|
||||||
from src.llm.slices.common import ( # noqa: F401
|
from dataclasses import dataclass
|
||||||
MatchContext,
|
from typing import TYPE_CHECKING
|
||||||
MatchHeader,
|
|
||||||
SliceResult,
|
from sqlalchemy import select
|
||||||
_is_stats_available,
|
from sqlalchemy.orm import selectinload
|
||||||
_outcome,
|
|
||||||
header_text,
|
from src.db.base import AsyncSessionLocal
|
||||||
load_match_header,
|
from src.db.models import Match
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _outcome(home_goals: int, away_goals: int, side: str) -> str:
|
||||||
|
"""从某队视角看赛果: W/D/L。"""
|
||||||
|
if home_goals is None or away_goals is None:
|
||||||
|
return "?"
|
||||||
|
if side == "home":
|
||||||
|
return "W" if home_goals > away_goals else ("D" if home_goals == away_goals else "L")
|
||||||
|
return "W" if away_goals > home_goals else ("D" if away_goals == home_goals else "L")
|
||||||
|
|
||||||
|
|
||||||
|
def _is_stats_available(stats, before) -> bool:
|
||||||
|
"""检查统计数据在 cutoff 时间是否已可用。
|
||||||
|
|
||||||
|
available_at 语义:该条统计「对外可被使用」的最早时间,
|
||||||
|
至少不得早于比赛结束。用于回测防泄漏。
|
||||||
|
|
||||||
|
规则:
|
||||||
|
- before is None(实盘):available_at 为 None 时允许(兼容旧数据)
|
||||||
|
- before is not None(回测):available_at 为 None 视为不可用(保守)
|
||||||
|
- available_at > cutoff:不可用(数据在 cutoff 之后才生成)
|
||||||
|
"""
|
||||||
|
if before is None:
|
||||||
|
# 实盘模式:无时间信息时允许(兼容旧数据)
|
||||||
|
return True
|
||||||
|
# 回测模式(cutoff 不为 None):
|
||||||
|
# available_at 为 None → 无法确认是否在 cutoff 前可用,保守视为不可用
|
||||||
|
if stats.available_at is None:
|
||||||
|
return False
|
||||||
|
return stats.available_at <= before
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class SliceResult:
|
||||||
|
"""数据切片的显式结果(替代「靠文案子串猜有无数据」)。
|
||||||
|
|
||||||
|
旧实现用 `"无数据" in slice_text` 判断,依赖具体文案 —— 一旦某个切片
|
||||||
|
写成「无比分数据」「无伤停数据」这类变体,判断就会静默失配
|
||||||
|
(见审查报告 P2-1)。这里让切片函数直接声明 `has_data`,不再猜。
|
||||||
|
"""
|
||||||
|
text: str
|
||||||
|
has_data: bool
|
||||||
|
n_records: int = 0
|
||||||
|
|
||||||
|
def __str__(self) -> str: # 让老调用点可直接当 str 用
|
||||||
|
return self.text
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MatchContext:
|
||||||
|
match_id: int
|
||||||
|
text: str
|
||||||
|
has_stats: bool
|
||||||
|
has_standings: bool
|
||||||
|
match_dt: object | None = None # 比赛时间(回测防泄漏 + 快照用)
|
||||||
|
cutoff: object | None = None # 实际使用的数据截止时间(用于落库记录)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MatchHeader:
|
||||||
|
"""比赛基础信息(所有 agent 共享)。"""
|
||||||
|
match_id: int
|
||||||
|
home_name: str
|
||||||
|
away_name: str
|
||||||
|
league_name: str
|
||||||
|
season: str | None
|
||||||
|
match_date: str
|
||||||
|
match_dt: object # 原始 datetime,回测防泄漏用
|
||||||
|
stage: str | None
|
||||||
|
home_team_id: int
|
||||||
|
away_team_id: int
|
||||||
|
league_id: int
|
||||||
|
|
||||||
|
|
||||||
|
async def load_match_header(match_id: int, db: AsyncSession | None = None) -> MatchHeader:
|
||||||
|
"""加载比赛头信息(各 agent 共用)。
|
||||||
|
|
||||||
|
Args:
|
||||||
|
match_id: 比赛 ID
|
||||||
|
db: 可选的共享 session。不传则自建(向后兼容)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
|
m = await _load_match(db, match_id)
|
||||||
|
return _to_header(m)
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
m = await _load_match(new_db, match_id)
|
||||||
|
return _to_header(m)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_header(m: Match) -> MatchHeader:
|
||||||
|
return MatchHeader(
|
||||||
|
match_id=m.id,
|
||||||
|
home_name=m.home_team.name_zh or m.home_team.name,
|
||||||
|
away_name=m.away_team.name_zh or m.away_team.name,
|
||||||
|
league_name=m.league.name if m.league else "?",
|
||||||
|
season=m.season,
|
||||||
|
match_date=m.match_date.strftime("%Y-%m-%d %H:%M UTC") if m.match_date else "?",
|
||||||
|
match_dt=m.match_date,
|
||||||
|
stage=m.match_stage,
|
||||||
|
home_team_id=m.home_team_id,
|
||||||
|
away_team_id=m.away_team_id,
|
||||||
|
league_id=m.league_id,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ── 领域切片 ──
|
|
||||||
from src.llm.slices.form import form_slice # noqa: F401
|
|
||||||
from src.llm.slices.h2h import h2h_slice # noqa: F401
|
|
||||||
from src.llm.slices.home_away import home_away_slice # noqa: F401
|
|
||||||
from src.llm.slices.standings import standings_slice # noqa: F401
|
|
||||||
from src.llm.slices.stats import stats_slice # noqa: F401
|
|
||||||
|
|
||||||
# ── 底层查询助手(切片函数共用;orchestrator/tests 直接引用) ──
|
def header_text(h: MatchHeader) -> str:
|
||||||
from src.llm.slices.form import _get_form # noqa: F401
|
stage = f" {h.stage}" if h.stage else ""
|
||||||
from src.llm.slices.h2h import _get_h2h # noqa: F401
|
return (
|
||||||
from src.llm.slices.home_away import _get_home_away # noqa: F401
|
f"对阵: {h.home_name} vs {h.away_name} | {h.league_name} {h.season or '?'}{stage} | {h.match_date}"
|
||||||
|
)
|
||||||
|
|
||||||
# ── 单 agent 聚合入口 ──
|
|
||||||
from src.llm.slices.aggregate import build_context # noqa: F401
|
# ============================================================
|
||||||
|
# 切片函数: 每个领域 agent 一个
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
async def h2h_slice(header: MatchHeader, *, limit: int = 8, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
|
"""E - 历史交锋切片: 过去数年 + 近期交手数据,提取交手规律。before=match_date 用于回测。
|
||||||
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
|
h2h = await _get_h2h(db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
h2h = await _get_h2h(new_db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
||||||
|
lines = [f"── 历史交锋(近 {limit} 次) ──"]
|
||||||
|
n_with_score = 0
|
||||||
|
if h2h:
|
||||||
|
# 从当前主队视角统计:判断当前主队在每场交锋中是主是客
|
||||||
|
current_home_wins = current_home_draws = current_home_losses = 0
|
||||||
|
for hm in h2h:
|
||||||
|
d = hm.match_date.strftime("%Y-%m") if hm.match_date else "?"
|
||||||
|
if hm.home_goals is not None:
|
||||||
|
n_with_score += 1
|
||||||
|
# 判断当前主队当时是主队还是客队
|
||||||
|
if hm.home_team_id == header.home_team_id:
|
||||||
|
# 当前主队当时是主队
|
||||||
|
if hm.home_goals > hm.away_goals:
|
||||||
|
current_home_wins += 1
|
||||||
|
elif hm.home_goals == hm.away_goals:
|
||||||
|
current_home_draws += 1
|
||||||
|
else:
|
||||||
|
current_home_losses += 1
|
||||||
|
else:
|
||||||
|
# 当前主队当时是客队(从客队视角看赛果)
|
||||||
|
if hm.away_goals > hm.home_goals:
|
||||||
|
current_home_wins += 1
|
||||||
|
elif hm.away_goals == hm.home_goals:
|
||||||
|
current_home_draws += 1
|
||||||
|
else:
|
||||||
|
current_home_losses += 1
|
||||||
|
lines.append(f" {d}: {hm.home_team.name} {hm.home_goals}-{hm.away_goals} {hm.away_team.name}")
|
||||||
|
else:
|
||||||
|
lines.append(f" {d}: {hm.home_team.name} vs {hm.away_team.name} (无比分)")
|
||||||
|
total = current_home_wins + current_home_draws + current_home_losses
|
||||||
|
if total:
|
||||||
|
lines.append(
|
||||||
|
f" 总计 {total} 场(从当前主队 {header.home_name} 视角): "
|
||||||
|
f"{current_home_wins}胜 {current_home_draws}平 {current_home_losses}负"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
lines.append(" 无数据")
|
||||||
|
# has_data 以「有比分的交锋」为准:仅有对阵无比分时不足以支撑分析
|
||||||
|
return SliceResult(text="\n".join(lines), has_data=n_with_score > 0, n_records=n_with_score)
|
||||||
|
|
||||||
|
|
||||||
|
async def form_slice(header: MatchHeader, *, limit: int = 5, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
|
"""A - 近期状态切片: 两队近 N 场赛果、关键事件、走势判断。before=match_date 用于回测。
|
||||||
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
|
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
||||||
|
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_form = await _get_form(new_db, header.home_team_id, before=before, limit=limit)
|
||||||
|
away_form = await _get_form(new_db, header.away_team_id, before=before, limit=limit)
|
||||||
|
lines = []
|
||||||
|
n_scored = 0
|
||||||
|
# P0-1 修复:每场历史比赛必须根据「该队当时是主是客」判断 side,
|
||||||
|
# 不能用本场 side 硬套 —— 否则客场输球会被算成主场赢球。
|
||||||
|
for label, name, form, team_id in (
|
||||||
|
("主队", header.home_name, home_form, header.home_team_id),
|
||||||
|
("客队", header.away_name, away_form, header.away_team_id),
|
||||||
|
):
|
||||||
|
lines.append(f"── {label}近况({name},近 {limit} 场) ──")
|
||||||
|
if form:
|
||||||
|
wins = draws = losses = 0
|
||||||
|
for fm in form:
|
||||||
|
is_home = (fm.home_team_id == team_id)
|
||||||
|
side = "home" if is_home else "away"
|
||||||
|
o = _outcome(fm.home_goals, fm.away_goals, side)
|
||||||
|
if o == "W": wins += 1
|
||||||
|
elif o == "D": draws += 1
|
||||||
|
else: losses += 1
|
||||||
|
if fm.home_goals is not None:
|
||||||
|
n_scored += 1
|
||||||
|
score = f"{fm.home_goals}-{fm.away_goals}" if fm.home_goals is not None else "vs"
|
||||||
|
xg = ""
|
||||||
|
if fm.stats and _is_stats_available(fm.stats, before) and fm.stats.home_xg is not None:
|
||||||
|
own = fm.stats.home_xg if is_home else fm.stats.away_xg
|
||||||
|
xg = f" (xG {own:.1f})"
|
||||||
|
opp = fm.away_team.name if is_home else fm.home_team.name
|
||||||
|
lines.append(f" {o} {score} vs {opp}{xg}")
|
||||||
|
lines.append(f" 近 {len(form)} 场: {wins}胜 {draws}平 {losses}负")
|
||||||
|
else:
|
||||||
|
lines.append(" 无数据")
|
||||||
|
return SliceResult(text="\n".join(lines), has_data=n_scored > 0, n_records=n_scored)
|
||||||
|
|
||||||
|
|
||||||
|
async def stats_slice(header: MatchHeader, *, limit: int = 10, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
|
"""B - 攻防数据切片: 进球、射门、控球,评估攻防强度。before=match_date 用于回测。
|
||||||
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
|
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
||||||
|
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_form = await _get_form(new_db, header.home_team_id, before=before, limit=limit)
|
||||||
|
away_form = await _get_form(new_db, header.away_team_id, before=before, limit=limit)
|
||||||
|
lines = [f"── 攻防数据(近 {limit} 场) ──"]
|
||||||
|
n_total = 0
|
||||||
|
# P0-1 修复:每场历史比赛必须根据「该队当时是主是客」判断 side,
|
||||||
|
# 不能用本场 side 硬套 —— 否则进球/失球/xG 全部算反。
|
||||||
|
for label, name, form, team_id in (
|
||||||
|
("主队", header.home_name, home_form, header.home_team_id),
|
||||||
|
("客队", header.away_name, away_form, header.away_team_id),
|
||||||
|
):
|
||||||
|
if form:
|
||||||
|
gf = ga = shots = sot = poss = xg = xga = 0
|
||||||
|
n = n_shots = n_poss = n_xg = 0
|
||||||
|
for fm in form:
|
||||||
|
if fm.home_goals is None: continue
|
||||||
|
is_home = (fm.home_team_id == team_id)
|
||||||
|
gf += fm.home_goals if is_home else fm.away_goals
|
||||||
|
ga += fm.away_goals if is_home else fm.home_goals
|
||||||
|
n += 1
|
||||||
|
# 只使用 cutoff 之前已可用的统计数据
|
||||||
|
if fm.stats and _is_stats_available(fm.stats, before):
|
||||||
|
if fm.stats.home_shots is not None:
|
||||||
|
shots += fm.stats.home_shots if is_home else fm.stats.away_shots
|
||||||
|
sot += fm.stats.home_shots_on_target if is_home else fm.stats.away_shots_on_target
|
||||||
|
n_shots += 1
|
||||||
|
if fm.stats.home_possession is not None:
|
||||||
|
poss += fm.stats.home_possession if is_home else (100 - fm.stats.home_possession)
|
||||||
|
n_poss += 1
|
||||||
|
if fm.stats.home_xg is not None:
|
||||||
|
xg += fm.stats.home_xg if is_home else fm.stats.away_xg
|
||||||
|
xga += fm.stats.away_xg if is_home else fm.stats.home_xg
|
||||||
|
n_xg += 1
|
||||||
|
n_total += n
|
||||||
|
if n > 0:
|
||||||
|
lines.append(f" {label} {name}:")
|
||||||
|
lines.append(f" 场均进球 {gf/n:.2f}, 场均失球 {ga/n:.2f}")
|
||||||
|
if n_shots: lines.append(f" 场均射门 {shots/n_shots:.1f}, 射正 {sot/n_shots:.1f}")
|
||||||
|
if n_poss: lines.append(f" 平均控球 {poss/n_poss:.1f}%")
|
||||||
|
if n_xg: lines.append(f" 场均 xG {xg/n_xg:.2f}, 场均被 xG {xga/n_xg:.2f}")
|
||||||
|
else:
|
||||||
|
lines.append(f" {label} {name}: 无比分数据")
|
||||||
|
else:
|
||||||
|
lines.append(f" {label} {name}: 无数据")
|
||||||
|
return SliceResult(text="\n".join(lines), has_data=n_total > 0, n_records=n_total)
|
||||||
|
|
||||||
|
|
||||||
|
async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
|
"""C - 主客因素切片: 主场战绩 vs 客场战绩,评估地理优势影响。before=match_date 用于回测。
|
||||||
|
|
||||||
|
db: 可选共享 session,避免每个切片独立建连(见模块 docstring)。
|
||||||
|
"""
|
||||||
|
if db is not None:
|
||||||
|
home_home = await _get_home_away(db, header.home_team_id, "home", before=before, limit=limit)
|
||||||
|
away_away = await _get_home_away(db, header.away_team_id, "away", before=before, limit=limit)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
home_home = await _get_home_away(new_db, header.home_team_id, "home", before=before, limit=limit)
|
||||||
|
away_away = await _get_home_away(new_db, header.away_team_id, "away", before=before, limit=limit)
|
||||||
|
lines = ["── 主客因素 ──"]
|
||||||
|
n_total = 0
|
||||||
|
for label, name, matches, side in (
|
||||||
|
("主队主场", header.home_name, home_home, "home"),
|
||||||
|
("客队客场", header.away_name, away_away, "away"),
|
||||||
|
):
|
||||||
|
if matches:
|
||||||
|
wins = draws = losses = gf = ga = 0
|
||||||
|
for m in matches:
|
||||||
|
if m.home_goals is None: continue
|
||||||
|
o = _outcome(m.home_goals, m.away_goals, side)
|
||||||
|
if o == "W": wins += 1
|
||||||
|
elif o == "D": draws += 1
|
||||||
|
else: losses += 1
|
||||||
|
gf += m.home_goals if side == "home" else m.away_goals
|
||||||
|
ga += m.away_goals if side == "home" else m.home_goals
|
||||||
|
n = wins + draws + losses
|
||||||
|
n_total += n
|
||||||
|
if n > 0:
|
||||||
|
pct = wins / n * 100
|
||||||
|
lines.append(f" {label} {name}(近 {n} 场): {wins}胜 {draws}平 {losses}负, 胜率 {pct:.0f}%")
|
||||||
|
lines.append(f" 场均进球 {gf/n:.2f}, 场均失球 {ga/n:.2f}")
|
||||||
|
else:
|
||||||
|
lines.append(f" {label} {name}: 无比分数据")
|
||||||
|
else:
|
||||||
|
lines.append(f" {label} {name}: 无数据")
|
||||||
|
return SliceResult(text="\n".join(lines), has_data=n_total > 0, n_records=n_total)
|
||||||
|
|
||||||
|
|
||||||
|
async def standings_slice(header: MatchHeader, *, before=None, db: AsyncSession | None = None) -> SliceResult:
|
||||||
|
"""D - 联赛排名切片: 两队积分榜位置、积分、近期走势(form)、分区,评估整体实力差距。
|
||||||
|
|
||||||
|
before 参数保留与其他切片一致的签名(积分榜是最新快照,无历史版本,不受 cutoff 影响)。
|
||||||
|
db: 可选共享 session(见模块 docstring)。
|
||||||
|
|
||||||
|
语义区分:
|
||||||
|
- 两队都有积分榜行 → has_data=True(明确的排名信息)
|
||||||
|
- 任一队缺失 → has_data=False(升班马/杯赛无榜,信息不完整时明确声明)
|
||||||
|
"""
|
||||||
|
from src.db.models import League, Standing
|
||||||
|
|
||||||
|
if db is not None:
|
||||||
|
league = (await db.execute(select(League).where(League.id == header.league_id))).scalar_one_or_none()
|
||||||
|
rows = (
|
||||||
|
(
|
||||||
|
await db.execute(
|
||||||
|
select(Standing)
|
||||||
|
.options(selectinload(Standing.team))
|
||||||
|
.where(Standing.league_id == header.league_id)
|
||||||
|
.order_by(Standing.position.asc())
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
if league
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
async with AsyncSessionLocal() as new_db:
|
||||||
|
return await standings_slice(header, before=before, db=new_db)
|
||||||
|
|
||||||
|
lines = [f"── 联赛排名({header.league_name} 共 {len(rows)} 队) ──"]
|
||||||
|
n_records = 0
|
||||||
|
|
||||||
|
def _fmt(row) -> str:
|
||||||
|
zg = f" xG差 {row.xgd:+.1f}" if row.xg_for is not None and row.xg_against is not None and row.goal_diff is not None else ""
|
||||||
|
form = f" 近5场 {row.form}" if row.form else ""
|
||||||
|
zone = f" [{row.zone}]" if row.zone else ""
|
||||||
|
return (
|
||||||
|
f" 第 {row.position} 名: {row.points} 分 / {row.played} 场 "
|
||||||
|
f"({row.won}胜{row.drawn}平{row.lost}负, 进{row.goals_for}失{row.goals_against} 净胜{row.goal_diff:+d}"
|
||||||
|
f"{zg}){form}{zone}"
|
||||||
|
)
|
||||||
|
|
||||||
|
for label, team_id in (("主队", header.home_team_id), ("客队", header.away_team_id)):
|
||||||
|
row = next((r for r in rows if r.team_id == team_id), None)
|
||||||
|
if row is None:
|
||||||
|
lines.append(f" {label}: 暂无积分榜数据(可能杯赛/赛季未开始)")
|
||||||
|
else:
|
||||||
|
n_records += 1
|
||||||
|
lines.append(f" {label} {header.home_name if label == '主队' else header.away_name}:")
|
||||||
|
lines.append(_fmt(row))
|
||||||
|
|
||||||
|
# 两队排名对比摘要
|
||||||
|
home_row = next((r for r in rows if r.team_id == header.home_team_id), None)
|
||||||
|
away_row = next((r for r in rows if r.team_id == header.away_team_id), None)
|
||||||
|
if home_row and away_row:
|
||||||
|
diff = home_row.position - away_row.position # 正数=主队排名更靠前(名次更小)
|
||||||
|
lead = f"主队排名高 {diff} 位" if diff > 0 else (f"客队排名高 {-diff} 位" if diff < 0 else "两队同排名结构")
|
||||||
|
pts_diff = home_row.points - away_row.points
|
||||||
|
lines.append(f" 排名对比: {lead}, 分差 {pts_diff:+d}")
|
||||||
|
|
||||||
|
# has_data: 两队都有行才算完整;只有一队时仍有价值,但标记不完整
|
||||||
|
has_data = n_records >= 1
|
||||||
|
return SliceResult(text="\n".join(lines), has_data=has_data, n_records=n_records)
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 单 agent 路径: 拼接全部切片(行为与旧版一致)
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5, backtest: bool = False, cutoff_at=None) -> MatchContext:
|
||||||
|
"""单 agent 路径的完整上下文: 拼接全部切片(before=cutoff,防未来信息)。
|
||||||
|
|
||||||
|
has_stats / has_standings 直接取切片显式声明的 has_data,
|
||||||
|
不再靠文案子串匹配(见审查报告 P2-1)。
|
||||||
|
|
||||||
|
P2-6: backtest=True 时 cutoff = match_date - 1天,确保只用赛前数据。
|
||||||
|
cutoff_at: 显式截止时间(优先于 backtest 自动计算)。
|
||||||
|
|
||||||
|
P1-1: 使用单个共享 session 贯穿所有切片查询,避免连接池耗尽。
|
||||||
|
"""
|
||||||
|
async with AsyncSessionLocal() as db:
|
||||||
|
header = await load_match_header(match_id, db=db)
|
||||||
|
# 计算数据截止时间: 显式 > backtest 自动计算 > 默认(比赛时间)
|
||||||
|
if cutoff_at is not None:
|
||||||
|
cutoff = cutoff_at
|
||||||
|
elif backtest and header.match_dt:
|
||||||
|
from datetime import timedelta
|
||||||
|
cutoff = header.match_dt - timedelta(days=1)
|
||||||
|
else:
|
||||||
|
cutoff = header.match_dt
|
||||||
|
parts = [header_text(header), ""]
|
||||||
|
|
||||||
|
form_res = await form_slice(header, limit=form_last, before=cutoff, db=db)
|
||||||
|
parts.append(form_res.text)
|
||||||
|
parts.append("")
|
||||||
|
|
||||||
|
h2h_res = await h2h_slice(header, limit=h2h_last, before=cutoff, db=db)
|
||||||
|
parts.append(h2h_res.text)
|
||||||
|
parts.append("")
|
||||||
|
|
||||||
|
stats_res = await stats_slice(header, before=cutoff, db=db)
|
||||||
|
parts.append(stats_res.text)
|
||||||
|
parts.append("")
|
||||||
|
|
||||||
|
home_away_res = await home_away_slice(header, before=cutoff, db=db)
|
||||||
|
parts.append(home_away_res.text)
|
||||||
|
parts.append("")
|
||||||
|
|
||||||
|
standings_res = await standings_slice(header, before=cutoff, db=db)
|
||||||
|
parts.append(standings_res.text)
|
||||||
|
|
||||||
|
return MatchContext(
|
||||||
|
match_id=match_id,
|
||||||
|
text="\n".join(parts),
|
||||||
|
has_stats=form_res.has_data or stats_res.has_data,
|
||||||
|
has_standings=standings_res.has_data,
|
||||||
|
match_dt=header.match_dt,
|
||||||
|
cutoff=cutoff,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ============================================================
|
||||||
|
# 底层查询(切片函数共用)
|
||||||
|
# ============================================================
|
||||||
|
|
||||||
|
async def _load_match(db, match_id: int) -> Match:
|
||||||
|
stmt = (
|
||||||
|
select(Match)
|
||||||
|
.where(Match.id == match_id)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.league),
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
selectinload(Match.stats),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
m = (await db.execute(stmt)).scalar_one_or_none()
|
||||||
|
if m is None:
|
||||||
|
raise ValueError(f"match {match_id} not found")
|
||||||
|
return m
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_form(db, team_id: int, before, *, limit: int = 5) -> list[Match]:
|
||||||
|
"""某队近 N 场(已完赛)。before=None 表示不限制(预测赛前的场景由调用方保证)。
|
||||||
|
|
||||||
|
必须预加载 stats / home_team / away_team:切片函数会读取这些关系,
|
||||||
|
而 async session 下惰性加载会抛 MissingGreenlet。
|
||||||
|
(models.py 已声明 lazy="selectin",此处显式声明以固化查询意图。)
|
||||||
|
"""
|
||||||
|
stmt = (
|
||||||
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.stats),
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
|
.where(Match.match_status == "finished")
|
||||||
|
.where(Match.home_goals.is_not(None))
|
||||||
|
.where((Match.home_team_id == team_id) | (Match.away_team_id == team_id))
|
||||||
|
.order_by(Match.match_date.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
if before is not None:
|
||||||
|
stmt = stmt.where(Match.match_date < before)
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
return list(result.scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_h2h(db, home_id: int, away_id: int, before, *, limit: int = 5) -> list[Match]:
|
||||||
|
"""两队交锋史。需预加载 home_team / away_team(切片输出队名)。"""
|
||||||
|
stmt = (
|
||||||
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
|
.where(Match.match_status == "finished")
|
||||||
|
.where(Match.home_goals.is_not(None))
|
||||||
|
.where(
|
||||||
|
((Match.home_team_id == home_id) & (Match.away_team_id == away_id))
|
||||||
|
| ((Match.home_team_id == away_id) & (Match.away_team_id == home_id))
|
||||||
|
)
|
||||||
|
.order_by(Match.match_date.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
if before is not None:
|
||||||
|
stmt = stmt.where(Match.match_date < before)
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
return list(result.scalars().all())
|
||||||
|
|
||||||
|
|
||||||
|
async def _get_home_away(db, team_id: int, side: str, before, *, limit: int = 10) -> list[Match]:
|
||||||
|
"""某队主场/客场近 N 场。side='home' 取主场,'away' 取客场。
|
||||||
|
|
||||||
|
当前只用标量字段,但统一预加载以免后续扩展时踩坑。
|
||||||
|
"""
|
||||||
|
stmt = (
|
||||||
|
select(Match)
|
||||||
|
.options(
|
||||||
|
selectinload(Match.stats),
|
||||||
|
selectinload(Match.home_team),
|
||||||
|
selectinload(Match.away_team),
|
||||||
|
)
|
||||||
|
.where(Match.match_status == "finished")
|
||||||
|
.where(Match.home_goals.is_not(None))
|
||||||
|
.order_by(Match.match_date.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
if side == "home":
|
||||||
|
stmt = stmt.where(Match.home_team_id == team_id)
|
||||||
|
else:
|
||||||
|
stmt = stmt.where(Match.away_team_id == team_id)
|
||||||
|
if before is not None:
|
||||||
|
stmt = stmt.where(Match.match_date < before)
|
||||||
|
result = await db.execute(stmt)
|
||||||
|
return list(result.scalars().all())
|
||||||
|
|||||||
+6
-3
@@ -7,7 +7,6 @@ from sqlalchemy import func, or_, select
|
|||||||
|
|
||||||
from src.db.models import Prediction, Match, League
|
from src.db.models import Prediction, Match, League
|
||||||
from src.db.unit_of_work import get_uow
|
from src.db.unit_of_work import get_uow
|
||||||
from src.llm.utils import actual_1x2
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -34,8 +33,12 @@ async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int
|
|||||||
|
|
||||||
|
|
||||||
def _actual_1x2(home: int, away: int) -> str:
|
def _actual_1x2(home: int, away: int) -> str:
|
||||||
"""根据实际比分返胜平负(委托 utils.actual_1x2 单一权威源)。"""
|
"""根据实际比分返胜平负。"""
|
||||||
return actual_1x2(home, away)
|
if home > away:
|
||||||
|
return "1"
|
||||||
|
if home == away:
|
||||||
|
return "X"
|
||||||
|
return "2"
|
||||||
|
|
||||||
|
|
||||||
def _build_filters(
|
def _build_filters(
|
||||||
|
|||||||
+69
-191
@@ -1,16 +1,4 @@
|
|||||||
"""预测服务:拼上下文 → 调 LLM → 存预测。
|
"""预测服务:拼上下文 → 调 LLM → 存预测。"""
|
||||||
|
|
||||||
落库层级(table: predictions):
|
|
||||||
|
|
||||||
| 模式 | 落库位置(服务层) | 路由层(routes/predict.py) |
|
|
||||||
|-----------|-------------------------------------------------------------|---------------------------|
|
|
||||||
| single | `_predict_single` → `_insert_or_find_by_fingerprint` | 不读 DB,仅映射 result → PredictOut |
|
|
||||||
| multi | `orchestrator.predict_match_multi` → `_insert_or_find_by_fingerprint` | 不读 DB,仅映射 result → PredictOut |
|
|
||||||
| baseline | `predict_baseline` → `_insert_or_find_by_fingerprint` | 不读 DB,仅映射 result → PredictOut |
|
|
||||||
|
|
||||||
三种模式统一在服务层经 UnitOfWork 落库并回填真实 prediction_id;
|
|
||||||
路由层永不写入 predictions,只读 result.prediction_id 做响应映射。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import functools
|
import functools
|
||||||
@@ -37,7 +25,9 @@ _PROMPT_DIR = Path(__file__).resolve().parent / "prompts"
|
|||||||
# ── LLM 响应缓存(match+provider+model+version → 结果) ──
|
# ── LLM 响应缓存(match+provider+model+version → 结果) ──
|
||||||
_CACHE_TTL_SEC = 300 # 5 分钟
|
_CACHE_TTL_SEC = 300 # 5 分钟
|
||||||
_CACHE_MAX_SIZE = 200 # P3-1: 有上限,避免长期运行内存无限增长
|
_CACHE_MAX_SIZE = 200 # P3-1: 有上限,避免长期运行内存无限增长
|
||||||
_CACHE_PREFIX = "predict:" # Redis key 前缀
|
# P1-5: 缓存仅在 asyncio 协程内同步访问(dict 操作 GIL 原子),无需 threading.Lock。
|
||||||
|
# 删除 _cache_lock,避免同步锁阻塞事件循环;dict 的 get/set 在 CPython 下原子。
|
||||||
|
_cache: dict[str, tuple[float, PredictResult]] = {}
|
||||||
|
|
||||||
|
|
||||||
def _cache_key(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> str:
|
def _cache_key(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> str:
|
||||||
@@ -46,132 +36,43 @@ def _cache_key(match_id: int, provider: str, model: str, version: str, tpl_hash:
|
|||||||
仅用 version 做键不够 —— 编辑器里改动 `match_prediction_v1.md` 而版本号
|
仅用 version 做键不够 —— 编辑器里改动 `match_prediction_v1.md` 而版本号
|
||||||
不变时,进程内缓存仍会返回旧模板产生的旧结果(见审查报告 P2-6)。
|
不变时,进程内缓存仍会返回旧模板产生的旧结果(见审查报告 P2-6)。
|
||||||
把模板内容 hash 纳入键,模板一改缓存自动失效。
|
把模板内容 hash 纳入键,模板一改缓存自动失效。
|
||||||
内存与 Redis 共用同一键格式,TTL 一致。
|
|
||||||
"""
|
"""
|
||||||
return f"{_CACHE_PREFIX}{match_id}:{provider}:{model}:{version}:{tpl_hash[:12]}"
|
return f"{match_id}:{provider}:{model}:{version}:{tpl_hash[:12]}"
|
||||||
|
|
||||||
|
|
||||||
# ── 缓存后端:内存(LRU+TTL),可选 Redis ──────────────────────────
|
def _get_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> PredictResult | None:
|
||||||
class _CacheBackend:
|
# P1-5: 无锁访问。dict get/del 在 CPython GIL 下原子,且无 await 穿插。
|
||||||
"""缓存后端统一接口:_get 同步返回(命中时),_set 异步(Redis 为 async,内存同步)。"""
|
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
||||||
|
entry = _cache.get(key)
|
||||||
def _raw_key(self, key: str) -> str:
|
|
||||||
return key
|
|
||||||
|
|
||||||
def get(self, key: str) -> PredictResult | None:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
async def set(self, key: str, result: PredictResult, ttl: int) -> None:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
|
|
||||||
class _MemoryCache(_CacheBackend):
|
|
||||||
"""进程内 LRU+TTL 缓存(默认后端)。"""
|
|
||||||
|
|
||||||
def __init__(self) -> None:
|
|
||||||
self._store: dict[str, tuple[float, PredictResult]] = {}
|
|
||||||
|
|
||||||
def get(self, key: str) -> PredictResult | None:
|
|
||||||
entry = self._store.get(key)
|
|
||||||
if entry is not None:
|
if entry is not None:
|
||||||
ts, result = entry
|
ts, result = entry
|
||||||
if time.time() - ts < _CACHE_TTL_SEC:
|
if time.time() - ts < _CACHE_TTL_SEC:
|
||||||
return result
|
return result
|
||||||
self._store.pop(key, None)
|
_cache.pop(key, None)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
async def set(self, key: str, result: PredictResult, ttl: int) -> None:
|
|
||||||
self._store[key] = (time.time(), result)
|
def _set_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str, result: PredictResult) -> None:
|
||||||
if len(self._store) > _CACHE_MAX_SIZE:
|
# P1-5: 无锁写入。同上,dict set 原子。
|
||||||
oldest_key = min(self._store, key=lambda k: self._store[k][0])
|
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
||||||
self._store.pop(oldest_key, None)
|
_cache[key] = (time.time(), result)
|
||||||
|
# P3-1: 超过上限时淘汰最旧条目(按时间戳排序)
|
||||||
|
if len(_cache) > _CACHE_MAX_SIZE:
|
||||||
|
oldest_key = min(_cache, key=lambda k: _cache[k][0])
|
||||||
|
_cache.pop(oldest_key, None)
|
||||||
|
|
||||||
|
|
||||||
class _RedisCache(_CacheBackend):
|
def clear_prompt_cache() -> None:
|
||||||
"""可选 Redis 后端: PREDICT_CACHE_URL 非空时启用。
|
"""清空 prompt 模板缓存(供开发/热更新时手动调用)。
|
||||||
|
|
||||||
失败降级:读失败返回 None(跳过缓存),写失败打 warning;不中断预测主流程。
|
lru_cache 的模板缓存是进程级的,改完 .md 需要重启进程才能生效;
|
||||||
不强制依赖 redis 包——未安装时启动回退内存并 warning。
|
提供显式清理入口,避免"改了模板却看不到变化"的困惑(见审查报告 P2-5)。
|
||||||
"""
|
"""
|
||||||
|
_load_prompt_template.cache_clear()
|
||||||
def __init__(self, url: str) -> None:
|
logger.info("prompt 模板缓存已清空")
|
||||||
self._url = url
|
|
||||||
self._redis = None # type: ignore[var-annotated]
|
|
||||||
self._memory_fallback = _MemoryCache()
|
|
||||||
self._available: bool | None = None # None=未探测,True=可用,False=不可用
|
|
||||||
|
|
||||||
async def _ensure_conn(self) -> bool:
|
|
||||||
"""懒初始化 Redis 连接;失败返回 False 并降级内存。"""
|
|
||||||
if self._available is not None:
|
|
||||||
return self._available
|
|
||||||
try:
|
|
||||||
from redis.asyncio import Redis
|
|
||||||
|
|
||||||
self._redis = Redis.from_url(self._url, decode_responses=True, socket_timeout=2.0)
|
|
||||||
await self._redis.ping()
|
|
||||||
self._available = True
|
|
||||||
logger.info("predict cache: Redis 后端已连接 %s", self._url.replace(self._url.split("@")[-1] if "@" in self._url else self._url, "***") if "://" in self._url else "redis")
|
|
||||||
except Exception as e:
|
|
||||||
self._available = False
|
|
||||||
logger.warning("predict cache: Redis 连接失败(%s),降级内存缓存", e)
|
|
||||||
return self._available
|
|
||||||
|
|
||||||
def get(self, key: str) -> PredictResult | None:
|
|
||||||
# Redis get 是 async 的,此处统一由调用方走 async 路径;
|
|
||||||
# 同步 get 仅用于不可降级场景——Redis 模式下直接返回 None,
|
|
||||||
# 实际读取通过 get_async 完成。
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def get_async(self, key: str) -> PredictResult | None:
|
|
||||||
if not await self._ensure_conn():
|
|
||||||
return self._memory_fallback.get(key)
|
|
||||||
try:
|
|
||||||
import pickle
|
|
||||||
|
|
||||||
raw = await self._redis.get(key) # type: ignore[union-attr]
|
|
||||||
if raw is None:
|
|
||||||
return None
|
|
||||||
return pickle.loads(raw.encode("latin-1")) if isinstance(raw, str) else pickle.loads(raw)
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("predict cache: Redis GET 失败(%s),跳过缓存", e)
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def set(self, key: str, result: PredictResult, ttl: int) -> None:
|
|
||||||
if not await self._ensure_conn():
|
|
||||||
await self._memory_fallback.set(key, result, ttl)
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
import pickle
|
|
||||||
|
|
||||||
payload = pickle.dumps(result).decode("latin-1")
|
|
||||||
await self._redis.set(key, payload, ex=ttl) # type: ignore[union-attr]
|
|
||||||
except Exception as e:
|
|
||||||
logger.warning("predict cache: Redis SET 失败(%s),降级内存写入", e)
|
|
||||||
await self._memory_fallback.set(key, result, ttl)
|
|
||||||
|
|
||||||
|
|
||||||
def _build_cache_backend() -> _CacheBackend:
|
|
||||||
url = getattr(settings, "PREDICT_CACHE_URL", None)
|
|
||||||
if url:
|
|
||||||
return _RedisCache(url)
|
|
||||||
return _MemoryCache()
|
|
||||||
|
|
||||||
|
|
||||||
_cache_backend: _CacheBackend = _build_cache_backend()
|
|
||||||
|
|
||||||
|
|
||||||
async def _get_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str) -> PredictResult | None:
|
|
||||||
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
|
||||||
if isinstance(_cache_backend, _RedisCache):
|
|
||||||
return await _cache_backend.get_async(key)
|
|
||||||
return _cache_backend.get(key)
|
|
||||||
|
|
||||||
|
|
||||||
async def _set_cached(match_id: int, provider: str, model: str, version: str, tpl_hash: str, result: PredictResult) -> None:
|
|
||||||
key = _cache_key(match_id, provider, model, version, tpl_hash)
|
|
||||||
await _cache_backend.set(key, result, _CACHE_TTL_SEC)
|
|
||||||
|
|
||||||
|
|
||||||
|
@functools.lru_cache(maxsize=8)
|
||||||
def _load_prompt_template(version: str = "v1") -> str:
|
def _load_prompt_template(version: str = "v1") -> str:
|
||||||
"""缓存 prompt 模板(进程生命周期内每个版本只读一次)。"""
|
"""缓存 prompt 模板(进程生命周期内每个版本只读一次)。"""
|
||||||
path = _PROMPT_DIR / f"match_prediction_{version}.md"
|
path = _PROMPT_DIR / f"match_prediction_{version}.md"
|
||||||
@@ -220,59 +121,43 @@ class PredictResult:
|
|||||||
raw: dict | None = None
|
raw: dict | None = None
|
||||||
|
|
||||||
|
|
||||||
def _compute_fingerprint(values: dict) -> str:
|
async def _upsert_prediction(
|
||||||
"""P0-03: 预测指纹(规范 JSON 的 SHA-256)。
|
session,
|
||||||
|
*,
|
||||||
|
match_id: int,
|
||||||
|
provider_name: str,
|
||||||
|
model: str,
|
||||||
|
mode: str,
|
||||||
|
run_type: str,
|
||||||
|
values: dict,
|
||||||
|
) -> Prediction:
|
||||||
|
"""按 (match, provider, model, mode, run_type) 唯一约束写入预测。
|
||||||
|
|
||||||
捕获影响预测输出的全部因素:输入、提示、模型、采样、截止时间、专家。
|
已存在且未结算 → 覆盖更新(重新预测语义);已结算 → 拒绝(保护评估数据)。
|
||||||
同 fingerprint → 返回已有行(不 UPDATE/INSERT);不同 → INSERT 新行。
|
run_type 区分 live/backtest,避免回测覆盖实盘预测。
|
||||||
"""
|
"""
|
||||||
import json as _json
|
|
||||||
|
|
||||||
canonical = {
|
|
||||||
"match_id": values.get("match_id"),
|
|
||||||
"prediction_cutoff_at": _iso(values.get("prediction_cutoff_at")),
|
|
||||||
"prompt_version": values.get("prompt_version"),
|
|
||||||
"prompt_hash": values.get("prompt_hash"),
|
|
||||||
"system_prompt_hash": values.get("system_prompt_hash"),
|
|
||||||
"provider": values.get("provider"),
|
|
||||||
"model": values.get("model"),
|
|
||||||
"mode": values.get("mode"),
|
|
||||||
"run_type": values.get("run_type"),
|
|
||||||
"temperature": values.get("temperature"),
|
|
||||||
"context_hash": values.get("context_hash"),
|
|
||||||
"agent_ids": sorted(values.get("agent_ids") or []),
|
|
||||||
}
|
|
||||||
blob = _json.dumps(canonical, sort_keys=True, separators=(',', ':'))
|
|
||||||
return hashlib.sha256(blob.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
|
|
||||||
def _iso(v) -> str | None:
|
|
||||||
if v is None:
|
|
||||||
return None
|
|
||||||
if hasattr(v, "isoformat"):
|
|
||||||
return v.isoformat()
|
|
||||||
return str(v)
|
|
||||||
|
|
||||||
|
|
||||||
async def _insert_or_find_by_fingerprint(session, *, values: dict) -> Prediction:
|
|
||||||
"""P0-03: 幂等插入——同 input_hash 返回已有行(不 UPDATE);不同则 INSERT。
|
|
||||||
|
|
||||||
不再按 (match, provider, model, mode, run_type) 做 upsert,避免覆盖已有预测。
|
|
||||||
values 必须包含 fingerprint 所需全部字段(见 _compute_fingerprint)。
|
|
||||||
"""
|
|
||||||
fingerprint = _compute_fingerprint(values)
|
|
||||||
values["input_hash"] = fingerprint
|
|
||||||
|
|
||||||
existing = (
|
existing = (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(Prediction).where(Prediction.input_hash == fingerprint)
|
select(Prediction).where(
|
||||||
|
Prediction.match_id == match_id,
|
||||||
|
Prediction.provider == provider_name,
|
||||||
|
Prediction.model == model,
|
||||||
|
Prediction.mode == mode,
|
||||||
|
Prediction.run_type == run_type,
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None and existing.settled:
|
||||||
# 同指纹 → 直接返回,绝不覆盖 pred_* / reasoning / agent_outputs
|
raise ValueError("该比赛已有已结算的预测,不能重新预测")
|
||||||
return existing
|
|
||||||
|
|
||||||
pred = Prediction(**{k: v for k, v in values.items() if hasattr(Prediction, k)})
|
pred = existing if existing is not None else Prediction(
|
||||||
|
match_id=match_id, provider=provider_name, model=model,
|
||||||
|
)
|
||||||
|
pred.mode = mode
|
||||||
|
pred.run_type = run_type
|
||||||
|
for k, v in values.items():
|
||||||
|
setattr(pred, k, v)
|
||||||
|
if existing is None:
|
||||||
session.add(pred)
|
session.add(pred)
|
||||||
await session.flush() # 拿到自增 id;事务由 UnitOfWork 退出时提交
|
await session.flush() # 拿到自增 id;事务由 UnitOfWork 退出时提交
|
||||||
return pred
|
return pred
|
||||||
@@ -350,7 +235,7 @@ async def _predict_single(
|
|||||||
|
|
||||||
# 0. 查缓存(同 match+provider+model+version+模板hash 5 分钟内直接返)
|
# 0. 查缓存(同 match+provider+model+version+模板hash 5 分钟内直接返)
|
||||||
if use_cache:
|
if use_cache:
|
||||||
cached = await _get_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash)
|
cached = _get_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash)
|
||||||
if cached is not None:
|
if cached is not None:
|
||||||
logger.debug("predict cache hit match=%s", match_id)
|
logger.debug("predict cache hit match=%s", match_id)
|
||||||
return cached
|
return cached
|
||||||
@@ -358,26 +243,23 @@ async def _predict_single(
|
|||||||
# 1. 拼上下文(backtest/cutoff 防泄漏)
|
# 1. 拼上下文(backtest/cutoff 防泄漏)
|
||||||
ctx = await build_context(match_id, backtest=backtest, cutoff_at=cutoff_at)
|
ctx = await build_context(match_id, backtest=backtest, cutoff_at=cutoff_at)
|
||||||
|
|
||||||
# 1.5 计算快照元数据(用于可复现性 + P0-03 指纹)
|
# 1.5 计算快照元数据(用于可复现性)
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
match_kickoff_at = ctx.match_dt
|
match_kickoff_at = ctx.match_dt
|
||||||
# 使用上下文实际计算的 cutoff(回测时可能为 match_dt-1天),而非开球时间
|
# 使用上下文实际计算的 cutoff(回测时可能为 match_dt-1天),而非开球时间
|
||||||
prediction_cutoff_at = ctx.cutoff if ctx.cutoff is not None else ctx.match_dt
|
prediction_cutoff_at = ctx.cutoff if ctx.cutoff is not None else ctx.match_dt
|
||||||
|
input_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
# 2. 拼 prompt(指定版本)
|
# 2. 拼 prompt(指定版本)
|
||||||
template = _load_prompt_template(version)
|
template = _load_prompt_template(version)
|
||||||
prompt_hash = _prompt_template_hash(version)
|
|
||||||
user_prompt = template.replace("{{context}}", ctx.text)
|
user_prompt = template.replace("{{context}}", ctx.text)
|
||||||
system_prompt = "你是一个严谨的足球预测专家。只输出 JSON。"
|
|
||||||
context_hash = hashlib.sha256(ctx.text.encode("utf-8")).hexdigest()
|
|
||||||
|
|
||||||
# 3. 调 LLM
|
# 3. 调 LLM
|
||||||
temperature = 0.3
|
|
||||||
resp = await provider.chat(
|
resp = await provider.chat(
|
||||||
system=system_prompt,
|
system="你是一个严谨的足球预测专家。只输出 JSON。",
|
||||||
user=user_prompt,
|
user=user_prompt,
|
||||||
json_mode=True,
|
json_mode=True,
|
||||||
temperature=temperature,
|
temperature=0.3,
|
||||||
max_tokens=4096, # 推理模型的 reasoning 也计入输出 token,需留足余量
|
max_tokens=4096, # 推理模型的 reasoning 也计入输出 token,需留足余量
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -404,20 +286,15 @@ async def _predict_single(
|
|||||||
if m is None:
|
if m is None:
|
||||||
raise ValueError(f"match {match_id} not found")
|
raise ValueError(f"match {match_id} not found")
|
||||||
|
|
||||||
pred = await _insert_or_find_by_fingerprint(
|
pred = await _upsert_prediction(
|
||||||
session,
|
session,
|
||||||
|
match_id=match_id,
|
||||||
|
provider_name=settings.LLM_PROVIDER,
|
||||||
|
model=provider.model,
|
||||||
|
mode="single",
|
||||||
|
run_type="backtest" if backtest else "live",
|
||||||
values={
|
values={
|
||||||
"match_id": match_id,
|
|
||||||
"provider": settings.LLM_PROVIDER,
|
|
||||||
"model": provider.model,
|
|
||||||
"mode": "single",
|
|
||||||
"run_type": "backtest" if backtest else "live",
|
|
||||||
"prompt_version": version,
|
"prompt_version": version,
|
||||||
"prompt_hash": prompt_hash,
|
|
||||||
"system_prompt_hash": hashlib.sha256(system_prompt.encode("utf-8")).hexdigest(),
|
|
||||||
"temperature": temperature,
|
|
||||||
"context_hash": context_hash,
|
|
||||||
"agent_ids": [],
|
|
||||||
"prompt_tokens": resp.prompt_tokens,
|
"prompt_tokens": resp.prompt_tokens,
|
||||||
"completion_tokens": resp.completion_tokens,
|
"completion_tokens": resp.completion_tokens,
|
||||||
"latency_ms": resp.latency_ms,
|
"latency_ms": resp.latency_ms,
|
||||||
@@ -433,6 +310,7 @@ async def _predict_single(
|
|||||||
"match_kickoff_at": match_kickoff_at,
|
"match_kickoff_at": match_kickoff_at,
|
||||||
"prediction_cutoff_at": prediction_cutoff_at,
|
"prediction_cutoff_at": prediction_cutoff_at,
|
||||||
"prediction_created_at": now,
|
"prediction_created_at": now,
|
||||||
|
"input_hash": input_hash,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -456,7 +334,7 @@ async def _predict_single(
|
|||||||
|
|
||||||
# 5. 写入缓存(仅当允许缓存时)
|
# 5. 写入缓存(仅当允许缓存时)
|
||||||
if use_cache:
|
if use_cache:
|
||||||
await _set_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash, result)
|
_set_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash, result)
|
||||||
logger.info(
|
logger.info(
|
||||||
"预测完成 match=%s mode=%s status=%s pred=%s:%s (%s) latency=%sms",
|
"预测完成 match=%s mode=%s status=%s pred=%s:%s (%s) latency=%sms",
|
||||||
match_id, "single", "success",
|
match_id, "single", "success",
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
"""slices 包:按领域拆分的数据切片(form/stats/h2h/home_away/standings)。
|
|
||||||
|
|
||||||
对外统一经 src.llm.context_builder 再导出;本包 __init__ 不承载导出,
|
|
||||||
保持「context_builder 是唯一公开入口」的 import 约定。
|
|
||||||
"""
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
"""单 agent 聚合路径: 拼接全部切片(build_context)。
|
|
||||||
|
|
||||||
共享 session 贯穿所有切片(见 context_builder 模块 docstring 的性能说明)。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.llm.slices.common import MatchContext, header_text, load_match_header
|
|
||||||
from src.llm.slices.form import form_slice
|
|
||||||
from src.llm.slices.h2h import h2h_slice
|
|
||||||
from src.llm.slices.home_away import home_away_slice
|
|
||||||
from src.llm.slices.standings import standings_slice
|
|
||||||
from src.llm.slices.stats import stats_slice
|
|
||||||
|
|
||||||
|
|
||||||
async def build_context(match_id: int, *, form_last: int = 5, h2h_last: int = 5, backtest: bool = False, cutoff_at=None) -> MatchContext:
|
|
||||||
"""单 agent 路径的完整上下文: 拼接全部切片(before=cutoff,防未来信息)。
|
|
||||||
|
|
||||||
has_stats / has_standings 直接取切片显式声明的 has_data,
|
|
||||||
不再靠文案子串匹配(见审查报告 P2-1)。
|
|
||||||
|
|
||||||
P2-6: backtest=True 时 cutoff = match_date - 1天,确保只用赛前数据。
|
|
||||||
cutoff_at: 显式截止时间(优先于 backtest 自动计算)。
|
|
||||||
|
|
||||||
P1-1: 使用单个共享 session 贯穿所有切片查询,避免连接池耗尽。
|
|
||||||
"""
|
|
||||||
async with AsyncSessionLocal() as db:
|
|
||||||
header = await load_match_header(match_id, db=db)
|
|
||||||
# 计算数据截止时间: 显式 > backtest 自动计算 > 默认(比赛时间)
|
|
||||||
if cutoff_at is not None:
|
|
||||||
cutoff = cutoff_at
|
|
||||||
elif backtest and header.match_dt:
|
|
||||||
from datetime import timedelta
|
|
||||||
cutoff = header.match_dt - timedelta(days=1)
|
|
||||||
else:
|
|
||||||
cutoff = header.match_dt
|
|
||||||
parts = [header_text(header), ""]
|
|
||||||
|
|
||||||
form_res = await form_slice(header, limit=form_last, before=cutoff, db=db)
|
|
||||||
parts.append(form_res.text)
|
|
||||||
parts.append("")
|
|
||||||
|
|
||||||
h2h_res = await h2h_slice(header, limit=h2h_last, before=cutoff, db=db)
|
|
||||||
parts.append(h2h_res.text)
|
|
||||||
parts.append("")
|
|
||||||
|
|
||||||
stats_res = await stats_slice(header, before=cutoff, db=db)
|
|
||||||
parts.append(stats_res.text)
|
|
||||||
parts.append("")
|
|
||||||
|
|
||||||
home_away_res = await home_away_slice(header, before=cutoff, db=db)
|
|
||||||
parts.append(home_away_res.text)
|
|
||||||
parts.append("")
|
|
||||||
|
|
||||||
standings_res = await standings_slice(header, before=cutoff, db=db)
|
|
||||||
parts.append(standings_res.text)
|
|
||||||
|
|
||||||
return MatchContext(
|
|
||||||
match_id=match_id,
|
|
||||||
text="\n".join(parts),
|
|
||||||
has_stats=form_res.has_data or stats_res.has_data,
|
|
||||||
has_standings=standings_res.has_data,
|
|
||||||
match_dt=header.match_dt,
|
|
||||||
cutoff=cutoff,
|
|
||||||
)
|
|
||||||
@@ -1,149 +0,0 @@
|
|||||||
"""切片共享基础:结果类型 / 比赛头信息 / 赛果与统计可用性判定。
|
|
||||||
|
|
||||||
从 context_builder.py 按领域拆出(单文件 → slices 包),仅做搬迁无逻辑修改。
|
|
||||||
各领域切片见同包 form/h2h/stats/home_away/standings 模块;
|
|
||||||
聚合入口 build_context 见 aggregate.py;对外统一经 context_builder 再导出。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import logging
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import selectinload
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.db.models import Match
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def _outcome(home_goals: int, away_goals: int, side: str) -> str:
|
|
||||||
"""从某队视角看赛果: W/D/L。"""
|
|
||||||
if home_goals is None or away_goals is None:
|
|
||||||
return "?"
|
|
||||||
if side == "home":
|
|
||||||
return "W" if home_goals > away_goals else ("D" if home_goals == away_goals else "L")
|
|
||||||
return "W" if away_goals > home_goals else ("D" if away_goals == home_goals else "L")
|
|
||||||
|
|
||||||
|
|
||||||
def _is_stats_available(stats, before) -> bool:
|
|
||||||
"""检查统计数据在 cutoff 时间是否已可用。
|
|
||||||
|
|
||||||
available_at 语义:该条统计「对外可被使用」的最早时间,
|
|
||||||
至少不得早于比赛结束。用于回测防泄漏。
|
|
||||||
|
|
||||||
规则:
|
|
||||||
- before is None(实盘):available_at 为 None 时允许(兼容旧数据)
|
|
||||||
- before is not None(回测):available_at 为 None 视为不可用(保守)
|
|
||||||
- available_at > cutoff:不可用(数据在 cutoff 之后才生成)
|
|
||||||
"""
|
|
||||||
if before is None:
|
|
||||||
# 实盘模式:无时间信息时允许(兼容旧数据)
|
|
||||||
return True
|
|
||||||
# 回测模式(cutoff 不为 None):
|
|
||||||
# available_at 为 None → 无法确认是否在 cutoff 前可用,保守视为不可用
|
|
||||||
if stats.available_at is None:
|
|
||||||
return False
|
|
||||||
return stats.available_at <= before
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class SliceResult:
|
|
||||||
"""数据切片的显式结果(替代「靠文案子串猜有无数据」)。
|
|
||||||
|
|
||||||
旧实现用 `"无数据" in slice_text` 判断,依赖具体文案 —— 一旦某个切片
|
|
||||||
写成「无比分数据」「无伤停数据」这类变体,判断就会静默失配
|
|
||||||
(见审查报告 P2-1)。这里让切片函数直接声明 `has_data`,不再猜。
|
|
||||||
"""
|
|
||||||
text: str
|
|
||||||
has_data: bool
|
|
||||||
n_records: int = 0
|
|
||||||
|
|
||||||
def __str__(self) -> str: # 让老调用点可直接当 str 用
|
|
||||||
return self.text
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class MatchContext:
|
|
||||||
match_id: int
|
|
||||||
text: str
|
|
||||||
has_stats: bool
|
|
||||||
has_standings: bool
|
|
||||||
match_dt: object | None = None # 比赛时间(回测防泄漏 + 快照用)
|
|
||||||
cutoff: object | None = None # 实际使用的数据截止时间(用于落库记录)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class MatchHeader:
|
|
||||||
"""比赛基础信息(所有 agent 共享)。"""
|
|
||||||
match_id: int
|
|
||||||
home_name: str
|
|
||||||
away_name: str
|
|
||||||
league_name: str
|
|
||||||
season: str | None
|
|
||||||
match_date: str
|
|
||||||
match_dt: object # 原始 datetime,回测防泄漏用
|
|
||||||
stage: str | None
|
|
||||||
home_team_id: int
|
|
||||||
away_team_id: int
|
|
||||||
league_id: int
|
|
||||||
|
|
||||||
|
|
||||||
async def _load_match(db, match_id: int) -> Match:
|
|
||||||
stmt = (
|
|
||||||
select(Match)
|
|
||||||
.where(Match.id == match_id)
|
|
||||||
.options(
|
|
||||||
selectinload(Match.league),
|
|
||||||
selectinload(Match.home_team),
|
|
||||||
selectinload(Match.away_team),
|
|
||||||
selectinload(Match.stats),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
m = (await db.execute(stmt)).scalar_one_or_none()
|
|
||||||
if m is None:
|
|
||||||
raise ValueError(f"match {match_id} not found")
|
|
||||||
return m
|
|
||||||
|
|
||||||
|
|
||||||
async def load_match_header(match_id: int, db: AsyncSession | None = None) -> MatchHeader:
|
|
||||||
"""加载比赛头信息(各 agent 共用)。
|
|
||||||
|
|
||||||
Args:
|
|
||||||
match_id: 比赛 ID
|
|
||||||
db: 可选的共享 session。不传则自建(向后兼容)。
|
|
||||||
"""
|
|
||||||
if db is not None:
|
|
||||||
m = await _load_match(db, match_id)
|
|
||||||
return _to_header(m)
|
|
||||||
async with AsyncSessionLocal() as new_db:
|
|
||||||
m = await _load_match(new_db, match_id)
|
|
||||||
return _to_header(m)
|
|
||||||
|
|
||||||
|
|
||||||
def _to_header(m: Match) -> MatchHeader:
|
|
||||||
return MatchHeader(
|
|
||||||
match_id=m.id,
|
|
||||||
home_name=m.home_team.name_zh or m.home_team.name,
|
|
||||||
away_name=m.away_team.name_zh or m.away_team.name,
|
|
||||||
league_name=m.league.name if m.league else "?",
|
|
||||||
season=m.season,
|
|
||||||
match_date=m.match_date.strftime("%Y-%m-%d %H:%M UTC") if m.match_date else "?",
|
|
||||||
match_dt=m.match_date,
|
|
||||||
stage=m.match_stage,
|
|
||||||
home_team_id=m.home_team_id,
|
|
||||||
away_team_id=m.away_team_id,
|
|
||||||
league_id=m.league_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def header_text(h: MatchHeader) -> str:
|
|
||||||
stage = f" {h.stage}" if h.stage else ""
|
|
||||||
return (
|
|
||||||
f"对阵: {h.home_name} vs {h.away_name} | {h.league_name} {h.season or '?'}{stage} | {h.match_date}"
|
|
||||||
)
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
"""A - 近期状态切片: 近 N 场赛果 / 走势(form)。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import selectinload
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.db.models import Match
|
|
||||||
from src.llm.slices.common import MatchHeader, SliceResult, _is_stats_available, _outcome
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
|
|
||||||
async def form_slice(header: MatchHeader, *, limit: int = 5, before=None, db: AsyncSession | None = None) -> SliceResult:
|
|
||||||
"""A - 近期状态切片: 两队近 N 场赛果、关键事件、走势判断。before=match_date 用于回测。
|
|
||||||
|
|
||||||
db: 可选共享 session,避免每个切片独立建连(见 context_builder 模块 docstring)。
|
|
||||||
"""
|
|
||||||
if db is not None:
|
|
||||||
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
|
||||||
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
|
||||||
else:
|
|
||||||
async with AsyncSessionLocal() as new_db:
|
|
||||||
home_form = await _get_form(new_db, header.home_team_id, before=before, limit=limit)
|
|
||||||
away_form = await _get_form(new_db, header.away_team_id, before=before, limit=limit)
|
|
||||||
lines = []
|
|
||||||
n_scored = 0
|
|
||||||
# P0-1 修复:每场历史比赛必须根据「该队当时是主是客」判断 side,
|
|
||||||
# 不能用本场 side 硬套 —— 否则客场输球会被算成主场赢球。
|
|
||||||
for label, name, form, team_id in (
|
|
||||||
("主队", header.home_name, home_form, header.home_team_id),
|
|
||||||
("客队", header.away_name, away_form, header.away_team_id),
|
|
||||||
):
|
|
||||||
lines.append(f"── {label}近况({name},近 {limit} 场) ──")
|
|
||||||
if form:
|
|
||||||
wins = draws = losses = 0
|
|
||||||
for fm in form:
|
|
||||||
is_home = (fm.home_team_id == team_id)
|
|
||||||
side = "home" if is_home else "away"
|
|
||||||
o = _outcome(fm.home_goals, fm.away_goals, side)
|
|
||||||
if o == "W": wins += 1
|
|
||||||
elif o == "D": draws += 1
|
|
||||||
else: losses += 1
|
|
||||||
if fm.home_goals is not None:
|
|
||||||
n_scored += 1
|
|
||||||
score = f"{fm.home_goals}-{fm.away_goals}" if fm.home_goals is not None else "vs"
|
|
||||||
xg = ""
|
|
||||||
if fm.stats and _is_stats_available(fm.stats, before) and fm.stats.home_xg is not None:
|
|
||||||
own = fm.stats.home_xg if is_home else fm.stats.away_xg
|
|
||||||
xg = f" (xG {own:.1f})"
|
|
||||||
opp = fm.away_team.name if is_home else fm.home_team.name
|
|
||||||
lines.append(f" {o} {score} vs {opp}{xg}")
|
|
||||||
lines.append(f" 近 {len(form)} 场: {wins}胜 {draws}平 {losses}负")
|
|
||||||
else:
|
|
||||||
lines.append(" 无数据")
|
|
||||||
return SliceResult(text="\n".join(lines), has_data=n_scored > 0, n_records=n_scored)
|
|
||||||
|
|
||||||
|
|
||||||
async def _get_form(db, team_id: int, before, *, limit: int = 5) -> list[Match]:
|
|
||||||
"""某队近 N 场(已完赛)。before=None 表示不限制(预测赛前的场景由调用方保证)。
|
|
||||||
|
|
||||||
必须预加载 stats / home_team / away_team:切片函数会读取这些关系,
|
|
||||||
而 async session 下惰性加载会抛 MissingGreenlet。
|
|
||||||
(models.py 已声明 lazy="selectin",此处显式声明以固化查询意图。)
|
|
||||||
"""
|
|
||||||
stmt = (
|
|
||||||
select(Match)
|
|
||||||
.options(
|
|
||||||
selectinload(Match.stats),
|
|
||||||
selectinload(Match.home_team),
|
|
||||||
selectinload(Match.away_team),
|
|
||||||
)
|
|
||||||
.where(Match.match_status == "finished")
|
|
||||||
.where(Match.score_status == "known")
|
|
||||||
.where(Match.home_goals.is_not(None))
|
|
||||||
.where((Match.home_team_id == team_id) | (Match.away_team_id == team_id))
|
|
||||||
.order_by(Match.match_date.desc())
|
|
||||||
.limit(limit)
|
|
||||||
)
|
|
||||||
if before is not None:
|
|
||||||
stmt = stmt.where(Match.match_date < before)
|
|
||||||
result = await db.execute(stmt)
|
|
||||||
return list(result.scalars().all())
|
|
||||||
@@ -1,89 +0,0 @@
|
|||||||
"""E - 历史交锋切片: 交手史与胜负规律(h2h)。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import selectinload
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.db.models import Match
|
|
||||||
from src.llm.slices.common import MatchHeader, SliceResult
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
|
|
||||||
async def h2h_slice(header: MatchHeader, *, limit: int = 8, before=None, db: AsyncSession | None = None) -> SliceResult:
|
|
||||||
"""E - 历史交锋切片: 过去数年 + 近期交手数据,提取交手规律。before=match_date 用于回测。
|
|
||||||
|
|
||||||
db: 可选共享 session,避免每个切片独立建连(见 context_builder 模块 docstring)。
|
|
||||||
"""
|
|
||||||
if db is not None:
|
|
||||||
h2h = await _get_h2h(db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
|
||||||
else:
|
|
||||||
async with AsyncSessionLocal() as new_db:
|
|
||||||
h2h = await _get_h2h(new_db, header.home_team_id, header.away_team_id, before=before, limit=limit)
|
|
||||||
lines = [f"── 历史交锋(近 {limit} 次) ──"]
|
|
||||||
n_with_score = 0
|
|
||||||
if h2h:
|
|
||||||
# 从当前主队视角统计:判断当前主队在每场交锋中是主是客
|
|
||||||
current_home_wins = current_home_draws = current_home_losses = 0
|
|
||||||
for hm in h2h:
|
|
||||||
d = hm.match_date.strftime("%Y-%m") if hm.match_date else "?"
|
|
||||||
if hm.home_goals is not None:
|
|
||||||
n_with_score += 1
|
|
||||||
# 判断当前主队当时是主队还是客队
|
|
||||||
if hm.home_team_id == header.home_team_id:
|
|
||||||
# 当前主队当时是主队
|
|
||||||
if hm.home_goals > hm.away_goals:
|
|
||||||
current_home_wins += 1
|
|
||||||
elif hm.home_goals == hm.away_goals:
|
|
||||||
current_home_draws += 1
|
|
||||||
else:
|
|
||||||
current_home_losses += 1
|
|
||||||
else:
|
|
||||||
# 当前主队当时是客队(从客队视角看赛果)
|
|
||||||
if hm.away_goals > hm.home_goals:
|
|
||||||
current_home_wins += 1
|
|
||||||
elif hm.away_goals == hm.home_goals:
|
|
||||||
current_home_draws += 1
|
|
||||||
else:
|
|
||||||
current_home_losses += 1
|
|
||||||
lines.append(f" {d}: {hm.home_team.name} {hm.home_goals}-{hm.away_goals} {hm.away_team.name}")
|
|
||||||
else:
|
|
||||||
lines.append(f" {d}: {hm.home_team.name} vs {hm.away_team.name} (无比分)")
|
|
||||||
total = current_home_wins + current_home_draws + current_home_losses
|
|
||||||
if total:
|
|
||||||
lines.append(
|
|
||||||
f" 总计 {total} 场(从当前主队 {header.home_name} 视角): "
|
|
||||||
f"{current_home_wins}胜 {current_home_draws}平 {current_home_losses}负"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
lines.append(" 无数据")
|
|
||||||
# has_data 以「有比分的交锋」为准:仅有对阵无比分时不足以支撑分析
|
|
||||||
return SliceResult(text="\n".join(lines), has_data=n_with_score > 0, n_records=n_with_score)
|
|
||||||
|
|
||||||
|
|
||||||
async def _get_h2h(db, home_id: int, away_id: int, before, *, limit: int = 5) -> list[Match]:
|
|
||||||
"""两队交锋史。需预加载 home_team / away_team(切片输出队名)。"""
|
|
||||||
stmt = (
|
|
||||||
select(Match)
|
|
||||||
.options(
|
|
||||||
selectinload(Match.home_team),
|
|
||||||
selectinload(Match.away_team),
|
|
||||||
)
|
|
||||||
.where(Match.match_status == "finished")
|
|
||||||
.where(Match.score_status == "known")
|
|
||||||
.where(Match.home_goals.is_not(None))
|
|
||||||
.where(
|
|
||||||
((Match.home_team_id == home_id) & (Match.away_team_id == away_id))
|
|
||||||
| ((Match.home_team_id == away_id) & (Match.away_team_id == home_id))
|
|
||||||
)
|
|
||||||
.order_by(Match.match_date.desc())
|
|
||||||
.limit(limit)
|
|
||||||
)
|
|
||||||
if before is not None:
|
|
||||||
stmt = stmt.where(Match.match_date < before)
|
|
||||||
result = await db.execute(stmt)
|
|
||||||
return list(result.scalars().all())
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
"""C - 主客因素切片: 主场战绩 vs 客场战绩(home_away)。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import selectinload
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.db.models import Match
|
|
||||||
from src.llm.slices.common import MatchHeader, SliceResult, _outcome
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
|
|
||||||
async def home_away_slice(header: MatchHeader, *, limit: int = 10, before=None, db: AsyncSession | None = None) -> SliceResult:
|
|
||||||
"""C - 主客因素切片: 主场战绩 vs 客场战绩,评估地理优势影响。before=match_date 用于回测。
|
|
||||||
|
|
||||||
db: 可选共享 session,避免每个切片独立建连(见 context_builder 模块 docstring)。
|
|
||||||
"""
|
|
||||||
if db is not None:
|
|
||||||
home_home = await _get_home_away(db, header.home_team_id, "home", before=before, limit=limit)
|
|
||||||
away_away = await _get_home_away(db, header.away_team_id, "away", before=before, limit=limit)
|
|
||||||
else:
|
|
||||||
async with AsyncSessionLocal() as new_db:
|
|
||||||
home_home = await _get_home_away(new_db, header.home_team_id, "home", before=before, limit=limit)
|
|
||||||
away_away = await _get_home_away(new_db, header.away_team_id, "away", before=before, limit=limit)
|
|
||||||
lines = ["── 主客因素 ──"]
|
|
||||||
n_total = 0
|
|
||||||
for label, name, matches, side in (
|
|
||||||
("主队主场", header.home_name, home_home, "home"),
|
|
||||||
("客队客场", header.away_name, away_away, "away"),
|
|
||||||
):
|
|
||||||
if matches:
|
|
||||||
wins = draws = losses = gf = ga = 0
|
|
||||||
for m in matches:
|
|
||||||
if m.home_goals is None: continue
|
|
||||||
o = _outcome(m.home_goals, m.away_goals, side)
|
|
||||||
if o == "W": wins += 1
|
|
||||||
elif o == "D": draws += 1
|
|
||||||
else: losses += 1
|
|
||||||
gf += m.home_goals if side == "home" else m.away_goals
|
|
||||||
ga += m.away_goals if side == "home" else m.home_goals
|
|
||||||
n = wins + draws + losses
|
|
||||||
n_total += n
|
|
||||||
if n > 0:
|
|
||||||
pct = wins / n * 100
|
|
||||||
lines.append(f" {label} {name}(近 {n} 场): {wins}胜 {draws}平 {losses}负, 胜率 {pct:.0f}%")
|
|
||||||
lines.append(f" 场均进球 {gf/n:.2f}, 场均失球 {ga/n:.2f}")
|
|
||||||
else:
|
|
||||||
lines.append(f" {label} {name}: 无比分数据")
|
|
||||||
else:
|
|
||||||
lines.append(f" {label} {name}: 无数据")
|
|
||||||
return SliceResult(text="\n".join(lines), has_data=n_total > 0, n_records=n_total)
|
|
||||||
|
|
||||||
|
|
||||||
async def _get_home_away(db, team_id: int, side: str, before, *, limit: int = 10) -> list[Match]:
|
|
||||||
"""某队主场/客场近 N 场。side='home' 取主场,'away' 取客场。
|
|
||||||
|
|
||||||
当前只用标量字段,但统一预加载以免后续扩展时踩坑。
|
|
||||||
"""
|
|
||||||
stmt = (
|
|
||||||
select(Match)
|
|
||||||
.options(
|
|
||||||
selectinload(Match.stats),
|
|
||||||
selectinload(Match.home_team),
|
|
||||||
selectinload(Match.away_team),
|
|
||||||
)
|
|
||||||
.where(Match.match_status == "finished")
|
|
||||||
.where(Match.score_status == "known")
|
|
||||||
.where(Match.home_goals.is_not(None))
|
|
||||||
.order_by(Match.match_date.desc())
|
|
||||||
.limit(limit)
|
|
||||||
)
|
|
||||||
if side == "home":
|
|
||||||
stmt = stmt.where(Match.home_team_id == team_id)
|
|
||||||
else:
|
|
||||||
stmt = stmt.where(Match.away_team_id == team_id)
|
|
||||||
if before is not None:
|
|
||||||
stmt = stmt.where(Match.match_date < before)
|
|
||||||
result = await db.execute(stmt)
|
|
||||||
return list(result.scalars().all())
|
|
||||||
@@ -1,91 +0,0 @@
|
|||||||
"""D - 联赛排名切片: 积分榜快照(支持 cutoff 的历史还原,standings)。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.orm import selectinload
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.db.models import League, Standing
|
|
||||||
from src.llm.slices.common import MatchHeader, SliceResult
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
|
|
||||||
async def standings_slice(header: MatchHeader, *, before=None, db: AsyncSession | None = None) -> SliceResult:
|
|
||||||
"""D - 联赛排名切片: 两队积分榜位置、积分、近期走势(form)、分区,评估整体实力差距。
|
|
||||||
|
|
||||||
P0-02: 支持 cutoff(before)。取 available_at<=cutoff 的每队最新快照
|
|
||||||
(DISTINCT ON team_id ORDER available_at DESC);before=None 时 cutoff=now()。
|
|
||||||
回测时可还原历史时刻榜单,不再只是"最新快照、忽略 cutoff"。
|
|
||||||
|
|
||||||
db: 可选共享 session(见 context_builder 模块 docstring)。
|
|
||||||
|
|
||||||
语义区分:
|
|
||||||
- 两队都有积分榜行 → has_data=True(明确的排名信息)
|
|
||||||
- 任一队缺失 → has_data=False(升班马/杯赛无榜,信息不完整时明确声明)
|
|
||||||
"""
|
|
||||||
# P0-02: before=None → cutoff=now()(取最新可用快照)
|
|
||||||
if before is None:
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
before = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
if db is not None:
|
|
||||||
league = (await db.execute(select(League).where(League.id == header.league_id))).scalar_one_or_none()
|
|
||||||
# P0-02: DISTINCT ON (team_id) 取 available_at<=cutoff 的最新快照
|
|
||||||
rows = (
|
|
||||||
(
|
|
||||||
await db.execute(
|
|
||||||
select(Standing)
|
|
||||||
.options(selectinload(Standing.team))
|
|
||||||
.where(Standing.league_id == header.league_id)
|
|
||||||
.where(Standing.available_at <= before)
|
|
||||||
.distinct(Standing.team_id)
|
|
||||||
.order_by(Standing.team_id, Standing.available_at.desc())
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.scalars()
|
|
||||||
.all()
|
|
||||||
if league
|
|
||||||
else []
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
async with AsyncSessionLocal() as new_db:
|
|
||||||
return await standings_slice(header, before=before, db=new_db)
|
|
||||||
|
|
||||||
lines = [f"── 联赛排名({header.league_name} 共 {len(rows)} 队) ──"]
|
|
||||||
n_records = 0
|
|
||||||
|
|
||||||
def _fmt(row) -> str:
|
|
||||||
zg = f" xG差 {row.xgd:+.1f}" if row.xg_for is not None and row.xg_against is not None and row.goal_diff is not None else ""
|
|
||||||
form = f" 近5场 {row.form}" if row.form else ""
|
|
||||||
zone = f" [{row.zone}]" if row.zone else ""
|
|
||||||
return (
|
|
||||||
f" 第 {row.position} 名: {row.points} 分 / {row.played} 场 "
|
|
||||||
f"({row.won}胜{row.drawn}平{row.lost}负, 进{row.goals_for}失{row.goals_against} 净胜{row.goal_diff:+d}"
|
|
||||||
f"{zg}){form}{zone}"
|
|
||||||
)
|
|
||||||
|
|
||||||
for label, team_id in (("主队", header.home_team_id), ("客队", header.away_team_id)):
|
|
||||||
row = next((r for r in rows if r.team_id == team_id), None)
|
|
||||||
if row is None:
|
|
||||||
lines.append(f" {label}: 暂无积分榜数据(可能杯赛/赛季未开始)")
|
|
||||||
else:
|
|
||||||
n_records += 1
|
|
||||||
lines.append(f" {label} {header.home_name if label == '主队' else header.away_name}:")
|
|
||||||
lines.append(_fmt(row))
|
|
||||||
|
|
||||||
# 两队排名对比摘要
|
|
||||||
home_row = next((r for r in rows if r.team_id == header.home_team_id), None)
|
|
||||||
away_row = next((r for r in rows if r.team_id == header.away_team_id), None)
|
|
||||||
if home_row and away_row:
|
|
||||||
diff = home_row.position - away_row.position # 正数=主队排名更靠前(名次更小)
|
|
||||||
lead = f"主队排名高 {diff} 位" if diff > 0 else (f"客队排名高 {-diff} 位" if diff < 0 else "两队同排名结构")
|
|
||||||
pts_diff = home_row.points - away_row.points
|
|
||||||
lines.append(f" 排名对比: {lead}, 分差 {pts_diff:+d}")
|
|
||||||
|
|
||||||
# has_data: 两队都有行才算完整;只有一队时仍有价值,但标记不完整
|
|
||||||
has_data = n_records >= 1
|
|
||||||
return SliceResult(text="\n".join(lines), has_data=has_data, n_records=n_records)
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
"""B - 攻防数据切片: 进球/射门/控球/xG 聚合(stats)。"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from typing import TYPE_CHECKING
|
|
||||||
|
|
||||||
from src.db.base import AsyncSessionLocal
|
|
||||||
from src.llm.slices.common import MatchHeader, SliceResult, _is_stats_available
|
|
||||||
from src.llm.slices.form import _get_form
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
|
||||||
|
|
||||||
|
|
||||||
async def stats_slice(header: MatchHeader, *, limit: int = 10, before=None, db: AsyncSession | None = None) -> SliceResult:
|
|
||||||
"""B - 攻防数据切片: 进球、射门、控球,评估攻防强度。before=match_date 用于回测。
|
|
||||||
|
|
||||||
db: 可选共享 session,避免每个切片独立建连(见 context_builder 模块 docstring)。
|
|
||||||
"""
|
|
||||||
if db is not None:
|
|
||||||
home_form = await _get_form(db, header.home_team_id, before=before, limit=limit)
|
|
||||||
away_form = await _get_form(db, header.away_team_id, before=before, limit=limit)
|
|
||||||
else:
|
|
||||||
async with AsyncSessionLocal() as new_db:
|
|
||||||
home_form = await _get_form(new_db, header.home_team_id, before=before, limit=limit)
|
|
||||||
away_form = await _get_form(new_db, header.away_team_id, before=before, limit=limit)
|
|
||||||
lines = [f"── 攻防数据(近 {limit} 场) ──"]
|
|
||||||
n_total = 0
|
|
||||||
# P0-1 修复:每场历史比赛必须根据「该队当时是主是客」判断 side,
|
|
||||||
# 不能用本场 side 硬套 —— 否则进球/失球/xG 全部算反。
|
|
||||||
for label, name, form, team_id in (
|
|
||||||
("主队", header.home_name, home_form, header.home_team_id),
|
|
||||||
("客队", header.away_name, away_form, header.away_team_id),
|
|
||||||
):
|
|
||||||
if form:
|
|
||||||
gf = ga = shots = sot = poss = xg = xga = 0
|
|
||||||
n = n_shots = n_poss = n_xg = 0
|
|
||||||
for fm in form:
|
|
||||||
if fm.home_goals is None: continue
|
|
||||||
is_home = (fm.home_team_id == team_id)
|
|
||||||
gf += fm.home_goals if is_home else fm.away_goals
|
|
||||||
ga += fm.away_goals if is_home else fm.home_goals
|
|
||||||
n += 1
|
|
||||||
# 只使用 cutoff 之前已可用的统计数据
|
|
||||||
if fm.stats and _is_stats_available(fm.stats, before):
|
|
||||||
if fm.stats.home_shots is not None:
|
|
||||||
shots += fm.stats.home_shots if is_home else fm.stats.away_shots
|
|
||||||
sot += fm.stats.home_shots_on_target if is_home else fm.stats.away_shots_on_target
|
|
||||||
n_shots += 1
|
|
||||||
if fm.stats.home_possession is not None:
|
|
||||||
poss += fm.stats.home_possession if is_home else (100 - fm.stats.home_possession)
|
|
||||||
n_poss += 1
|
|
||||||
if fm.stats.home_xg is not None:
|
|
||||||
xg += fm.stats.home_xg if is_home else fm.stats.away_xg
|
|
||||||
xga += fm.stats.away_xg if is_home else fm.stats.home_xg
|
|
||||||
n_xg += 1
|
|
||||||
n_total += n
|
|
||||||
if n > 0:
|
|
||||||
lines.append(f" {label} {name}:")
|
|
||||||
lines.append(f" 场均进球 {gf/n:.2f}, 场均失球 {ga/n:.2f}")
|
|
||||||
if n_shots: lines.append(f" 场均射门 {shots/n_shots:.1f}, 射正 {sot/n_shots:.1f}")
|
|
||||||
if n_poss: lines.append(f" 平均控球 {poss/n_poss:.1f}%")
|
|
||||||
if n_xg: lines.append(f" 场均 xG {xg/n_xg:.2f}, 场均被 xG {xga/n_xg:.2f}")
|
|
||||||
else:
|
|
||||||
lines.append(f" {label} {name}: 无比分数据")
|
|
||||||
else:
|
|
||||||
lines.append(f" {label} {name}: 无数据")
|
|
||||||
return SliceResult(text="\n".join(lines), has_data=n_total > 0, n_records=n_total)
|
|
||||||
+9
-1
@@ -3,9 +3,17 @@ from __future__ import annotations
|
|||||||
|
|
||||||
|
|
||||||
def actual_1x2(home: int, away: int) -> str:
|
def actual_1x2(home: int, away: int) -> str:
|
||||||
"""实际比分 → 胜平负(单一权威源:backtest.py 与 eval.py 共用)。"""
|
"""实际比分 → 胜平负。
|
||||||
|
|
||||||
|
单一权威源: backtest.py 和 eval.py 共用,避免重复定义。
|
||||||
|
"""
|
||||||
if home > away:
|
if home > away:
|
||||||
return "1"
|
return "1"
|
||||||
if home == away:
|
if home == away:
|
||||||
return "X"
|
return "X"
|
||||||
return "2"
|
return "2"
|
||||||
|
|
||||||
|
|
||||||
|
def is_correct_1x2(pred: str | None, actual: str) -> bool:
|
||||||
|
"""预测是否命中胜平负。"""
|
||||||
|
return pred == actual
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ class TestOrchestratorWritesAgentWeights:
|
|||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_orchestrator_writes_agent_weights_to_upsert(self):
|
async def test_orchestrator_writes_agent_weights_to_upsert(self):
|
||||||
"""orchestrator 应将 agent_weights 传入 _insert_or_find_by_fingerprint。"""
|
"""orchestrator 应将 agent_weights 传入 _upsert_prediction。"""
|
||||||
from src.llm.agents import orchestrator as orch_mod
|
from src.llm.agents import orchestrator as orch_mod
|
||||||
from src.llm.agents.base import AgentReport
|
from src.llm.agents.base import AgentReport
|
||||||
from src.llm.context_builder import MatchHeader
|
from src.llm.context_builder import MatchHeader
|
||||||
@@ -103,8 +103,8 @@ class TestOrchestratorWritesAgentWeights:
|
|||||||
"agent_weights": {"form": 0.3, "home_away": 0.5, "stats": 0.2},
|
"agent_weights": {"form": 0.3, "home_away": 0.5, "stats": 0.2},
|
||||||
}, 100, 50
|
}, 100, 50
|
||||||
|
|
||||||
async def mock_upsert(session, *, values):
|
async def mock_upsert(session, **kw):
|
||||||
captured_values.update(values)
|
captured_values.update(kw.get("values", {}))
|
||||||
p = MagicMock()
|
p = MagicMock()
|
||||||
p.id = 1
|
p.id = 1
|
||||||
p.provider = "test"
|
p.provider = "test"
|
||||||
@@ -116,7 +116,7 @@ class TestOrchestratorWritesAgentWeights:
|
|||||||
p.subjective_confidence = 0.7
|
p.subjective_confidence = 0.7
|
||||||
p.reasoning = "test"
|
p.reasoning = "test"
|
||||||
p.agent_outputs = []
|
p.agent_outputs = []
|
||||||
p.agent_weights = values.get("agent_weights")
|
p.agent_weights = kw["values"].get("agent_weights")
|
||||||
return p
|
return p
|
||||||
|
|
||||||
class FakeUow:
|
class FakeUow:
|
||||||
@@ -130,14 +130,14 @@ class TestOrchestratorWritesAgentWeights:
|
|||||||
with patch.object(orch_mod, "run_specialists", mock_specialists), \
|
with patch.object(orch_mod, "run_specialists", mock_specialists), \
|
||||||
patch.object(orch_mod, "_agent_provider", mock_provider), \
|
patch.object(orch_mod, "_agent_provider", mock_provider), \
|
||||||
patch.object(orch_mod, "load_match_header", mock_header), \
|
patch.object(orch_mod, "load_match_header", mock_header), \
|
||||||
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
||||||
patch.object(orch_mod, "run_aggregator", mock_aggregator), \
|
patch.object(orch_mod, "run_aggregator", mock_aggregator), \
|
||||||
patch.object(orch_mod, "get_uow", FakeUow):
|
patch.object(orch_mod, "get_uow", FakeUow):
|
||||||
|
|
||||||
result = await orch_mod.predict_match_multi(999)
|
result = await orch_mod.predict_match_multi(999)
|
||||||
|
|
||||||
# 断言 agent_weights 被写入
|
# 断言 agent_weights 被写入
|
||||||
assert "agent_weights" in captured_values, "agent_weights 应传入 _insert_or_find_by_fingerprint"
|
assert "agent_weights" in captured_values, "agent_weights 应传入 _upsert_prediction"
|
||||||
assert captured_values["agent_weights"] is not None, "agent_weights 不应为 None"
|
assert captured_values["agent_weights"] is not None, "agent_weights 不应为 None"
|
||||||
assert "form" in captured_values["agent_weights"], "agent_weights 应包含专家权重"
|
assert "form" in captured_values["agent_weights"], "agent_weights 应包含专家权重"
|
||||||
print(f"PASS: agent_weights = {captured_values['agent_weights']}")
|
print(f"PASS: agent_weights = {captured_values['agent_weights']}")
|
||||||
|
|||||||
@@ -79,18 +79,18 @@ class TestWriteBufferStrategy:
|
|||||||
def test_bzzoirot_new_match_available_at_is_two_hours_after_kickoff(self):
|
def test_bzzoirot_new_match_available_at_is_two_hours_after_kickoff(self):
|
||||||
"""bzzoiro 新建比赛(stats 回填创建 MatchStats)时 available_at 应为开球 + 2 小时。"""
|
"""bzzoiro 新建比赛(stats 回填创建 MatchStats)时 available_at 应为开球 + 2 小时。"""
|
||||||
import inspect
|
import inspect
|
||||||
from src.data import bzzoiro_stats
|
from src.data import bzzoiro
|
||||||
|
|
||||||
source = inspect.getsource(bzzoiro_stats)
|
source = inspect.getsource(bzzoiro)
|
||||||
assert 'timedelta(hours=2)' in source, \
|
assert 'timedelta(hours=2)' in source, \
|
||||||
"bzzoiro 应使用 match_date + timedelta(hours=2) 作为 available_at"
|
"bzzoiro 应使用 match_date + timedelta(hours=2) 作为 available_at"
|
||||||
|
|
||||||
def test_bzzoirot_multiple_writes_use_two_hour_buffer(self):
|
def test_bzzoirot_multiple_writes_use_two_hour_buffer(self):
|
||||||
"""bzzoiro 多处写入(创建/更新)都应使用 2 小时缓冲。"""
|
"""bzzoiro 多处写入(创建/更新)都应使用 2 小时缓冲。"""
|
||||||
import inspect
|
import inspect
|
||||||
from src.data import bzzoiro_stats
|
from src.data import bzzoiro
|
||||||
|
|
||||||
source = inspect.getsource(bzzoiro_stats)
|
source = inspect.getsource(bzzoiro)
|
||||||
count = source.count('timedelta(hours=2)')
|
count = source.count('timedelta(hours=2)')
|
||||||
assert count >= 2, f"期望至少 2 处 timedelta(hours=2),实际 {count} 处"
|
assert count >= 2, f"期望至少 2 处 timedelta(hours=2),实际 {count} 处"
|
||||||
|
|
||||||
|
|||||||
+2
-29
@@ -5,7 +5,6 @@
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from types import SimpleNamespace
|
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -13,24 +12,6 @@ import pytest
|
|||||||
from src.llm.baseline import _avg_goals, predict_baseline
|
from src.llm.baseline import _avg_goals, predict_baseline
|
||||||
|
|
||||||
|
|
||||||
class _FakeUoW:
|
|
||||||
"""P3-2:baseline 在服务层落库,测试需 mock get_uow。"""
|
|
||||||
|
|
||||||
async def __aenter__(self):
|
|
||||||
return SimpleNamespace(
|
|
||||||
execute=lambda *a, **k: SimpleNamespace(scalar_one_or_none=lambda: None),
|
|
||||||
add=lambda *a, **k: None,
|
|
||||||
flush=lambda *a, **k: None,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def __aexit__(self, *a):
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
async def _fake_upsert(session, **kw):
|
|
||||||
return SimpleNamespace(id=1)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_avg_goals_no_data_returns_zero():
|
async def test_avg_goals_no_data_returns_zero():
|
||||||
"""无历史数据时场均进球为 0(不抛异常)。"""
|
"""无历史数据时场均进球为 0(不抛异常)。"""
|
||||||
@@ -87,9 +68,7 @@ async def test_predict_baseline_no_llm():
|
|||||||
match_status = "scheduled"
|
match_status = "scheduled"
|
||||||
|
|
||||||
with patch("src.llm.baseline._avg_goals", fake_avg), \
|
with patch("src.llm.baseline._avg_goals", fake_avg), \
|
||||||
patch("src.llm.baseline.AsyncSessionLocal") as SLC, \
|
patch("src.llm.baseline.AsyncSessionLocal") as SLC:
|
||||||
patch("src.db.unit_of_work.get_uow", _FakeUoW), \
|
|
||||||
patch("src.llm.baseline._insert_or_find_by_fingerprint", _fake_upsert):
|
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
async def get(self, cls, mid):
|
async def get(self, cls, mid):
|
||||||
return FakeMatch()
|
return FakeMatch()
|
||||||
@@ -114,8 +93,6 @@ async def test_predict_baseline_no_llm():
|
|||||||
assert result.pred_1x2 == "X"
|
assert result.pred_1x2 == "X"
|
||||||
assert result.subjective_confidence == 0.5
|
assert result.subjective_confidence == 0.5
|
||||||
assert "非投注建议" in result.reasoning
|
assert "非投注建议" in result.reasoning
|
||||||
# P3-2:服务层落库,回填真实 prediction_id
|
|
||||||
assert result.prediction_id == 1
|
|
||||||
# 确认未调用任何 LLM 相关模块
|
# 确认未调用任何 LLM 相关模块
|
||||||
assert "home_10" in captured and "away_20" in captured
|
assert "home_10" in captured and "away_20" in captured
|
||||||
|
|
||||||
@@ -135,9 +112,7 @@ async def test_predict_baseline_clamps_to_range():
|
|||||||
match_status = "scheduled"
|
match_status = "scheduled"
|
||||||
|
|
||||||
with patch("src.llm.baseline._avg_goals", fake_avg), \
|
with patch("src.llm.baseline._avg_goals", fake_avg), \
|
||||||
patch("src.llm.baseline.AsyncSessionLocal") as SLC, \
|
patch("src.llm.baseline.AsyncSessionLocal") as SLC:
|
||||||
patch("src.db.unit_of_work.get_uow", _FakeUoW), \
|
|
||||||
patch("src.llm.baseline._insert_or_find_by_fingerprint", _fake_upsert):
|
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
async def get(self, cls, mid):
|
async def get(self, cls, mid):
|
||||||
return FakeMatch()
|
return FakeMatch()
|
||||||
@@ -153,5 +128,3 @@ async def test_predict_baseline_clamps_to_range():
|
|||||||
assert result.pred_home_goals == 10.0 # clamped
|
assert result.pred_home_goals == 10.0 # clamped
|
||||||
assert result.pred_away_goals == 0.0 # clamped
|
assert result.pred_away_goals == 0.0 # clamped
|
||||||
assert result.pred_1x2 == "1" # 10:0 主胜
|
assert result.pred_1x2 == "1" # 10:0 主胜
|
||||||
# P3-2:服务层落库,回填真实 prediction_id
|
|
||||||
assert result.prediction_id == 1
|
|
||||||
|
|||||||
@@ -62,35 +62,12 @@ async def test_predict_baseline_returns_predict_result():
|
|||||||
async def __aexit__(self, *a):
|
async def __aexit__(self, *a):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# P3-2:baseline 在服务层落库(get_uow + _insert_or_find_by_fingerprint),需 mock 掉。
|
|
||||||
class FakeUoW:
|
|
||||||
async def __aenter__(self):
|
|
||||||
return _make_session()
|
|
||||||
|
|
||||||
async def __aexit__(self, *a):
|
|
||||||
return None
|
|
||||||
|
|
||||||
captured = {}
|
|
||||||
|
|
||||||
async def fake_upsert(session, *, values):
|
|
||||||
captured.update(values)
|
|
||||||
return SimpleNamespace(id=77)
|
|
||||||
|
|
||||||
# baseline.py 内部 from-import get_uow / _insert_or_find_by_fingerprint,需 patch 真实来源模块。
|
|
||||||
with patch("src.llm.baseline._avg_goals", fake_avg), \
|
with patch("src.llm.baseline._avg_goals", fake_avg), \
|
||||||
patch("src.llm.baseline.AsyncSessionLocal") as SLC, \
|
patch("src.llm.baseline.AsyncSessionLocal") as SLC:
|
||||||
patch("src.db.unit_of_work.get_uow", FakeUoW), \
|
|
||||||
patch("src.llm.baseline._insert_or_find_by_fingerprint", fake_upsert):
|
|
||||||
SLC.return_value = FakeCM()
|
SLC.return_value = FakeCM()
|
||||||
|
|
||||||
result = await predict_baseline(1)
|
result = await predict_baseline(1)
|
||||||
|
|
||||||
# P3-2:验证服务层落库被调用且属性映射正确
|
|
||||||
assert captured["match_id"] == 1
|
|
||||||
assert captured["provider"] == "baseline"
|
|
||||||
assert captured["run_type"] == "live"
|
|
||||||
assert captured["pred_home_goals"] == 2.0
|
|
||||||
|
|
||||||
assert isinstance(result, PredictResult)
|
assert isinstance(result, PredictResult)
|
||||||
assert result.mode == "baseline"
|
assert result.mode == "baseline"
|
||||||
assert result.provider == "baseline"
|
assert result.provider == "baseline"
|
||||||
@@ -167,31 +144,15 @@ def test_predict_route_has_no_dict_branch():
|
|||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
# 4. P3-2:baseline 服务层落库属性映射(落库已从路由移到 baseline.py)
|
# 4. _persist_baseline 属性映射(baseline 落库语义不变)
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|
||||||
|
|
||||||
class _FakeResult:
|
|
||||||
"""支持 .scalar_one_or_none() 的最小假结果集。"""
|
|
||||||
|
|
||||||
def __init__(self, items):
|
|
||||||
self._items = list(items)
|
|
||||||
|
|
||||||
def scalars(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def all(self):
|
|
||||||
return self._items
|
|
||||||
|
|
||||||
def scalar_one_or_none(self):
|
|
||||||
return self._items[0] if self._items else None
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeUoW:
|
class _FakeUoW:
|
||||||
"""替代 get_uow 的最小上下文管理器(session.execute 是 async 的)。"""
|
"""替代 get_uow 的最小上下文管理器。"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.session = _make_session()
|
self.session = SimpleNamespace()
|
||||||
|
|
||||||
async def __aenter__(self):
|
async def __aenter__(self):
|
||||||
return self.session
|
return self.session
|
||||||
@@ -199,101 +160,57 @@ class _FakeUoW:
|
|||||||
async def __aexit__(self, *a):
|
async def __aexit__(self, *a):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def __call__(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
|
|
||||||
def _make_session(existing=None):
|
|
||||||
"""构造带 async execute / add / flush 的假 session。"""
|
|
||||||
sess = SimpleNamespace()
|
|
||||||
|
|
||||||
async def execute(*a, **k):
|
|
||||||
return _FakeResult(existing or [])
|
|
||||||
|
|
||||||
sess.execute = execute
|
|
||||||
sess.add = lambda *a, **k: None
|
|
||||||
|
|
||||||
async def flush(*a, **k):
|
|
||||||
return None
|
|
||||||
|
|
||||||
sess.flush = flush
|
|
||||||
return sess
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_baseline_service_persists_with_correct_attributes(monkeypatch):
|
async def test_persist_baseline_maps_attributes(monkeypatch):
|
||||||
"""P3-2:baseline 在服务层(predict_baseline)落库,属性映射与路由旧版一致。"""
|
|
||||||
captured = {}
|
captured = {}
|
||||||
|
|
||||||
async def fake_upsert(session, *, values):
|
async def fake_upsert(session, **kwargs):
|
||||||
captured.update(values)
|
captured.update(kwargs)
|
||||||
return SimpleNamespace(id=77)
|
return SimpleNamespace(id=77)
|
||||||
|
|
||||||
class FakeMatch:
|
|
||||||
id = 1
|
|
||||||
home_team_id = 10
|
|
||||||
away_team_id = 20
|
|
||||||
league_id = 1
|
|
||||||
match_status = "scheduled"
|
|
||||||
|
|
||||||
class FakeSession:
|
|
||||||
async def get(self, cls, mid):
|
|
||||||
return FakeMatch()
|
|
||||||
|
|
||||||
class FakeSLC:
|
|
||||||
async def __aenter__(self):
|
|
||||||
return FakeSession()
|
|
||||||
|
|
||||||
async def __aexit__(self, *a):
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def fake_avg(db, *, team_id, side, league_id, before):
|
|
||||||
return 2.0 if side == "home" else 1.0
|
|
||||||
|
|
||||||
monkeypatch.setattr("src.llm.baseline._avg_goals", fake_avg)
|
|
||||||
monkeypatch.setattr("src.llm.baseline.AsyncSessionLocal", FakeSLC)
|
|
||||||
monkeypatch.setattr("src.db.unit_of_work.get_uow", lambda: _FakeUoW())
|
monkeypatch.setattr("src.db.unit_of_work.get_uow", lambda: _FakeUoW())
|
||||||
# baseline.py 模块级 import _insert_or_find_by_fingerprint(第 15 行),需 patch baseline 模块属性
|
monkeypatch.setattr("src.llm.predict._upsert_prediction", fake_upsert)
|
||||||
monkeypatch.setattr("src.llm.baseline._insert_or_find_by_fingerprint", fake_upsert)
|
|
||||||
|
|
||||||
result = await predict_baseline(1)
|
from src.api.routes.predict import _persist_baseline
|
||||||
|
|
||||||
# 落库被调用且属性映射正确
|
baseline = PredictResult(
|
||||||
assert captured, f"predict_baseline 应调用 _insert_or_find_by_fingerprint 落库,但 captured 为空(result.prediction_id={result.prediction_id!r})"
|
prediction_id=0, # baseline 不在服务层落库,由 _persist_baseline 落库后取得真实 id
|
||||||
|
provider="baseline",
|
||||||
|
model="baseline",
|
||||||
|
prompt_version="baseline_v1",
|
||||||
|
mode="baseline",
|
||||||
|
pred_home_goals=2.0,
|
||||||
|
pred_away_goals=1.0,
|
||||||
|
alt_pred_home_goals=None,
|
||||||
|
alt_pred_away_goals=None,
|
||||||
|
pred_1x2="1",
|
||||||
|
subjective_confidence=0.5,
|
||||||
|
reasoning="r",
|
||||||
|
context="",
|
||||||
|
status="success",
|
||||||
|
latency_ms=0,
|
||||||
|
prompt_tokens=0,
|
||||||
|
completion_tokens=0,
|
||||||
|
raw={"home_avg": 2.1, "away_avg": 1.4},
|
||||||
|
)
|
||||||
|
|
||||||
|
pid = await _persist_baseline(1, baseline)
|
||||||
|
|
||||||
|
assert pid == 77
|
||||||
assert captured["match_id"] == 1
|
assert captured["match_id"] == 1
|
||||||
assert captured["provider"] == "baseline"
|
assert captured["provider_name"] == "baseline"
|
||||||
assert captured["model"] == "baseline"
|
assert captured["model"] == "baseline"
|
||||||
assert captured["mode"] == "baseline"
|
assert captured["mode"] == "baseline"
|
||||||
assert captured["run_type"] == "live"
|
assert captured["run_type"] == "live"
|
||||||
assert captured["prompt_version"] == "baseline_v1"
|
v = captured["values"]
|
||||||
assert captured["pred_home_goals"] == 2.0
|
assert v["prompt_version"] == "baseline_v1"
|
||||||
assert captured["pred_away_goals"] == 1.0
|
assert v["pred_home_goals"] == 2.0
|
||||||
assert captured["pred_1x2"] == "1"
|
assert v["pred_away_goals"] == 1.0
|
||||||
assert captured["subjective_confidence"] == 0.5
|
assert v["pred_1x2"] == "1"
|
||||||
assert captured["prompt_tokens"] == 0
|
assert v["subjective_confidence"] == 0.5
|
||||||
assert captured["completion_tokens"] == 0
|
assert v["prompt_tokens"] == 0
|
||||||
assert captured["latency_ms"] == 0
|
assert v["completion_tokens"] == 0
|
||||||
assert captured["raw_response"] == {"home_avg": 2.0, "away_avg": 1.0}
|
assert v["latency_ms"] == 0
|
||||||
assert captured["status"] == "success"
|
assert v["raw_response"] == {"home_avg": 2.1, "away_avg": 1.4}
|
||||||
|
assert v["status"] == "success"
|
||||||
# 回填真实 prediction_id(服务层落库后取得)
|
|
||||||
assert result.prediction_id == 77
|
|
||||||
assert result.pred_1x2 == "1"
|
|
||||||
assert captured["match_id"] == 1
|
|
||||||
assert captured["provider"] == "baseline"
|
|
||||||
assert captured["model"] == "baseline"
|
|
||||||
assert captured["mode"] == "baseline"
|
|
||||||
assert captured["run_type"] == "live"
|
|
||||||
assert captured["prompt_version"] == "baseline_v1"
|
|
||||||
assert captured["pred_home_goals"] == 2.0
|
|
||||||
assert captured["pred_away_goals"] == 1.0
|
|
||||||
assert captured["pred_1x2"] == "1"
|
|
||||||
assert captured["subjective_confidence"] == 0.5
|
|
||||||
assert captured["prompt_tokens"] == 0
|
|
||||||
assert captured["completion_tokens"] == 0
|
|
||||||
assert captured["latency_ms"] == 0
|
|
||||||
assert captured["raw_response"] == {"home_avg": 2.0, "away_avg": 1.0}
|
|
||||||
assert captured["status"] == "success"
|
|
||||||
|
|
||||||
# 回填真实 prediction_id(服务层落库后取得)
|
|
||||||
assert result.prediction_id == 77
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ from datetime import date, datetime, timezone
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
import src.data.bzzoiro as bz
|
import src.data.bzzoiro as bz
|
||||||
from src.db.models import DataLineage, League, Match, RawEvent, Team, TeamAlias
|
from src.db.models import DataLineage, League, Match, RawEvent, Team
|
||||||
|
|
||||||
|
|
||||||
def _event(eid=1001, status="finished", home="Arsenal", away="Chelsea", hs=2, as_=1):
|
def _event(eid=1001, status="finished", home="Arsenal", away="Chelsea", hs=2, as_=1):
|
||||||
@@ -71,20 +71,11 @@ class _FakeDB:
|
|||||||
League: list(leagues),
|
League: list(leagues),
|
||||||
RawEvent: list(raw_events),
|
RawEvent: list(raw_events),
|
||||||
}
|
}
|
||||||
self._teams_by_id: dict[int, Team] = {t.id: t for t in teams if getattr(t, "id", None)}
|
self._next_id = 0
|
||||||
self._aliases: dict[str, TeamAlias] = {}
|
|
||||||
self._next_id = max((t.id for t in teams if getattr(t, "id", None)), default=0)
|
|
||||||
|
|
||||||
def add(self, obj):
|
def add(self, obj):
|
||||||
self.added.append(obj)
|
self.added.append(obj)
|
||||||
|
|
||||||
async def get(self, cls, key):
|
|
||||||
if cls is Team:
|
|
||||||
return self._teams_by_id.get(key)
|
|
||||||
if cls is TeamAlias:
|
|
||||||
return self._aliases.get(key)
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def execute(self, stmt):
|
async def execute(self, stmt):
|
||||||
entities = set()
|
entities = set()
|
||||||
for d in (stmt.column_descriptions or []):
|
for d in (stmt.column_descriptions or []):
|
||||||
@@ -99,8 +90,6 @@ class _FakeDB:
|
|||||||
if getattr(obj, "id", None) is None:
|
if getattr(obj, "id", None) is None:
|
||||||
self._next_id += 1
|
self._next_id += 1
|
||||||
obj.id = self._next_id
|
obj.id = self._next_id
|
||||||
if isinstance(obj, Team) and obj.id is not None:
|
|
||||||
self._teams_by_id[obj.id] = obj
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
@@ -255,14 +244,11 @@ class TestEventsBronzeOnUpdate:
|
|||||||
|
|
||||||
class TestEventsBronzeIsBestEffort:
|
class TestEventsBronzeIsBestEffort:
|
||||||
async def test_bronze_write_failure_does_not_break_ingest(self, monkeypatch):
|
async def test_bronze_write_failure_does_not_break_ingest(self, monkeypatch):
|
||||||
import src.data.bzzoiro_events as bz_events
|
|
||||||
|
|
||||||
async def _boom(*args, **kwargs):
|
async def _boom(*args, **kwargs):
|
||||||
raise RuntimeError("infra down")
|
raise RuntimeError("infra down")
|
||||||
|
|
||||||
# Bronze 写入助手直接 import 到 bzzoiro_events 命名空间,需 patch 该处
|
monkeypatch.setattr(bz, "_write_raw_event", _boom)
|
||||||
monkeypatch.setattr(bz_events, "_write_raw_event", _boom)
|
monkeypatch.setattr(bz, "_write_lineage", _boom)
|
||||||
monkeypatch.setattr(bz_events, "_write_lineage", _boom)
|
|
||||||
_patch_fetch(monkeypatch, [_event()])
|
_patch_fetch(monkeypatch, [_event()])
|
||||||
db = _FakeDB()
|
db = _FakeDB()
|
||||||
|
|
||||||
|
|||||||
@@ -70,7 +70,7 @@ class TestH2HCurrentHomePerspective:
|
|||||||
_make_h2h_match(101, home_id=2, away_id=1, home_goals=3, away_goals=1,
|
_make_h2h_match(101, home_id=2, away_id=1, home_goals=3, away_goals=1,
|
||||||
home_name="阿森纳", away_name="利物浦"),
|
home_name="阿森纳", away_name="利物浦"),
|
||||||
]
|
]
|
||||||
import src.llm.slices.h2h as cb
|
import src.llm.context_builder as cb
|
||||||
orig = cb._get_h2h
|
orig = cb._get_h2h
|
||||||
async def mock_get_h2h(db, home_id, away_id, before, *, limit):
|
async def mock_get_h2h(db, home_id, away_id, before, *, limit):
|
||||||
return matches
|
return matches
|
||||||
@@ -100,7 +100,7 @@ class TestH2HCurrentHomePerspective:
|
|||||||
_make_h2h_match(201, home_id=1, away_id=2, home_goals=2, away_goals=1,
|
_make_h2h_match(201, home_id=1, away_id=2, home_goals=2, away_goals=1,
|
||||||
home_name="曼城", away_name="诺维奇"),
|
home_name="曼城", away_name="诺维奇"),
|
||||||
]
|
]
|
||||||
import src.llm.slices.h2h as cb
|
import src.llm.context_builder as cb
|
||||||
|
|
||||||
async def mock_get_h2h(db, h, a, before, **kw):
|
async def mock_get_h2h(db, h, a, before, **kw):
|
||||||
# 真实契约是 async(见 context_builder.py 的 `h2h = await _get_h2h(...)`),
|
# 真实契约是 async(见 context_builder.py 的 `h2h = await _get_h2h(...)`),
|
||||||
@@ -122,7 +122,7 @@ class TestH2HCurrentHomePerspective:
|
|||||||
_make_h2h_match(301, home_id=2, away_id=1, home_goals=0, away_goals=2,
|
_make_h2h_match(301, home_id=2, away_id=1, home_goals=0, away_goals=2,
|
||||||
home_name="热刺", away_name="切尔西"), # 切尔西客场 2-0 赢
|
home_name="热刺", away_name="切尔西"), # 切尔西客场 2-0 赢
|
||||||
]
|
]
|
||||||
import src.llm.slices.h2h as cb
|
import src.llm.context_builder as cb
|
||||||
orig = cb._get_h2h
|
orig = cb._get_h2h
|
||||||
async def mock_get_h2h(db, h, a, before, *, limit):
|
async def mock_get_h2h(db, h, a, before, *, limit):
|
||||||
return matches
|
return matches
|
||||||
|
|||||||
@@ -0,0 +1,313 @@
|
|||||||
|
"""采集任务状态(ingest_jobs)测试。
|
||||||
|
|
||||||
|
背景: POST /ingest/bzzoiro 此前 fire-and-forget —— 触发后只能翻系统日志,
|
||||||
|
无法程序化查询「这次采集跑到哪了/成没成」。本次改造:
|
||||||
|
1. 路由创建 pending job → 响应返回 job_id
|
||||||
|
2. 后台 _run_bzzoiro 维护 running → success / failed + result/error
|
||||||
|
3. admin 端点 /admin/ingest/jobs/{job_id} 与列表可查询
|
||||||
|
|
||||||
|
守护点:
|
||||||
|
- job 更新是「尽力而为」: _update_job 自身失败被吞掉,不影响采集主流程
|
||||||
|
- job(任务级)与 IngestFailure 死信(行级)相互独立,可同时存在
|
||||||
|
(死信路径由 test_ingest_deadletter.py 守护,本文件不动 bzzoiro 内部)
|
||||||
|
|
||||||
|
范式: 假 UoW(记录 add / 返回预设 job)+ monkeypatch source,不依赖真实数据库
|
||||||
|
(与 test_ingest_deadletter.py / test_public_readonly_api.py 相同)。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import asynccontextmanager
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import src.api.routes.ingest as ingest
|
||||||
|
import src.api.routes.schedules as schedules
|
||||||
|
from src.api.deps import require_admin
|
||||||
|
from src.api.schemas import IngestBzzoiroRequest
|
||||||
|
from src.db.base import get_db_read
|
||||||
|
from src.db.models import IngestJob
|
||||||
|
|
||||||
|
|
||||||
|
# ── 假基础设施 ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResult:
|
||||||
|
"""支持 .scalars().all() / .scalar_one_or_none() / .scalar() 的最小假结果集。"""
|
||||||
|
|
||||||
|
def __init__(self, items):
|
||||||
|
self._items = items
|
||||||
|
|
||||||
|
def scalars(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return self._items
|
||||||
|
|
||||||
|
def scalar_one_or_none(self):
|
||||||
|
return self._items[0] if self._items else None
|
||||||
|
|
||||||
|
def scalar(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSession:
|
||||||
|
"""记录 add();execute 按预设队列依次返回(与真实 UoW 的单会话用法对齐)。"""
|
||||||
|
|
||||||
|
def __init__(self, results=None):
|
||||||
|
self.added = []
|
||||||
|
self._results = list(results or [])
|
||||||
|
|
||||||
|
def add(self, obj):
|
||||||
|
self.added.append(obj)
|
||||||
|
|
||||||
|
async def execute(self, stmt):
|
||||||
|
if self._results:
|
||||||
|
return self._results.pop(0)
|
||||||
|
return _FakeResult([])
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_uow(monkeypatch, session) -> None:
|
||||||
|
"""把 ingest 模块的 get_uow 指向假会话。"""
|
||||||
|
|
||||||
|
@asynccontextmanager
|
||||||
|
async def _fake():
|
||||||
|
yield session
|
||||||
|
|
||||||
|
monkeypatch.setattr(ingest, "get_uow", _fake)
|
||||||
|
|
||||||
|
|
||||||
|
def _ingest_app() -> TestClient:
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(ingest.router)
|
||||||
|
app.dependency_overrides[require_admin] = lambda: None
|
||||||
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _admin_app(fake_db) -> TestClient:
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(schedules.router)
|
||||||
|
app.dependency_overrides[require_admin] = lambda: None
|
||||||
|
app.dependency_overrides[get_db_read] = lambda: fake_db
|
||||||
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def _job(jid: str = "job-1", status: str = "success", **kw) -> IngestJob:
|
||||||
|
return IngestJob(
|
||||||
|
id=jid,
|
||||||
|
task=kw.pop("task", "standings"),
|
||||||
|
params=kw.pop("params", {"leagues": ["E0"]}),
|
||||||
|
status=status,
|
||||||
|
result=kw.pop("result", {"total_upserted": 20}),
|
||||||
|
error=kw.pop("error", None),
|
||||||
|
created_at=kw.pop("created_at", datetime.now(timezone.utc)),
|
||||||
|
**kw,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── 1. 路由:创建 job + 返回 job_id ─────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestRouteCreatesJob:
|
||||||
|
def test_post_returns_job_id_and_persists_pending_job(self, monkeypatch):
|
||||||
|
session = _FakeSession()
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
|
||||||
|
spawned: list = []
|
||||||
|
monkeypatch.setattr(ingest, "_spawn", lambda coro: spawned.append(coro))
|
||||||
|
|
||||||
|
client = _ingest_app()
|
||||||
|
resp = client.post(
|
||||||
|
"/api/v1/ingest/bzzoiro",
|
||||||
|
json={"task": "standings", "leagues": ["E0", "SP1"], "season": "2026-2027"},
|
||||||
|
)
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert body["ok"] is True
|
||||||
|
|
||||||
|
jobs = [o for o in session.added if isinstance(o, IngestJob)]
|
||||||
|
assert len(jobs) == 1
|
||||||
|
job = jobs[0]
|
||||||
|
assert body["job_id"] == job.id
|
||||||
|
assert job.status == "pending"
|
||||||
|
assert job.task == "standings"
|
||||||
|
# params 记录的是「实际将执行」的参数(含默认联赛展开)
|
||||||
|
assert job.params["leagues"] == ["E0", "SP1"]
|
||||||
|
assert job.params["season"] == "2026-2027"
|
||||||
|
|
||||||
|
# 后台协程被捕获但未执行;显式关闭避免 un-awaited 告警
|
||||||
|
assert len(spawned) == 1
|
||||||
|
spawned[0].close()
|
||||||
|
|
||||||
|
def test_post_invalid_task_422_and_no_job(self, monkeypatch):
|
||||||
|
session = _FakeSession()
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
monkeypatch.setattr(ingest, "_spawn", lambda coro: coro.close())
|
||||||
|
|
||||||
|
client = _ingest_app()
|
||||||
|
resp = client.post("/api/v1/ingest/bzzoiro", json={"task": "bogus"})
|
||||||
|
assert resp.status_code == 422
|
||||||
|
assert not [o for o in session.added if isinstance(o, IngestJob)]
|
||||||
|
|
||||||
|
|
||||||
|
# ── 2. 后台执行:状态流转 ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestRunBzzoiroJobLifecycle:
|
||||||
|
async def test_success_flow_updates_job_running_then_success(self, monkeypatch):
|
||||||
|
job = _job(jid="job-ok", status="pending", task="events")
|
||||||
|
session = _FakeSession(results=[_FakeResult([job])] * 10)
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
|
||||||
|
class _FakeSource:
|
||||||
|
async def ingest(self, session, **kw):
|
||||||
|
return {"inserted": 3, "updated": 1, "total_inserted": 3,
|
||||||
|
"total_updated": 1, "errors": ["e1", "e2"]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
||||||
|
|
||||||
|
req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished")
|
||||||
|
await ingest._run_bzzoiro("job-ok", "events", ["E0"], req)
|
||||||
|
|
||||||
|
assert job.status == "success"
|
||||||
|
assert job.started_at is not None
|
||||||
|
assert job.finished_at is not None
|
||||||
|
assert job.error is None
|
||||||
|
# result 含 events 子任务摘要,errors 截断到前 10 条
|
||||||
|
assert job.result["events"]["total_inserted"] == 3
|
||||||
|
assert job.result["events"]["errors"] == ["e1", "e2"]
|
||||||
|
|
||||||
|
async def test_failure_flow_marks_failed_with_error(self, monkeypatch):
|
||||||
|
job = _job(jid="job-bad", status="running", task="standings", result=None)
|
||||||
|
session = _FakeSession(results=[_FakeResult([job])] * 10)
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
|
||||||
|
async def _boom(*args, **kwargs):
|
||||||
|
raise RuntimeError("network down")
|
||||||
|
|
||||||
|
monkeypatch.setattr(ingest, "ingest_bzzoiro_standings", _boom)
|
||||||
|
|
||||||
|
req = IngestBzzoiroRequest(task="standings", leagues=["E0"])
|
||||||
|
await ingest._run_bzzoiro("job-bad", "standings", ["E0"], req)
|
||||||
|
|
||||||
|
assert job.status == "failed"
|
||||||
|
assert "network down" in job.error
|
||||||
|
assert job.finished_at is not None
|
||||||
|
assert job.result is None
|
||||||
|
|
||||||
|
async def test_all_task_collects_per_task_summaries(self, monkeypatch):
|
||||||
|
job = _job(jid="job-all", status="pending", task="all")
|
||||||
|
session = _FakeSession(results=[_FakeResult([job])] * 20)
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
|
||||||
|
class _FakeSource:
|
||||||
|
async def ingest(self, session, **kw):
|
||||||
|
return {"inserted": 1, "updated": 0, "total_inserted": 1,
|
||||||
|
"total_updated": 0, "errors": []}
|
||||||
|
|
||||||
|
async def _standings(*args, **kwargs):
|
||||||
|
return {"total_upserted": 20, "errors": []}
|
||||||
|
|
||||||
|
async def _stats(*args, **kwargs):
|
||||||
|
return {"total_inserted": 5, "total_updated": 2, "errors": []}
|
||||||
|
|
||||||
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
||||||
|
monkeypatch.setattr(ingest, "ingest_bzzoiro_standings", _standings)
|
||||||
|
monkeypatch.setattr(ingest, "ingest_bzzoiro_event_stats", _stats)
|
||||||
|
|
||||||
|
req = IngestBzzoiroRequest(task="all", leagues=["E0"])
|
||||||
|
await ingest._run_bzzoiro("job-all", "all", ["E0"], req)
|
||||||
|
|
||||||
|
assert job.status == "success"
|
||||||
|
assert set(job.result.keys()) == {"events", "standings", "stats"}
|
||||||
|
assert job.result["standings"]["total_upserted"] == 20
|
||||||
|
assert job.result["stats"]["total_inserted"] == 5
|
||||||
|
|
||||||
|
async def test_update_job_failure_does_not_break_ingest(self, monkeypatch):
|
||||||
|
"""_update_job 抛错必须被吞掉:job 可观测性失败 ≠ 采集失败。"""
|
||||||
|
session = _FakeSession()
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
|
||||||
|
class _FakeSource:
|
||||||
|
async def ingest(self, session, **kw):
|
||||||
|
return {"inserted": 1, "updated": 0, "total_inserted": 1,
|
||||||
|
"total_updated": 0, "errors": []}
|
||||||
|
|
||||||
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
||||||
|
|
||||||
|
# 让 execute 抛错(_update_job 内部会捕获)
|
||||||
|
async def _broken_execute(stmt):
|
||||||
|
raise RuntimeError("db gone")
|
||||||
|
|
||||||
|
session.execute = _broken_execute
|
||||||
|
|
||||||
|
req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished")
|
||||||
|
# 不抛异常即通过;采集逻辑本身照常跑完
|
||||||
|
await ingest._run_bzzoiro("job-x", "events", ["E0"], req)
|
||||||
|
|
||||||
|
async def test_job_and_deadletter_are_independent_layers(self, monkeypatch):
|
||||||
|
"""任务级(job)与行级(死信)互不干扰:source 内部返回 errors 时,
|
||||||
|
job 仍为 success(部分失败不算任务失败),死信由 bzzoiro 层另行记录。"""
|
||||||
|
job = _job(jid="job-part", status="pending", task="events")
|
||||||
|
session = _FakeSession(results=[_FakeResult([job])] * 10)
|
||||||
|
_patch_uow(monkeypatch, session)
|
||||||
|
|
||||||
|
class _FakeSource:
|
||||||
|
async def ingest(self, session, **kw):
|
||||||
|
# 模拟 bzzoiro 管线:单条失败已写死信,汇总 errors 非空但返回正常
|
||||||
|
return {"inserted": 9, "updated": 0, "total_inserted": 9,
|
||||||
|
"total_updated": 0, "errors": ["league F1 fetch failed"]}
|
||||||
|
|
||||||
|
monkeypatch.setattr(ingest, "get_source", lambda name: _FakeSource())
|
||||||
|
|
||||||
|
req = IngestBzzoiroRequest(task="events", leagues=["E0"], status="finished")
|
||||||
|
await ingest._run_bzzoiro("job-part", "events", ["E0"], req)
|
||||||
|
|
||||||
|
assert job.status == "success"
|
||||||
|
assert job.result["events"]["errors"] == ["league F1 fetch failed"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── 3. admin 查询端点 ──────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestAdminJobEndpoints:
|
||||||
|
def test_get_job_detail_200(self, monkeypatch):
|
||||||
|
job = _job(jid="abc-123", status="running")
|
||||||
|
client = _admin_app(_FakeSession(results=[_FakeResult([job])]))
|
||||||
|
|
||||||
|
resp = client.get("/api/v1/admin/ingest/jobs/abc-123")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = resp.json()
|
||||||
|
assert body["id"] == "abc-123"
|
||||||
|
assert body["status"] == "running"
|
||||||
|
assert body["task"] == "standings"
|
||||||
|
assert body["params"] == {"leagues": ["E0"]}
|
||||||
|
assert body["result"] == {"total_upserted": 20}
|
||||||
|
assert body["error"] is None
|
||||||
|
assert body["created_at"] is not None
|
||||||
|
|
||||||
|
def test_get_job_detail_404(self):
|
||||||
|
client = _admin_app(_FakeSession(results=[_FakeResult([])]))
|
||||||
|
resp = client.get("/api/v1/admin/ingest/jobs/missing")
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|
||||||
|
def test_list_jobs_returns_serialized_rows(self):
|
||||||
|
j1 = _job(jid="j1", status="success")
|
||||||
|
j2 = _job(jid="j2", status="failed", task="events",
|
||||||
|
result=None, error="RuntimeError('x')")
|
||||||
|
client = _admin_app(_FakeSession(results=[_FakeResult([j1, j2])]))
|
||||||
|
|
||||||
|
resp = client.get("/api/v1/admin/ingest/jobs?limit=10")
|
||||||
|
assert resp.status_code == 200
|
||||||
|
rows = resp.json()
|
||||||
|
assert [r["id"] for r in rows] == ["j1", "j2"]
|
||||||
|
assert rows[1]["status"] == "failed"
|
||||||
|
assert rows[1]["error"] == "RuntimeError('x')"
|
||||||
|
|
||||||
|
def test_job_routes_live_under_admin_router_with_require_admin(self):
|
||||||
|
"""结构守护:job 端点必须挂在 /api/v1/admin 路由(路由级 require_admin)。"""
|
||||||
|
assert any(dep.dependency is require_admin for dep in schedules.router.dependencies)
|
||||||
|
paths = {getattr(r, "path", "") for r in schedules.router.routes}
|
||||||
|
assert "/api/v1/admin/ingest/jobs" in paths
|
||||||
|
assert "/api/v1/admin/ingest/jobs/{job_id}" in paths
|
||||||
@@ -206,7 +206,7 @@ class TestBacktestXgNotVisible:
|
|||||||
|
|
||||||
header = _make_header(match_dt)
|
header = _make_header(match_dt)
|
||||||
|
|
||||||
import src.llm.slices.stats as cb
|
import src.llm.context_builder as cb
|
||||||
|
|
||||||
async def mock_get_form(db, team_id, before, *, limit=10):
|
async def mock_get_form(db, team_id, before, *, limit=10):
|
||||||
# before=cutoff(1月13日),比赛在1月15日,满足 before 条件
|
# before=cutoff(1月13日),比赛在1月15日,满足 before 条件
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ class TestAllExpertsFailed:
|
|||||||
async def mock_load_header(mid, db=None):
|
async def mock_load_header(mid, db=None):
|
||||||
return header
|
return header
|
||||||
|
|
||||||
# Mock _insert_or_find_by_fingerprint — 捕获写入的 status
|
# Mock _upsert_prediction — 捕获写入的 status
|
||||||
captured_status = {}
|
captured_status = {}
|
||||||
|
|
||||||
async def mock_upsert(session, **kw):
|
async def mock_upsert(session, **kw):
|
||||||
@@ -109,7 +109,7 @@ class TestAllExpertsFailed:
|
|||||||
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
||||||
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
||||||
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
||||||
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
||||||
patch.object(orch_mod, "get_uow", FakeUow):
|
patch.object(orch_mod, "get_uow", FakeUow):
|
||||||
|
|
||||||
result = await orch_mod.predict_match_multi(999)
|
result = await orch_mod.predict_match_multi(999)
|
||||||
@@ -170,7 +170,7 @@ class TestAllExpertsFailed:
|
|||||||
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
||||||
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
||||||
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
||||||
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
||||||
patch.object(orch_mod, "get_uow", FakeUow):
|
patch.object(orch_mod, "get_uow", FakeUow):
|
||||||
|
|
||||||
result = await orch_mod.predict_match_multi(999)
|
result = await orch_mod.predict_match_multi(999)
|
||||||
@@ -235,7 +235,7 @@ class TestPartialExpertsOk:
|
|||||||
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
||||||
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
||||||
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
||||||
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
||||||
patch.object(orch_mod, "run_aggregator", mock_aggregator), \
|
patch.object(orch_mod, "run_aggregator", mock_aggregator), \
|
||||||
patch.object(orch_mod, "get_uow", FakeUow):
|
patch.object(orch_mod, "get_uow", FakeUow):
|
||||||
|
|
||||||
@@ -268,7 +268,7 @@ class TestNoAggregatorCallOnDegraded:
|
|||||||
captured_values = {}
|
captured_values = {}
|
||||||
|
|
||||||
async def mock_upsert(session, **kw):
|
async def mock_upsert(session, **kw):
|
||||||
# model / provider_name / mode 是 _insert_or_find_by_fingerprint 的顶层关键字参数,
|
# model / provider_name / mode 是 _upsert_prediction 的顶层关键字参数,
|
||||||
# 不在 values 字典里(见 orchestrator.py 的调用点)。原测试只取
|
# 不在 values 字典里(见 orchestrator.py 的调用点)。原测试只取
|
||||||
# kw["values"],导致 model 断言永远为 None。
|
# kw["values"],导致 model 断言永远为 None。
|
||||||
captured_values.update(kw.get("values", {}))
|
captured_values.update(kw.get("values", {}))
|
||||||
@@ -292,7 +292,7 @@ class TestNoAggregatorCallOnDegraded:
|
|||||||
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
with patch.object(orch_mod, "run_specialists", mock_run_specialists), \
|
||||||
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
patch.object(orch_mod, "_agent_provider", mock_agent_provider), \
|
||||||
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
patch.object(orch_mod, "load_match_header", mock_load_header), \
|
||||||
patch.object(orch_mod, "_insert_or_find_by_fingerprint", mock_upsert), \
|
patch.object(orch_mod, "_upsert_prediction", mock_upsert), \
|
||||||
patch.object(orch_mod, "get_uow", FakeUow):
|
patch.object(orch_mod, "get_uow", FakeUow):
|
||||||
|
|
||||||
await orch_mod.predict_match_multi(999)
|
await orch_mod.predict_match_multi(999)
|
||||||
@@ -300,6 +300,7 @@ class TestNoAggregatorCallOnDegraded:
|
|||||||
# 断言:aggregator provider 未被调用
|
# 断言:aggregator provider 未被调用
|
||||||
assert len(aggregator_called) == 0, \
|
assert len(aggregator_called) == 0, \
|
||||||
f"全失败时不应调用 aggregator provider,实际调用: {aggregator_called}"
|
f"全失败时不应调用 aggregator provider,实际调用: {aggregator_called}"
|
||||||
# P0-03:degraded 路径 status=degraded(model 可能为 None,由 aggregator 降级逻辑决定)
|
# 断言:model 使用 settings 默认值
|
||||||
|
assert captured_values.get("model") is not None
|
||||||
assert captured_values.get("status") == "degraded"
|
assert captured_values.get("status") == "degraded"
|
||||||
print(f"PASS: 全失败 → aggregator provider 未调用,status={captured_values.get('status')}")
|
print(f"PASS: 全失败 → aggregator provider 未调用,model={captured_values.get('model')}")
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ class TestFormSliceHomeAwayIdentity:
|
|||||||
home_name="曼城",
|
home_name="曼城",
|
||||||
away_name="利物浦",
|
away_name="利物浦",
|
||||||
)
|
)
|
||||||
import src.llm.slices.form as cb
|
import src.llm.context_builder as cb
|
||||||
orig_get_form = cb._get_form
|
orig_get_form = cb._get_form
|
||||||
async def mock_get_form(db, team_id, before, *, limit):
|
async def mock_get_form(db, team_id, before, *, limit):
|
||||||
return [hist_match] if team_id == 1 else []
|
return [hist_match] if team_id == 1 else []
|
||||||
@@ -132,7 +132,7 @@ class TestFormSliceHomeAwayIdentity:
|
|||||||
home_name="阿森纳",
|
home_name="阿森纳",
|
||||||
away_name="切尔西",
|
away_name="切尔西",
|
||||||
)
|
)
|
||||||
import src.llm.slices.form as cb
|
import src.llm.context_builder as cb
|
||||||
orig_get_form = cb._get_form
|
orig_get_form = cb._get_form
|
||||||
async def mock_get_form(db, team_id, before, *, limit):
|
async def mock_get_form(db, team_id, before, *, limit):
|
||||||
return [hist_match] if team_id == 2 else []
|
return [hist_match] if team_id == 2 else []
|
||||||
@@ -169,7 +169,7 @@ class TestStatsSliceHomeAwayIdentity:
|
|||||||
stats=_make_stats(home_xg=2.5, away_xg=0.8, home_shots=15, away_shots=5,
|
stats=_make_stats(home_xg=2.5, away_xg=0.8, home_shots=15, away_shots=5,
|
||||||
home_sot=6, away_sot=2, home_poss=60.0),
|
home_sot=6, away_sot=2, home_poss=60.0),
|
||||||
)
|
)
|
||||||
import src.llm.slices.stats as cb
|
import src.llm.context_builder as cb
|
||||||
orig_get_form = cb._get_form
|
orig_get_form = cb._get_form
|
||||||
async def mock_get_form(db, team_id, before, *, limit):
|
async def mock_get_form(db, team_id, before, *, limit):
|
||||||
return [hist_match] if team_id == 1 else []
|
return [hist_match] if team_id == 1 else []
|
||||||
|
|||||||
@@ -1,170 +0,0 @@
|
|||||||
"""P0-03 核心测试: Prediction 幂等指纹。
|
|
||||||
|
|
||||||
- TestFingerprintLogic:用 mock session 验证同/不同 fingerprint 的 INSERT/返回逻辑(无 PG 依赖)。
|
|
||||||
- TestFingerprintDeterminism:纯 hash 稳定性(无 PG 依赖)。
|
|
||||||
|
|
||||||
运行: pytest tests/test_p0_prediction_fingerprint.py -v
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from unittest.mock import MagicMock
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.db.models import Prediction
|
|
||||||
from src.llm.predict import _compute_fingerprint, _insert_or_find_by_fingerprint
|
|
||||||
|
|
||||||
|
|
||||||
def _base_values(match_id, **overrides):
|
|
||||||
base = {
|
|
||||||
"match_id": match_id,
|
|
||||||
"provider": "test-provider",
|
|
||||||
"model": "test-model",
|
|
||||||
"mode": "single",
|
|
||||||
"run_type": "live",
|
|
||||||
"prompt_version": "v1",
|
|
||||||
"prompt_hash": "ph1",
|
|
||||||
"system_prompt_hash": "sh1",
|
|
||||||
"temperature": 0.3,
|
|
||||||
"context_hash": "ch1",
|
|
||||||
"agent_ids": [],
|
|
||||||
"prediction_cutoff_at": "2026-01-01T14:00:00+00:00",
|
|
||||||
}
|
|
||||||
base.update(overrides)
|
|
||||||
return base
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeSession:
|
|
||||||
"""模拟 session:记录 add;execute 返回预设的 existing row。"""
|
|
||||||
|
|
||||||
def __init__(self, existing=None):
|
|
||||||
self._existing = existing
|
|
||||||
self.added: list = []
|
|
||||||
self.flushed = 0
|
|
||||||
|
|
||||||
def add(self, obj):
|
|
||||||
self.added.append(obj)
|
|
||||||
|
|
||||||
async def execute(self, stmt):
|
|
||||||
existing = self._existing
|
|
||||||
|
|
||||||
class _R:
|
|
||||||
def scalar_one_or_none(inner_self):
|
|
||||||
return existing
|
|
||||||
|
|
||||||
return _R()
|
|
||||||
|
|
||||||
async def flush(self):
|
|
||||||
self.flushed += 1
|
|
||||||
|
|
||||||
async def refresh(self, obj):
|
|
||||||
if getattr(obj, "id", None) is None:
|
|
||||||
obj.id = 1
|
|
||||||
|
|
||||||
|
|
||||||
class TestFingerprintLogic:
|
|
||||||
"""P0-03:同 fingerprint 返回已有行(不 UPDATE/INSERT);不同 → INSERT。"""
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_same_fingerprint_returns_existing_without_update(self):
|
|
||||||
# 构造一个"已存在"的行
|
|
||||||
existing = Prediction(
|
|
||||||
id=42, match_id=1, provider="test-provider", model="test-model",
|
|
||||||
prompt_version="v1", input_hash="same-hash",
|
|
||||||
)
|
|
||||||
existing.pred_home_goals = 2.0
|
|
||||||
existing.prompt_version = "v1"
|
|
||||||
|
|
||||||
s = _FakeSession(existing=existing)
|
|
||||||
values = _base_values(1, prompt_version="v1") # 与 existing 同 fingerprint 需 input_hash 相同
|
|
||||||
|
|
||||||
# 但 fingerprint 是动态计算的,existing.input_hash 需匹配。直接让 fake 返回 existing。
|
|
||||||
result = await _insert_or_find_by_fingerprint(s, values=values)
|
|
||||||
|
|
||||||
# 应返回 existing,不 add 新行
|
|
||||||
assert result is existing, "同 fingerprint 必须返回已有行"
|
|
||||||
assert s.added == [], "同 fingerprint 不应 INSERT"
|
|
||||||
assert result.pred_home_goals == 2.0, "返回的应是已有行(字段不变)"
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_different_fingerprint_inserts_new(self):
|
|
||||||
# 无已有行 → INSERT
|
|
||||||
s = _FakeSession(existing=None)
|
|
||||||
values = _base_values(1, prompt_version="v1", context_hash="ch1")
|
|
||||||
|
|
||||||
result = await _insert_or_find_by_fingerprint(s, values=values)
|
|
||||||
|
|
||||||
assert len(s.added) == 1, "无已有行时应 INSERT"
|
|
||||||
assert isinstance(s.added[0], Prediction)
|
|
||||||
# input_hash 应被设为指纹
|
|
||||||
assert result.input_hash is not None and len(result.input_hash) == 64 # SHA-256 hex
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_fingerprint_computed_from_values(self):
|
|
||||||
"""fingerprint 应基于 values 的全部关键字段计算。"""
|
|
||||||
s1 = _FakeSession(existing=None)
|
|
||||||
s2 = _FakeSession(existing=None)
|
|
||||||
|
|
||||||
v1 = _base_values(1, prompt_version="v1")
|
|
||||||
v2 = _base_values(1, prompt_version="v1") # 同值
|
|
||||||
|
|
||||||
r1 = await _insert_or_find_by_fingerprint(s1, values=v1)
|
|
||||||
r2 = await _insert_or_find_by_fingerprint(s2, values=v2)
|
|
||||||
|
|
||||||
# 同值 → 同 fingerprint(跨 session 也一致)
|
|
||||||
assert r1.input_hash == r2.input_hash
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
|
||||||
async def test_existing_never_updated(self):
|
|
||||||
"""核心可信度:同 fingerprint 绝不覆盖 pred_/reasoning/agent_outputs。"""
|
|
||||||
existing = Prediction(
|
|
||||||
id=99, match_id=1, provider="p", model="m",
|
|
||||||
prompt_version="v1", input_hash="fixed-hash",
|
|
||||||
pred_home_goals=1.0, pred_away_goals=0.0,
|
|
||||||
reasoning="original", agent_outputs=[{"agent": "form"}],
|
|
||||||
)
|
|
||||||
s = _FakeSession(existing=existing)
|
|
||||||
|
|
||||||
# 即便传入不同的 pred_*,也应返回原行(字段不变)
|
|
||||||
values = _base_values(1, prompt_version="v1")
|
|
||||||
# 让 fake 返回 existing: 需 fingerprint 匹配。fake.execute 始终返回 existing。
|
|
||||||
result = await _insert_or_find_by_fingerprint(s, values=values)
|
|
||||||
|
|
||||||
assert result is existing
|
|
||||||
assert result.pred_home_goals == 1.0, "pred_home_goals 不应被覆盖"
|
|
||||||
assert result.reasoning == "original", "reasoning 不应被覆盖"
|
|
||||||
assert result.agent_outputs == [{"agent": "form"}], "agent_outputs 不应被覆盖"
|
|
||||||
|
|
||||||
|
|
||||||
class TestFingerprintDeterminism:
|
|
||||||
"""fingerprint 必须稳定(同输入 → 同 hash)。"""
|
|
||||||
|
|
||||||
def test_same_values_same_fingerprint(self):
|
|
||||||
v = _base_values(1)
|
|
||||||
assert _compute_fingerprint(v) == _compute_fingerprint(dict(v))
|
|
||||||
|
|
||||||
def test_different_prompt_version_different_fingerprint(self):
|
|
||||||
v1 = _base_values(1, prompt_version="v1")
|
|
||||||
v2 = _base_values(1, prompt_version="v2")
|
|
||||||
assert _compute_fingerprint(v1) != _compute_fingerprint(v2)
|
|
||||||
|
|
||||||
def test_different_agent_ids_different_fingerprint(self):
|
|
||||||
v1 = _base_values(1, agent_ids=["form", "stats"])
|
|
||||||
v2 = _base_values(1, agent_ids=["form", "h2h"])
|
|
||||||
assert _compute_fingerprint(v1) != _compute_fingerprint(v2)
|
|
||||||
|
|
||||||
def test_different_cutoff_different_fingerprint(self):
|
|
||||||
v1 = _base_values(1, prediction_cutoff_at="2026-01-01T14:00:00+00:00")
|
|
||||||
v2 = _base_values(1, prediction_cutoff_at="2026-01-01T10:00:00+00:00")
|
|
||||||
assert _compute_fingerprint(v1) != _compute_fingerprint(v2)
|
|
||||||
|
|
||||||
def test_different_context_different_fingerprint(self):
|
|
||||||
v1 = _base_values(1, context_hash="ch1")
|
|
||||||
v2 = _base_values(1, context_hash="ch2")
|
|
||||||
assert _compute_fingerprint(v1) != _compute_fingerprint(v2)
|
|
||||||
|
|
||||||
def test_agent_ids_order_independent(self):
|
|
||||||
"""agent_ids 排序后计算,顺序不影响 hash。"""
|
|
||||||
v1 = _base_values(1, agent_ids=["stats", "form"])
|
|
||||||
v2 = _base_values(1, agent_ids=["form", "stats"])
|
|
||||||
assert _compute_fingerprint(v1) == _compute_fingerprint(v2)
|
|
||||||
@@ -1,124 +0,0 @@
|
|||||||
"""P0-01 回归测试: missing score 不得变 0:0。
|
|
||||||
|
|
||||||
运行: pytest tests/test_p0_score_status.py -v
|
|
||||||
(无需真实 PG;用 fake DB + 模型元数据断言。)
|
|
||||||
"""
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from src.db.models import League, Match, Team
|
|
||||||
|
|
||||||
|
|
||||||
# ── fake DB(对齐现有测试约定) ────────────────────────────────────
|
|
||||||
class _FakeResult:
|
|
||||||
def __init__(self, items): self._items = list(items)
|
|
||||||
def scalars(self): return self
|
|
||||||
def all(self): return list(self._items)
|
|
||||||
def scalar_one_or_none(self): return self._items[0] if self._items else None
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeDB:
|
|
||||||
def __init__(self): self.added = []
|
|
||||||
def add(self, obj): self.added.append(obj)
|
|
||||||
async def execute(self, stmt): return _FakeResult([])
|
|
||||||
async def flush(self):
|
|
||||||
for o in self.added:
|
|
||||||
if getattr(o, "id", None) is None:
|
|
||||||
o.id = 1
|
|
||||||
|
|
||||||
|
|
||||||
def _league(lid=1):
|
|
||||||
lg = League(id=lid, code="E0", name="Test", country="X")
|
|
||||||
return lg
|
|
||||||
|
|
||||||
|
|
||||||
def _teams():
|
|
||||||
return Team(id=10, name="Arsenal FC", name_zh="阿森纳"), Team(id=20, name="Chelsea FC", name_zh="切尔西")
|
|
||||||
|
|
||||||
|
|
||||||
class TestScoreStatusConstraintPresence:
|
|
||||||
"""模型必须定义 score_status 相关 CHECK 约束。"""
|
|
||||||
|
|
||||||
def test_score_status_column_exists(self):
|
|
||||||
cols = {c.name for c in Match.__table__.columns}
|
|
||||||
assert "score_status" in cols
|
|
||||||
|
|
||||||
def test_score_integrity_check_exists(self):
|
|
||||||
names = {c.name for c in Match.__table__.constraints if c.name}
|
|
||||||
# 新约束 ck_matches_score_integrity 必须存在
|
|
||||||
assert any("score_integrity" in n for n in names), \
|
|
||||||
f"ck_matches_score_integrity 未找到,现有约束: {names}"
|
|
||||||
|
|
||||||
def test_old_finished_has_score_check_removed(self):
|
|
||||||
names = {c.name for c in Match.__table__.constraints}
|
|
||||||
assert "ck_matches_finished_has_score" not in names, \
|
|
||||||
"旧约束 ck_matches_finished_has_score 应已被替换"
|
|
||||||
|
|
||||||
|
|
||||||
class TestMatchAcceptsMissingScore:
|
|
||||||
"""Match 对象层面: 完赛 + score_status=missing + goals=NULL 必须可构造。"""
|
|
||||||
|
|
||||||
def test_construct_finished_missing_null_goals(self):
|
|
||||||
home, away = _teams()
|
|
||||||
m = Match(
|
|
||||||
id=1, league_id=_league().id, home_team_id=home.id, away_team_id=away.id,
|
|
||||||
match_date="2026-01-01 15:00:00+00:00",
|
|
||||||
match_status="finished", score_status="missing",
|
|
||||||
home_goals=None, away_goals=None,
|
|
||||||
)
|
|
||||||
assert m.home_goals is None
|
|
||||||
assert m.away_goals is None
|
|
||||||
assert m.score_status == "missing"
|
|
||||||
|
|
||||||
def test_add_to_fake_db(self):
|
|
||||||
db = _FakeDB()
|
|
||||||
home, away = _teams()
|
|
||||||
m = Match(
|
|
||||||
league_id=_league().id, home_team_id=home.id, away_team_id=away.id,
|
|
||||||
match_date="2026-01-01 15:00:00+00:00",
|
|
||||||
match_status="finished", score_status="missing",
|
|
||||||
home_goals=None, away_goals=None,
|
|
||||||
)
|
|
||||||
db.add(m)
|
|
||||||
|
|
||||||
def test_server_default_is_unknown(self):
|
|
||||||
"""score_status 列的 server_default 必须为 unknown(DB 插入未显式赋值时兜底)。"""
|
|
||||||
col = Match.__table__.c.score_status
|
|
||||||
assert col.server_default is not None
|
|
||||||
assert "unknown" in str(col.server_default.arg)
|
|
||||||
|
|
||||||
|
|
||||||
class TestNormalizeNoDowngrade:
|
|
||||||
"""normalize_bzzoiro 不得把完赛缺分静默降级为 scheduled。"""
|
|
||||||
|
|
||||||
def _raw(self, status="finished", home_score=None, away_score=None):
|
|
||||||
return {
|
|
||||||
"event_date": "2026-01-01 15:00:00",
|
|
||||||
"status": status,
|
|
||||||
"home_team": "Arsenal",
|
|
||||||
"away_team": "Chelsea",
|
|
||||||
"home_score": home_score,
|
|
||||||
"away_score": away_score,
|
|
||||||
}
|
|
||||||
|
|
||||||
def test_finished_missing_score_keeps_finished(self):
|
|
||||||
from src.data.normalize import normalize_bzzoiro
|
|
||||||
m = normalize_bzzoiro(self._raw("finished", None, None), "E0")
|
|
||||||
assert m is not None
|
|
||||||
assert m.match_status == "finished", "完赛缺分不得降级为 scheduled"
|
|
||||||
assert m.score_status == "missing"
|
|
||||||
assert m.home_goals is None
|
|
||||||
assert m.away_goals is None
|
|
||||||
|
|
||||||
def test_finished_with_score_is_known(self):
|
|
||||||
from src.data.normalize import normalize_bzzoiro
|
|
||||||
m = normalize_bzzoiro(self._raw("finished", 2, 1), "E0")
|
|
||||||
assert m.match_status == "finished"
|
|
||||||
assert m.score_status == "known"
|
|
||||||
assert m.home_goals == 2 and m.away_goals == 1
|
|
||||||
|
|
||||||
def test_scheduled_no_score_is_unknown(self):
|
|
||||||
from src.data.normalize import normalize_bzzoiro
|
|
||||||
m = normalize_bzzoiro(self._raw("scheduled", None, None), "E0")
|
|
||||||
assert m.match_status == "scheduled"
|
|
||||||
assert m.score_status == "unknown"
|
|
||||||
assert m.home_goals is None
|
|
||||||
@@ -1,138 +0,0 @@
|
|||||||
"""P0-02 回归测试: 积分榜改为追加快照(append-only) + available_at cutoff。
|
|
||||||
|
|
||||||
运行: pytest tests/test_p0_standings_cutoff.py -v
|
|
||||||
(模型约束用 fake DB;cutoff 过滤语义用 fake session 验证参数传递。)
|
|
||||||
"""
|
|
||||||
import pytest
|
|
||||||
from sqlalchemy import select
|
|
||||||
|
|
||||||
from src.db.models import League, Standing, Team
|
|
||||||
|
|
||||||
|
|
||||||
# ── fake DB(对齐现有测试约定) ────────────────────────────────────
|
|
||||||
class _FakeResult:
|
|
||||||
def __init__(self, items): self._items = list(items)
|
|
||||||
def scalars(self):
|
|
||||||
class _S:
|
|
||||||
def __init__(self, items): self._items = items
|
|
||||||
def all(self): return list(self._items)
|
|
||||||
return _S(self._items)
|
|
||||||
def scalar_one_or_none(self):
|
|
||||||
return self._items[0] if self._items else None
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeDB:
|
|
||||||
captured: list = []
|
|
||||||
|
|
||||||
def __init__(self, league=None, standing_rows=None):
|
|
||||||
self._league = league
|
|
||||||
self._rows = standing_rows or []
|
|
||||||
_FakeDB.captured = []
|
|
||||||
|
|
||||||
def add(self, obj):
|
|
||||||
_FakeDB.captured.append(obj)
|
|
||||||
|
|
||||||
async def execute(self, stmt):
|
|
||||||
# 记录生成的 SQL(字符串化)供断言
|
|
||||||
_FakeDB.captured.append(str(stmt))
|
|
||||||
compiled = str(stmt)
|
|
||||||
if "league" in compiled.lower() and "standing" not in compiled.lower():
|
|
||||||
return _FakeResult([self._league] if self._league else [])
|
|
||||||
return _FakeResult(self._rows)
|
|
||||||
|
|
||||||
async def flush(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeTeam:
|
|
||||||
def __init__(self, tid, name):
|
|
||||||
self.id = tid
|
|
||||||
self.name = name
|
|
||||||
self.name_zh = None
|
|
||||||
|
|
||||||
|
|
||||||
class _Header:
|
|
||||||
def __init__(self):
|
|
||||||
from src.llm.slices.common import MatchHeader
|
|
||||||
self._h = MatchHeader(
|
|
||||||
match_id=1, home_name="A", away_name="B", league_name="E0",
|
|
||||||
season="2026", match_date="2026-01-01", match_dt=None,
|
|
||||||
stage=None, home_team_id=10, away_team_id=20, league_id=1,
|
|
||||||
)
|
|
||||||
|
|
||||||
def __getattr__(self, name):
|
|
||||||
return getattr(self._h, name)
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsModel:
|
|
||||||
"""standings 模型必须有 available_at + 新唯一约束。"""
|
|
||||||
|
|
||||||
def test_available_at_column(self):
|
|
||||||
cols = {c.name for c in Standing.__table__.columns}
|
|
||||||
assert "available_at" in cols
|
|
||||||
|
|
||||||
def test_unique_constraint_includes_available_at(self):
|
|
||||||
names = {c.name for c in Standing.__table__.constraints}
|
|
||||||
assert any("available" in n and n.startswith("uq_") for n in names), \
|
|
||||||
f"缺少含 available_at 的唯一约束,现有: {names}"
|
|
||||||
|
|
||||||
def test_old_unique_constraint_removed(self):
|
|
||||||
names = {c.name for c in Standing.__table__.constraints}
|
|
||||||
assert "uq_standings_league_season_team" not in names, \
|
|
||||||
"旧约束 uq_standings_league_season_team 应已被替换"
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsSliceCutoff:
|
|
||||||
"""standings_slice 必须尊重 before(cutoff):before=None → now()。"""
|
|
||||||
|
|
||||||
def test_before_none_uses_now(self):
|
|
||||||
"""before=None 时应将 cutoff 视为 now()(取最新可用快照)。"""
|
|
||||||
from src.llm.slices import standings as st_mod
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
|
|
||||||
calls = {}
|
|
||||||
real_execute = None
|
|
||||||
|
|
||||||
class _DB:
|
|
||||||
def __init__(self): self._league = League(id=1, code="E0", name="E0")
|
|
||||||
def add(self, obj): pass
|
|
||||||
async def execute(self, stmt):
|
|
||||||
# 捕获 WHERE available_at <= ? 的参数
|
|
||||||
sql = str(stmt)
|
|
||||||
if "available_at" in sql:
|
|
||||||
# 提取编译后的 params
|
|
||||||
try:
|
|
||||||
params = stmt.compile().params
|
|
||||||
calls["cutoff"] = params.get("available_at_1")
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if "league" in sql.lower() and "standing" not in sql.lower():
|
|
||||||
return _FakeResult([self._league])
|
|
||||||
return _FakeResult([])
|
|
||||||
async def flush(self): pass
|
|
||||||
|
|
||||||
async def run():
|
|
||||||
db = _DB()
|
|
||||||
before = None
|
|
||||||
await st_mod.standings_slice(_Header(), before=before, db=db)
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
asyncio.run(run())
|
|
||||||
# before=None 时应注入 now() 作为 cutoff
|
|
||||||
assert "cutoff" in calls, "未对 available_at 施加 cutoff 过滤"
|
|
||||||
assert calls["cutoff"] is not None
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsAppendOnly:
|
|
||||||
"""采集应 INSERT 新行(带 available_at),不覆盖旧行。"""
|
|
||||||
|
|
||||||
def test_values_include_available_at(self):
|
|
||||||
"""采集构造的 Standing 必须含 available_at 字段。"""
|
|
||||||
from src.data import bzzoiro_standings as bzs
|
|
||||||
# 检查函数源码是否包含 available_at(编译期守卫)
|
|
||||||
import inspect
|
|
||||||
src = inspect.getsource(bzs.ingest_bzzoiro_standings)
|
|
||||||
assert "available_at" in src, "采集函数必须设置 available_at"
|
|
||||||
# 不应再出现按 (league, season, team) 的 upsert 查询
|
|
||||||
assert "scalar_one_or_none" not in src or "Standing.league_id == league.id" not in src.replace("available_at", ""), \
|
|
||||||
"不应再按 (league, season, team) 做 upsert 查询"
|
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
验证:
|
验证:
|
||||||
1. 唯一约束包含 mode + run_type
|
1. 唯一约束包含 mode + run_type
|
||||||
2. 同一场比赛 live 与 backtest 预测可共存,互不覆盖
|
2. 同一场比赛 live 与 backtest 预测可共存,互不覆盖
|
||||||
3. _insert_or_find_by_fingerprint 正确区分 run_type
|
3. _upsert_prediction 正确区分 run_type
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -12,7 +12,6 @@ from pathlib import Path
|
|||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy import Index
|
|
||||||
|
|
||||||
from src.db.models import Prediction, UniqueConstraint, CheckConstraint
|
from src.db.models import Prediction, UniqueConstraint, CheckConstraint
|
||||||
|
|
||||||
@@ -23,32 +22,20 @@ MIGRATION_PATH = REPO_ROOT / "alembic" / "versions" / "0013_predictions_unique_c
|
|||||||
|
|
||||||
|
|
||||||
class TestUniqueConstraint:
|
class TestUniqueConstraint:
|
||||||
"""P0-03: 验证幂等指纹唯一索引(替代旧 (match, provider, model, mode, run_type) 唯一约束)。"""
|
"""验证唯一约束包含 mode + run_type。"""
|
||||||
|
|
||||||
def test_input_hash_partial_unique_index(self):
|
def test_constraint_columns(self):
|
||||||
"""P0-03: input_hash 非空时必须唯一(同指纹 → 返回已有行,不 UPDATE/INSERT)。"""
|
"""唯一约束应包含 match_id, provider, model, mode, run_type。"""
|
||||||
idx = [
|
uc = [
|
||||||
i for i in Prediction.__table__.indexes
|
|
||||||
if i.unique and "input_hash" in i.name
|
|
||||||
]
|
|
||||||
assert len(idx) == 1, f"缺少 input_hash partial unique 索引,现有 indexes: {[i.name for i in Prediction.__table__.indexes]}"
|
|
||||||
# partial unique: postgresql_where 必须限制 input_hash IS NOT NULL
|
|
||||||
assert idx[0].dialect_kwargs.get("postgresql_where") is not None
|
|
||||||
|
|
||||||
def test_old_unique_constraint_removed(self):
|
|
||||||
"""P0-03: 旧 (match, provider, model, mode, run_type) 唯一约束必须已移除。"""
|
|
||||||
from sqlalchemy import UniqueConstraint
|
|
||||||
|
|
||||||
old = [
|
|
||||||
c for c in Prediction.__table__.constraints
|
c for c in Prediction.__table__.constraints
|
||||||
if isinstance(c, UniqueConstraint) and c.name == "uq_predictions_match_provider_model_mode_run_type"
|
if isinstance(c, UniqueConstraint) and "match" in c.name
|
||||||
]
|
]
|
||||||
assert len(old) == 0, f"旧约束必须已移除,但仍存在: {[c.name for c in old]}"
|
assert len(uc) == 1
|
||||||
|
cols = [c.name for c in uc[0].columns]
|
||||||
|
assert cols == ["match_id", "provider", "model", "mode", "run_type"]
|
||||||
|
|
||||||
def test_run_type_check_constraint(self):
|
def test_run_type_check_constraint(self):
|
||||||
"""应有 run_type 的 check constraint。"""
|
"""应有 run_type 的 check constraint。"""
|
||||||
from sqlalchemy import CheckConstraint
|
|
||||||
|
|
||||||
cc = [
|
cc = [
|
||||||
c for c in Prediction.__table__.constraints
|
c for c in Prediction.__table__.constraints
|
||||||
if isinstance(c, CheckConstraint) and "run_type" in c.name
|
if isinstance(c, CheckConstraint) and "run_type" in c.name
|
||||||
@@ -65,16 +52,13 @@ class TestUniqueConstraint:
|
|||||||
|
|
||||||
|
|
||||||
class TestUpsertPredictionSignature:
|
class TestUpsertPredictionSignature:
|
||||||
"""验证 _insert_or_find_by_fingerprint 签名(P0-03 指纹模式)。"""
|
"""验证 _upsert_prediction 函数签名包含 run_type。"""
|
||||||
|
|
||||||
def test_signature_uses_values_dict(self):
|
def test_signature_has_run_type(self):
|
||||||
"""P0-03: 新接口通过 values dict 接收全部字段(含 run_type/match_id/...)。"""
|
from src.llm.predict import _upsert_prediction
|
||||||
from src.llm.predict import _insert_or_find_by_fingerprint
|
|
||||||
|
|
||||||
sig = inspect.signature(_insert_or_find_by_fingerprint)
|
sig = inspect.signature(_upsert_prediction)
|
||||||
params = sig.parameters
|
assert "run_type" in sig.parameters
|
||||||
assert "session" in params
|
|
||||||
assert "values" in params # 所有业务字段走 values dict
|
|
||||||
|
|
||||||
def test_signature_has_backtest_in_predict_match(self):
|
def test_signature_has_backtest_in_predict_match(self):
|
||||||
from src.llm.predict import predict_match
|
from src.llm.predict import predict_match
|
||||||
|
|||||||
@@ -32,24 +32,17 @@ class TestEagerLoadCoverage:
|
|||||||
|
|
||||||
models.py 已声明 lazy="selectin" 兜底,但这里同时检查显式
|
models.py 已声明 lazy="selectin" 兜底,但这里同时检查显式
|
||||||
selectinload —— 显式声明是查询意图的固化,也被 P0 修复所依赖。
|
selectinload —— 显式声明是查询意图的固化,也被 P0 修复所依赖。
|
||||||
(context_builder 已按 slice 拆分到 src/llm/slices/,getter 随实现迁移。)
|
|
||||||
"""
|
"""
|
||||||
for rel in ("llm/slices/form.py", "llm/slices/h2h.py", "llm/slices/home_away.py"):
|
src = _read("llm/context_builder.py")
|
||||||
src = _read(rel)
|
|
||||||
for fn in ("_get_form", "_get_h2h", "_get_home_away"):
|
for fn in ("_get_form", "_get_h2h", "_get_home_away"):
|
||||||
# 截取函数体(仅当前文件定义了该函数才检查)
|
# 截取函数体
|
||||||
m = re.search(rf"async def {fn}\(.*?(?=\nasync def |\n# =|\Z)", src, re.S)
|
m = re.search(rf"async def {fn}\(.*?(?=\nasync def |\n# =|\Z)", src, re.S)
|
||||||
if not m:
|
assert m, f"{fn} 未找到"
|
||||||
continue
|
|
||||||
body = m.group(0)
|
body = m.group(0)
|
||||||
assert "selectinload" in body, (
|
assert "selectinload" in body, (
|
||||||
f"{fn} 查询 Match 但未 eager-load 关系 —— "
|
f"{fn} 查询 Match 但未 eager-load 关系 —— "
|
||||||
"this would raise MissingGreenlet in async SQLAlchemy (P0-2)"
|
"this would raise MissingGreenlet in async SQLAlchemy (P0-2)"
|
||||||
)
|
)
|
||||||
# 守卫完整性: 三个 getter 必须都能在 slices 包中找到
|
|
||||||
all_src = "\n".join(_read(r) for r in ("llm/slices/form.py", "llm/slices/h2h.py", "llm/slices/home_away.py"))
|
|
||||||
for fn in ("_get_form", "_get_h2h", "_get_home_away"):
|
|
||||||
assert f"async def {fn}(" in all_src, f"{fn} 未在 slices 包中找到(拆分后迁移缺失?)"
|
|
||||||
|
|
||||||
def test_backtest_candidates_eager_load(self):
|
def test_backtest_candidates_eager_load(self):
|
||||||
"""回测取历史比赛必须 eager-load(否则 session 关闭后访问关系必炸)。"""
|
"""回测取历史比赛必须 eager-load(否则 session 关闭后访问关系必炸)。"""
|
||||||
@@ -172,9 +165,7 @@ class TestBzzoiroLineage:
|
|||||||
if re.search(r"source_event_id\s*(?:is|==|!=)", stripped):
|
if re.search(r"source_event_id\s*(?:is|==|!=)", stripped):
|
||||||
continue
|
continue
|
||||||
if re.search(r"source_event_id\s*\.\s*\w+\s*\(", stripped):
|
if re.search(r"source_event_id\s*\.\s*\w+\s*\(", stripped):
|
||||||
continue # 方法调用(obj.source_event_id(...)),不是赋值
|
continue # 方法调用,不是赋值
|
||||||
if re.search(r"\w*source_event_id\s*\(", stripped):
|
|
||||||
continue # 方法调用(如 find_by_source_event_id(eid)),不是赋值
|
|
||||||
if self._ASSIGN_DIRECT.search(stripped):
|
if self._ASSIGN_DIRECT.search(stripped):
|
||||||
continue # 直接取配对 raw
|
continue # 直接取配对 raw
|
||||||
m_var = self._ASSIGN_VIA_VAR.search(stripped)
|
m_var = self._ASSIGN_VIA_VAR.search(stripped)
|
||||||
@@ -184,7 +175,7 @@ class TestBzzoiroLineage:
|
|||||||
return bad
|
return bad
|
||||||
|
|
||||||
def test_normalized_matches_carries_raw(self):
|
def test_normalized_matches_carries_raw(self):
|
||||||
src = _read("data/bzzoiro_events.py")
|
src = _read("data/bzzoiro.py")
|
||||||
# 规范化结果必须与原始 event 成对保存
|
# 规范化结果必须与原始 event 成对保存
|
||||||
assert "normalized_matches.append((nm, raw))" in src, (
|
assert "normalized_matches.append((nm, raw))" in src, (
|
||||||
"normalized_matches 未携带 (nm, raw) 元组 —— raw 变量泄漏会回归 (P0-3)"
|
"normalized_matches 未携带 (nm, raw) 元组 —— raw 变量泄漏会回归 (P0-3)"
|
||||||
@@ -202,7 +193,7 @@ class TestBzzoiroLineage:
|
|||||||
它不是 `raw.get(` 同一行,但同样正确。非法写法(回归)是直接
|
它不是 `raw.get(` 同一行,但同样正确。非法写法(回归)是直接
|
||||||
`existing_match.source_event_id = orphan_var`。
|
`existing_match.source_event_id = orphan_var`。
|
||||||
"""
|
"""
|
||||||
src = _read("data/bzzoiro_events.py")
|
src = _read("data/bzzoiro.py")
|
||||||
seg = self._consume_loop_body(src)
|
seg = self._consume_loop_body(src)
|
||||||
bad = self._bad_assignments(seg)
|
bad = self._bad_assignments(seg)
|
||||||
assert len(bad) == 0, (
|
assert len(bad) == 0, (
|
||||||
@@ -215,7 +206,7 @@ class TestBzzoiroLineage:
|
|||||||
下游 `_backfill_stats` 里合法地在 ORM 对象上访问 `m.source_event_id`
|
下游 `_backfill_stats` 里合法地在 ORM 对象上访问 `m.source_event_id`
|
||||||
(与配对 raw 无关)。若 seg 越界,test_no_orphan_raw_use 会误报。
|
(与配对 raw 无关)。若 seg 越界,test_no_orphan_raw_use 会误报。
|
||||||
"""
|
"""
|
||||||
src = _read("data/bzzoiro_events.py")
|
src = _read("data/bzzoiro.py")
|
||||||
seg = self._consume_loop_body(src)
|
seg = self._consume_loop_body(src)
|
||||||
assert "m.source_event_id" not in seg, (
|
assert "m.source_event_id" not in seg, (
|
||||||
"循环体截取越界,扫到了下游 stats 管线 —— 会误报 P0-3"
|
"循环体截取越界,扫到了下游 stats 管线 —— 会误报 P0-3"
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ import re
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from src.db.models import Team, TeamAlias
|
|
||||||
from src.data.key_ring import _mask
|
from src.data.key_ring import _mask
|
||||||
from src.llm import backtest as bt_mod
|
from src.llm import backtest as bt_mod
|
||||||
from src.llm.agents import orchestrator as orch_mod
|
from src.llm.agents import orchestrator as orch_mod
|
||||||
@@ -106,15 +105,6 @@ class _FakeDb:
|
|||||||
self.added: list = []
|
self.added: list = []
|
||||||
self.flush_count = 0
|
self.flush_count = 0
|
||||||
self._next_id = 1000
|
self._next_id = 1000
|
||||||
self._teams_by_id: dict[int, Team] = {}
|
|
||||||
self._aliases: dict[str, TeamAlias] = {}
|
|
||||||
|
|
||||||
async def get(self, cls, key):
|
|
||||||
if cls is Team:
|
|
||||||
return self._teams_by_id.get(key)
|
|
||||||
if cls is TeamAlias:
|
|
||||||
return self._aliases.get(key)
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def execute(self, _stmt):
|
async def execute(self, _stmt):
|
||||||
if self._results:
|
if self._results:
|
||||||
@@ -130,16 +120,12 @@ class _FakeDb:
|
|||||||
if getattr(obj, "id", None) is None:
|
if getattr(obj, "id", None) is None:
|
||||||
self._next_id += 1
|
self._next_id += 1
|
||||||
obj.id = self._next_id
|
obj.id = self._next_id
|
||||||
if isinstance(obj, Team) and getattr(obj, "id", None) is not None:
|
|
||||||
self._teams_by_id[obj.id] = obj
|
|
||||||
if isinstance(obj, TeamAlias):
|
|
||||||
self._aliases[obj.alias_normalized] = obj
|
|
||||||
|
|
||||||
|
|
||||||
async def test_r2_standings_actually_upserts(monkeypatch):
|
async def test_r2_standings_actually_upserts(monkeypatch):
|
||||||
"""行为测试: 喂一份积分榜 payload,断言真的构造了 Standing 且计数 > 0。"""
|
"""行为测试: 喂一份积分榜 payload,断言真的构造了 Standing 且计数 > 0。"""
|
||||||
import src.data.bzzoiro as bz
|
import src.data.bzzoiro as bz
|
||||||
from src.db.models import DataLineage, League, RawEvent, Standing, Team
|
from src.db.models import League, Standing, Team
|
||||||
|
|
||||||
payload = {
|
payload = {
|
||||||
"season": {"start_date": "2025-08-01", "end_date": "2026-05-31"},
|
"season": {"start_date": "2025-08-01", "end_date": "2026-05-31"},
|
||||||
@@ -180,10 +166,7 @@ async def test_r2_standings_actually_upserts(monkeypatch):
|
|||||||
|
|
||||||
standings = [o for o in db.added if isinstance(o, Standing)]
|
standings = [o for o in db.added if isinstance(o, Standing)]
|
||||||
assert len(standings) == 2, "应真的构造 Standing 行"
|
assert len(standings) == 2, "应真的构造 Standing 行"
|
||||||
# standings 采集接线 Bronze 后(RawEvent + DataLineage),add 的对象类型白名单随之放宽
|
assert all(isinstance(o, (Standing, Team)) for o in db.added)
|
||||||
assert all(
|
|
||||||
isinstance(o, (Standing, Team, RawEvent, DataLineage)) for o in db.added
|
|
||||||
)
|
|
||||||
|
|
||||||
first = standings[0]
|
first = standings[0]
|
||||||
assert first.league_id == 42
|
assert first.league_id == 42
|
||||||
@@ -194,8 +177,8 @@ async def test_r2_standings_actually_upserts(monkeypatch):
|
|||||||
assert first.zone == "Champions League" # 优先取 label
|
assert first.zone == "Champions League" # 优先取 label
|
||||||
|
|
||||||
|
|
||||||
async def test_r2_standings_append_new_row(monkeypatch):
|
async def test_r2_standings_upsert_updates_existing(monkeypatch):
|
||||||
"""P0-02 行为测试: 每次采集 INSERT 新行(带 available_at),不更新旧行。"""
|
"""行为测试: 已存在同 (league, season, team) 时应就地更新而非新增。"""
|
||||||
import src.data.bzzoiro as bz
|
import src.data.bzzoiro as bz
|
||||||
from src.db.models import League, Standing
|
from src.db.models import League, Standing
|
||||||
|
|
||||||
@@ -217,20 +200,16 @@ async def test_r2_standings_append_new_row(monkeypatch):
|
|||||||
existing = Standing(league_id=42, season="2025-2026", team_id=7, position=9)
|
existing = Standing(league_id=42, season="2025-2026", team_id=7, position=9)
|
||||||
existing.points = 1
|
existing.points = 1
|
||||||
|
|
||||||
# 查询顺序: League(命中) → Team 预载(命中) → (P0-02 不再查询 Standing)
|
# 查询顺序: League → Team 预载(命中) → Standing 查询(命中已有行)
|
||||||
db = _FakeDb(results=[_FakeResult([league]), _FakeResult([team])])
|
db = _FakeDb(results=[_FakeResult([league]), _FakeResult([team]), _FakeResult([existing])])
|
||||||
|
|
||||||
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
||||||
|
|
||||||
assert result["total_upserted"] == 1
|
assert result["total_upserted"] == 1
|
||||||
# P0-02: 追加快照——新增 Standing 行,旧行不被修改
|
assert existing.points == 30, "已有行应被就地更新"
|
||||||
new_rows = [o for o in db.added if isinstance(o, Standing)]
|
|
||||||
assert len(new_rows) == 1, "P0-02 应新增一条 Standing 行"
|
|
||||||
assert new_rows[0].points == 30, "新行应承载新采集数据"
|
|
||||||
assert new_rows[0].available_at is not None, "新行必须含 available_at"
|
|
||||||
# 旧行未被修改(仍保持原值)
|
|
||||||
assert existing.points == 1, "P0-02 旧行不应被覆盖"
|
|
||||||
assert result["leagues"]["EPL"]["teams_created"] == 0
|
assert result["leagues"]["EPL"]["teams_created"] == 0
|
||||||
|
# 不应新增 Standing(只有 league/team 层面的 add)
|
||||||
|
assert not [o for o in db.added if isinstance(o, Standing)]
|
||||||
|
|
||||||
|
|
||||||
def test_r2_source_contains_real_upsert_loop():
|
def test_r2_source_contains_real_upsert_loop():
|
||||||
@@ -241,9 +220,7 @@ def test_r2_source_contains_real_upsert_loop():
|
|||||||
assert "total_upserted" in src
|
assert "total_upserted" in src
|
||||||
assert 'result["total_upserted"] +=' in src, "total_upserted 必须真的被累加"
|
assert 'result["total_upserted"] +=' in src, "total_upserted 必须真的被累加"
|
||||||
assert "Standing(" in src, "必须真的构造 Standing"
|
assert "Standing(" in src, "必须真的构造 Standing"
|
||||||
# P0-02: 追加快照——每次 INSERT 新行(带 available_at),不查询旧行做 upsert
|
assert "select(Standing)" in src, "必须查询已有快照以决定 insert/update"
|
||||||
assert "available_at" in src, "P0-02 采集必须设置 available_at"
|
|
||||||
assert "scalar_one_or_none" not in src, "P0-02 不应再按 (league, season, team) 做 upsert 查询"
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
# ============================================================
|
||||||
|
|||||||
@@ -1,304 +0,0 @@
|
|||||||
"""standings 成功路径 Bronze 层回归测试(RawEvent + DataLineage)。
|
|
||||||
|
|
||||||
背景: events/stats 管线成功后均已补写 Bronze 层,唯独 standings 采集
|
|
||||||
成功后既不留原始载荷,也不留血缘 —— 三条管线的溯源链条在积分榜一环
|
|
||||||
缺失。本测试守护(与 test_events_bronze.py 对称):
|
|
||||||
1. 联赛成功 upsert → RawEvent(幂等键=standings:{league}:{season})
|
|
||||||
+ Lineage(target_table="standings", transform_name="standings_ingest")
|
|
||||||
2. 更新已有快照(非插入)同样写 Bronze —— 积分榜是快照,刷新即采集
|
|
||||||
3. RawEvent 幂等: 同 source_record_id 已存在则跳过,血缘照写
|
|
||||||
4. 基础设施写入失败 → 只 warning,不拖垮采集主流程
|
|
||||||
5. 抓取失败路径继续走 _safe_write_ingest_failure,且不写 Bronze
|
|
||||||
|
|
||||||
范式: 假 db(按查询实体分发预置数据 + 记录 add,flush 分配自增 id)
|
|
||||||
+ monkeypatch 抓取函数,不依赖真实数据库。
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
import src.data.bzzoiro as bz
|
|
||||||
from src.db.models import DataLineage, IngestFailure, League, RawEvent, Standing, Team, TeamAlias
|
|
||||||
|
|
||||||
|
|
||||||
def _payload():
|
|
||||||
"""构造一份最小合法的 bzzoiro /leagues/{id}/standings/ 原始载荷。"""
|
|
||||||
return {
|
|
||||||
"season": {"start_date": "2025-08-01", "end_date": "2026-05-31"},
|
|
||||||
"standings": [
|
|
||||||
{
|
|
||||||
"position": 1, "team_name": "Arsenal FC",
|
|
||||||
"played": 10, "won": 8, "drawn": 1, "lost": 1,
|
|
||||||
"gf": 22, "ga": 8, "gd": 14, "pts": 25,
|
|
||||||
"zone": {"key": "champions_league", "label": "Champions League"},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"position": 2, "team_name": "Chelsea FC",
|
|
||||||
"played": 10, "won": 6, "drawn": 2, "lost": 2,
|
|
||||||
"gf": 18, "ga": 12, "gd": 6, "pts": 20,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _patch_fetch(monkeypatch, payload):
|
|
||||||
async def _fetch(league_code, season=None):
|
|
||||||
return payload
|
|
||||||
|
|
||||||
monkeypatch.setattr(bz, "fetch_bzzoiro_standings", _fetch)
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeResult:
|
|
||||||
"""支持 .scalars().all() / .scalar_one_or_none() 的最小假结果集。"""
|
|
||||||
|
|
||||||
def __init__(self, items):
|
|
||||||
self._items = list(items)
|
|
||||||
|
|
||||||
def scalars(self):
|
|
||||||
return self
|
|
||||||
|
|
||||||
def __iter__(self):
|
|
||||||
return iter(self._items)
|
|
||||||
|
|
||||||
def all(self):
|
|
||||||
return self._items
|
|
||||||
|
|
||||||
def scalar_one_or_none(self):
|
|
||||||
return self._items[0] if self._items else None
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeDB:
|
|
||||||
"""按查询实体分发预置数据;记录 add();flush 为无 id 对象分配自增主键。"""
|
|
||||||
|
|
||||||
def __init__(self, leagues=(), teams=(), standings=(), raw_events=()):
|
|
||||||
self.added = []
|
|
||||||
self._by_entity = {
|
|
||||||
League: list(leagues),
|
|
||||||
Team: list(teams),
|
|
||||||
Standing: list(standings),
|
|
||||||
RawEvent: list(raw_events),
|
|
||||||
}
|
|
||||||
# session.get 查找表(Team/TeamAlias)
|
|
||||||
self._teams_by_id: dict[int, Team] = {t.id: t for t in teams if getattr(t, "id", None)}
|
|
||||||
self._aliases: dict[str, TeamAlias] = {}
|
|
||||||
self._next_id = max((t.id for t in teams if getattr(t, "id", None)), default=0)
|
|
||||||
|
|
||||||
def add(self, obj):
|
|
||||||
self.added.append(obj)
|
|
||||||
|
|
||||||
async def get(self, cls, key):
|
|
||||||
if cls is Team:
|
|
||||||
return self._teams_by_id.get(key)
|
|
||||||
if cls is TeamAlias:
|
|
||||||
return self._aliases.get(key)
|
|
||||||
return None
|
|
||||||
|
|
||||||
async def execute(self, stmt):
|
|
||||||
entities = set()
|
|
||||||
for d in (stmt.column_descriptions or []):
|
|
||||||
entities.add(d.get("entity") or d.get("type"))
|
|
||||||
for entity, items in self._by_entity.items():
|
|
||||||
if entity in entities:
|
|
||||||
return _FakeResult(self._filter(entity, items, stmt))
|
|
||||||
return _FakeResult([])
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _filter(entity, items, stmt):
|
|
||||||
"""RawEvent 查询按 source_record_id 过滤 —— 幂等测试需区分不同键。"""
|
|
||||||
if entity is RawEvent:
|
|
||||||
try:
|
|
||||||
params = stmt.compile().params
|
|
||||||
except Exception:
|
|
||||||
return items
|
|
||||||
rid = next((v for k, v in params.items() if "source_record_id" in k), None)
|
|
||||||
if rid is not None:
|
|
||||||
return [i for i in items if i.source_record_id == rid]
|
|
||||||
return items
|
|
||||||
|
|
||||||
async def flush(self):
|
|
||||||
for obj in self.added:
|
|
||||||
if getattr(obj, "id", None) is None:
|
|
||||||
self._next_id += 1
|
|
||||||
obj.id = self._next_id
|
|
||||||
# 同步 session.get 可查到新建 Team
|
|
||||||
if isinstance(obj, Team) and obj.id is not None:
|
|
||||||
self._teams_by_id[obj.id] = obj
|
|
||||||
|
|
||||||
|
|
||||||
def _preset_league():
|
|
||||||
lg = League(code="EPL", name="Premier League", country="England")
|
|
||||||
lg.id = 42
|
|
||||||
return lg
|
|
||||||
|
|
||||||
|
|
||||||
def _raw_events(db):
|
|
||||||
return [o for o in db.added if isinstance(o, RawEvent)]
|
|
||||||
|
|
||||||
|
|
||||||
def _lineages(db):
|
|
||||||
return [o for o in db.added if isinstance(o, DataLineage)]
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# 1. 成功 upsert → RawEvent + DataLineage
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsBronzeOnUpsert:
|
|
||||||
async def test_upsert_writes_raw_event_and_lineage(self, monkeypatch):
|
|
||||||
_patch_fetch(monkeypatch, _payload())
|
|
||||||
db = _FakeDB(leagues=[_preset_league()])
|
|
||||||
|
|
||||||
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
|
||||||
|
|
||||||
assert result["errors"] == []
|
|
||||||
assert result["total_upserted"] == 2
|
|
||||||
|
|
||||||
raws = _raw_events(db)
|
|
||||||
assert len(raws) == 1
|
|
||||||
raw = raws[0]
|
|
||||||
assert raw.source_system == "bzzoiro"
|
|
||||||
# 幂等键: 联赛 + 实际入库的赛季标签(由载荷日期推导,与 Standing.season 同口径)
|
|
||||||
assert raw.source_record_id == "standings:EPL:2025-2026"
|
|
||||||
assert raw.ingest_batch_id.startswith("bzzoiro-standings-EPL-")
|
|
||||||
# 整份原始载荷完整保留
|
|
||||||
assert raw.raw_payload["standings"][0]["team_name"] == "Arsenal FC"
|
|
||||||
|
|
||||||
lineages = _lineages(db)
|
|
||||||
assert len(lineages) == 1
|
|
||||||
lin = lineages[0]
|
|
||||||
assert lin.source_system == "bzzoiro"
|
|
||||||
assert lin.source_record_id == "standings:EPL:2025-2026"
|
|
||||||
assert lin.target_table == "standings"
|
|
||||||
assert lin.target_id == 42 # 联赛 id
|
|
||||||
assert lin.transform_name == "standings_ingest"
|
|
||||||
assert lin.transform_detail == {
|
|
||||||
"league": "EPL", "season": "2025-2026", "rows_upserted": 2,
|
|
||||||
}
|
|
||||||
# RawEvent 与 Lineage 同批次,便于按批追溯
|
|
||||||
assert lin.batch_id == raw.ingest_batch_id
|
|
||||||
|
|
||||||
async def test_updated_snapshot_also_writes_bronze(self, monkeypatch):
|
|
||||||
"""已有快照就地更新(非插入)同样是成功采集,必须留 Bronze 记录。"""
|
|
||||||
payload = _payload()
|
|
||||||
payload["standings"] = payload["standings"][:1] # 单队,便于命中同一行
|
|
||||||
_patch_fetch(monkeypatch, payload)
|
|
||||||
|
|
||||||
team = Team(name="Arsenal FC", name_zh="阿森纳")
|
|
||||||
team.id = 7
|
|
||||||
existing = Standing(league_id=42, season="2025-2026", team_id=7, position=9)
|
|
||||||
existing.points = 1
|
|
||||||
db = _FakeDB(leagues=[_preset_league()], teams=[team], standings=[existing])
|
|
||||||
|
|
||||||
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
|
||||||
|
|
||||||
assert result["total_upserted"] == 1
|
|
||||||
assert result["leagues"]["EPL"]["teams_created"] == 0
|
|
||||||
# 快照刷新也要留痕: RawEvent(幂等) + 血缘
|
|
||||||
assert len(_raw_events(db)) == 1
|
|
||||||
lineages = _lineages(db)
|
|
||||||
assert len(lineages) == 1
|
|
||||||
assert lineages[0].transform_name == "standings_ingest"
|
|
||||||
assert lineages[0].transform_detail["rows_upserted"] == 1
|
|
||||||
|
|
||||||
async def test_empty_upsert_writes_no_bronze(self, monkeypatch):
|
|
||||||
"""载荷有行但全部队名为空 → 没有任何 upsert,不应产生 RawEvent/Lineage。"""
|
|
||||||
payload = {"standings": [{"position": 1, "team_name": ""}]}
|
|
||||||
_patch_fetch(monkeypatch, payload)
|
|
||||||
db = _FakeDB(leagues=[_preset_league()])
|
|
||||||
|
|
||||||
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
|
||||||
|
|
||||||
assert result["total_upserted"] == 0
|
|
||||||
assert _raw_events(db) == []
|
|
||||||
assert _lineages(db) == []
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# 2. RawEvent 幂等: 同 source_record_id 跳过
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsRawEventIdempotent:
|
|
||||||
async def test_existing_raw_event_is_skipped(self, monkeypatch):
|
|
||||||
existing = RawEvent(
|
|
||||||
source_system="bzzoiro",
|
|
||||||
source_record_id="standings:EPL:2025-2026",
|
|
||||||
raw_payload={"old": True},
|
|
||||||
)
|
|
||||||
_patch_fetch(monkeypatch, _payload())
|
|
||||||
db = _FakeDB(leagues=[_preset_league()], raw_events=[existing])
|
|
||||||
|
|
||||||
await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
|
||||||
|
|
||||||
new_raws = [r for r in _raw_events(db) if r is not existing]
|
|
||||||
assert new_raws == []
|
|
||||||
assert len(_lineages(db)) == 1 # 血缘仍然记录本次采集
|
|
||||||
|
|
||||||
async def test_different_season_writes_new_raw_event(self, monkeypatch):
|
|
||||||
"""幂等键含赛季: 同联赛不同赛季各留一条 RawEvent。"""
|
|
||||||
payload = _payload()
|
|
||||||
payload["season"] = {"start_date": "2024-08-01", "end_date": "2025-05-31"}
|
|
||||||
existing = RawEvent(
|
|
||||||
source_system="bzzoiro",
|
|
||||||
source_record_id="standings:EPL:2025-2026",
|
|
||||||
raw_payload={"old": True},
|
|
||||||
)
|
|
||||||
_patch_fetch(monkeypatch, payload)
|
|
||||||
db = _FakeDB(leagues=[_preset_league()], raw_events=[existing])
|
|
||||||
|
|
||||||
await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
|
||||||
|
|
||||||
new_raws = [r for r in _raw_events(db) if r is not existing]
|
|
||||||
assert len(new_raws) == 1
|
|
||||||
assert new_raws[0].source_record_id == "standings:EPL:2024-2025"
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# 3. 基础设施写入失败: 尽力而为,不拖垮主流程
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsBronzeIsBestEffort:
|
|
||||||
async def test_bronze_write_failure_does_not_break_ingest(self, monkeypatch):
|
|
||||||
import src.data.bzzoiro_standings as bz_standings
|
|
||||||
|
|
||||||
async def _boom(*args, **kwargs):
|
|
||||||
raise RuntimeError("infra down")
|
|
||||||
|
|
||||||
# Bronze 写入助手直接 import 到 bzzoiro_standings 命名空间,需 patch 该处
|
|
||||||
monkeypatch.setattr(bz_standings, "_write_raw_event", _boom)
|
|
||||||
monkeypatch.setattr(bz_standings, "_write_lineage", _boom)
|
|
||||||
_patch_fetch(monkeypatch, _payload())
|
|
||||||
db = _FakeDB(leagues=[_preset_league()])
|
|
||||||
|
|
||||||
# 不应抛异常:Bronze 写不进去只记 warning
|
|
||||||
result = await bz.ingest_bzzoiro_standings(db, leagues=["EPL"])
|
|
||||||
|
|
||||||
assert result["total_upserted"] == 2
|
|
||||||
assert [o for o in db.added if isinstance(o, Standing)]
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# 4. 抓取失败: 继续写死信,且不写 Bronze
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
class TestStandingsFailurePathKeepsDeadLetter:
|
|
||||||
async def test_fetch_failure_writes_deadletter_and_no_bronze(self, monkeypatch):
|
|
||||||
async def _boom(league_code, season=None):
|
|
||||||
raise RuntimeError("upstream 500")
|
|
||||||
|
|
||||||
monkeypatch.setattr(bz, "fetch_bzzoiro_standings", _boom)
|
|
||||||
db = _FakeDB()
|
|
||||||
|
|
||||||
result = await bz.ingest_bzzoiro_standings(db, leagues=["SP1"], season="2025-2026")
|
|
||||||
|
|
||||||
assert result["errors"]
|
|
||||||
failures = [o for o in db.added if isinstance(o, IngestFailure)]
|
|
||||||
assert len(failures) == 1
|
|
||||||
assert failures[0].entity_type == "standings"
|
|
||||||
assert failures[0].error_type == "fetch_error"
|
|
||||||
# 失败路径绝不写 Bronze(没有任何成功 upsert)
|
|
||||||
assert _raw_events(db) == []
|
|
||||||
assert _lineages(db) == []
|
|
||||||
Reference in New Issue
Block a user