feat: 足球 LLM 预测服务初始提交
Profeto — 给 LLM 提供数据,让 LLM 预测足球比分。 核心模块: - FastAPI 后端 + PostgreSQL (SQLAlchemy async) - 多 Agent LLM 预测 (5 专家 + 终裁) - 数据采集 (bzzoiro / understat / injuries) - React 前端 (Vite + Tailwind) 包含: - 数据源抽象 (DataSource 协议 + 注册表) - Alembic 数据库迁移 - Prompt 模板 (单/多 Agent) - 核心路径单元测试
This commit is contained in:
shangfangjian
@@ -0,0 +1,226 @@
|
||||
"""Bzzoiro 数据源:抓取 + 入库。
|
||||
|
||||
迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json as _json
|
||||
import logging
|
||||
import time as _time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.core.config import settings
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||
from src.data.match_lookup import find_existing_match, get_or_create_team
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||
"""同步 HTTP(bzzoiro 客户端保持同步,在 async 函数里 run_in_executor)。"""
|
||||
base = settings.BZZOIRO_BASE.rstrip("/")
|
||||
url = f"{base}/{path.lstrip('/')}"
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
key = settings.BZZOIRO_KEY
|
||||
if not key:
|
||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", f"Token {key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return _json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
logger.warning("bzzoiro 429, retry %d", attempt + 1)
|
||||
_time.sleep(1)
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("bzzoiro rate limit exceeded")
|
||||
|
||||
|
||||
async def fetch_bzzoiro_events(
|
||||
league_code: str,
|
||||
*,
|
||||
status: str = "finished",
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> list[dict]:
|
||||
"""抓取 bzzoiro 原始事件(异步包装)。"""
|
||||
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
||||
if league_id is None:
|
||||
raise ValueError(f"未知联赛代码: {league_code}")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
rows: list[dict] = []
|
||||
offset = 0
|
||||
payload: dict | list = {}
|
||||
while True:
|
||||
params: dict = {
|
||||
"league_id": league_id,
|
||||
"status": status,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
if date_from:
|
||||
params["date_from"] = str(date_from)[:10]
|
||||
if date_to:
|
||||
params["date_to"] = str(date_to)[:10]
|
||||
# 显式位置参数,避免 lambda 闭包捕获循环变量
|
||||
payload = await loop.run_in_executor(None, _fetch_json_sync, "/events/", params)
|
||||
batch = payload.get("results") or []
|
||||
if not batch:
|
||||
break
|
||||
rows.extend(batch)
|
||||
total = payload.get("total")
|
||||
offset += limit
|
||||
if total is not None and offset >= total:
|
||||
break
|
||||
if len(batch) < limit:
|
||||
break
|
||||
await asyncio.sleep(REQUEST_INTERVAL)
|
||||
return rows
|
||||
|
||||
|
||||
@register
|
||||
class BzzoiroSource:
|
||||
"""bzzoiro 数据源(实现 DataSource 协议)。"""
|
||||
|
||||
name = "bzzoiro"
|
||||
|
||||
async def ingest(
|
||||
self,
|
||||
db,
|
||||
*,
|
||||
leagues: Iterable[str],
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
status: str = "finished",
|
||||
) -> dict:
|
||||
"""采集 bzzoiro → 入库。返回统计。"""
|
||||
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||
|
||||
for code in leagues:
|
||||
league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
|
||||
try:
|
||||
raw_events = await fetch_bzzoiro_events(code, status=status, date_from=date_from, date_to=date_to)
|
||||
except Exception as e:
|
||||
logger.exception("bzzoiro fetch failed for %s", code)
|
||||
league_r["errors"].append(f"fetch failed: {e}")
|
||||
result["leagues"][code] = league_r
|
||||
continue
|
||||
|
||||
# 获取或创建联赛
|
||||
stmt = select(League).where(League.code == code)
|
||||
league = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if league is None:
|
||||
league = League(code=code, name=LEAGUE_NAMES.get(code, code), country=LEAGUE_COUNTRIES.get(code))
|
||||
db.add(league)
|
||||
await db.flush()
|
||||
|
||||
for raw in raw_events:
|
||||
try:
|
||||
nm = normalize_bzzoiro(raw, code)
|
||||
if nm is None:
|
||||
continue
|
||||
nm.validate()
|
||||
except Exception as e:
|
||||
logger.debug("normalize skip: %s", e)
|
||||
league_r["errors"].append(f"normalize: {e}")
|
||||
continue
|
||||
|
||||
# 球队
|
||||
home_team = await get_or_create_team(db, nm.home_team)
|
||||
away_team = await get_or_create_team(db, nm.away_team)
|
||||
|
||||
# 查找已有比赛(天级匹配)
|
||||
existing = await find_existing_match(db, league.id, nm.home_team, nm.away_team, nm.date)
|
||||
|
||||
if existing is None:
|
||||
m = Match(
|
||||
league_id=league.id,
|
||||
season=nm.season_label or None,
|
||||
home_team_id=home_team.id,
|
||||
away_team_id=away_team.id,
|
||||
match_date=nm.date,
|
||||
match_date_date=nm.date.date() if hasattr(nm.date, "date") else nm.date,
|
||||
match_status=nm.match_status,
|
||||
home_goals=nm.home_goals,
|
||||
away_goals=nm.away_goals,
|
||||
home_ht_goals=nm.home_ht_goals,
|
||||
away_ht_goals=nm.away_ht_goals,
|
||||
match_stage=nm.match_stage,
|
||||
)
|
||||
db.add(m)
|
||||
await db.flush()
|
||||
if nm.home_xg is not None or nm.away_xg is not None:
|
||||
stats = MatchStats(
|
||||
match_id=m.id,
|
||||
home_xg=nm.home_xg,
|
||||
away_xg=nm.away_xg,
|
||||
home_shots=nm.home_shots,
|
||||
away_shots=nm.away_shots,
|
||||
home_shots_on_target=nm.home_shots_on_target,
|
||||
away_shots_on_target=nm.away_shots_on_target,
|
||||
home_corners=nm.home_corners,
|
||||
away_corners=nm.away_corners,
|
||||
home_possession=nm.home_possession,
|
||||
home_yellow_cards=nm.home_yellow_cards,
|
||||
away_yellow_cards=nm.away_yellow_cards,
|
||||
home_red_cards=nm.home_red_cards,
|
||||
away_red_cards=nm.away_red_cards,
|
||||
)
|
||||
db.add(stats)
|
||||
league_r["inserted"] += 1
|
||||
else:
|
||||
# 更新(只补空 / 状态升级)
|
||||
changed = False
|
||||
if existing.match_status != nm.match_status and nm.match_status == "finished":
|
||||
existing.match_status = nm.match_status
|
||||
changed = True
|
||||
if existing.home_goals is None and nm.home_goals is not None:
|
||||
existing.home_goals = nm.home_goals
|
||||
existing.away_goals = nm.away_goals
|
||||
existing.home_ht_goals = nm.home_ht_goals
|
||||
existing.away_ht_goals = nm.away_ht_goals
|
||||
changed = True
|
||||
if existing.match_stage is None and nm.match_stage:
|
||||
existing.match_stage = nm.match_stage
|
||||
changed = True
|
||||
# stats 只补空
|
||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
existing.stats = MatchStats(match_id=existing.id)
|
||||
db.add(existing.stats)
|
||||
await db.flush()
|
||||
if existing.stats is not None:
|
||||
for fld in ("home_xg", "away_xg", "home_shots", "away_shots",
|
||||
"home_shots_on_target", "away_shots_on_target",
|
||||
"home_corners", "away_corners", "home_possession",
|
||||
"home_yellow_cards", "away_yellow_cards",
|
||||
"home_red_cards", "away_red_cards"):
|
||||
if getattr(existing.stats, fld, None) is None:
|
||||
v = getattr(nm, fld, None)
|
||||
if v is not None:
|
||||
setattr(existing.stats, fld, v)
|
||||
changed = True
|
||||
if changed:
|
||||
league_r["updated"] += 1
|
||||
|
||||
await db.commit()
|
||||
result["leagues"][code] = league_r
|
||||
result["total_inserted"] += league_r["inserted"]
|
||||
result["total_updated"] += league_r["updated"]
|
||||
return result
|
||||
@@ -0,0 +1,46 @@
|
||||
"""数据源配置常量(联赛映射)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
# fdco 风格代码 → bzzoiro league_id
|
||||
BZZOIRO_LEAGUE_IDS: dict[str, int] = {
|
||||
"E0": 1, # Premier League
|
||||
"SP1": 3, # La Liga
|
||||
"D1": 5, # Bundesliga
|
||||
"I1": 4, # Serie A
|
||||
"F1": 6, # Ligue 1
|
||||
"CL": 7, # Champions League
|
||||
"EL": 8, # Europa League
|
||||
}
|
||||
|
||||
# fdco 代码 → understat 联赛代码
|
||||
FDCO_TO_UNDERSTAT: dict[str, str] = {
|
||||
"E0": "EPL",
|
||||
"SP1": "La_liga",
|
||||
"D1": "Bundesliga",
|
||||
"I1": "Serie_A",
|
||||
"F1": "Ligue_1",
|
||||
}
|
||||
|
||||
# fdco 代码 → 显示名
|
||||
LEAGUE_NAMES: dict[str, str] = {
|
||||
"E0": "Premier League",
|
||||
"SP1": "La Liga",
|
||||
"D1": "Bundesliga",
|
||||
"I1": "Serie A",
|
||||
"F1": "Ligue 1",
|
||||
"CL": "Champions League",
|
||||
"EL": "Europa League",
|
||||
}
|
||||
|
||||
# fdco 代码 → 国家
|
||||
LEAGUE_COUNTRIES: dict[str, str] = {
|
||||
"E0": "England",
|
||||
"SP1": "Spain",
|
||||
"D1": "Germany",
|
||||
"I1": "Italy",
|
||||
"F1": "France",
|
||||
"CL": "Europe",
|
||||
"EL": "Europe",
|
||||
}
|
||||
|
||||
REQUEST_INTERVAL = 1.2 # bzzoiro 限速(秒)
|
||||
@@ -0,0 +1,176 @@
|
||||
"""伤停数据采集器(api-football / api-sports.io)。
|
||||
|
||||
采集伤停数据并入库(injuries 表),供 injuries agent 使用。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from src.core.config import settings
|
||||
from src.core.http_client import get_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
API_BASE = "https://v3.football.api-sports.io"
|
||||
DEFAULT_HOST = "v3.football.api-sports.io"
|
||||
|
||||
# 缓存目录
|
||||
_CACHE_DIR = Path(__file__).resolve().parent.parent.parent / "data" / "injuries_cache"
|
||||
|
||||
|
||||
async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = None, league_id: int | None = None) -> list[dict]:
|
||||
"""采集伤停数据。
|
||||
|
||||
Args:
|
||||
date: 日期 (YYYY-MM-DD),返当天全部伤停
|
||||
fixture_id: 指定比赛 ID
|
||||
league_id: 指定联赛 ID
|
||||
|
||||
Returns:
|
||||
伤停记录列表
|
||||
"""
|
||||
api_key = settings.API_FOOTBALL_KEY
|
||||
if not api_key:
|
||||
raise RuntimeError("API_FOOTBALL_KEY 未设置")
|
||||
|
||||
cache_dir = _CACHE_DIR
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 缓存命中
|
||||
cache_key = f"injuries_{date}_{fixture_id}_{league_id}.json"
|
||||
cache_file = cache_dir / cache_key
|
||||
if cache_file.exists():
|
||||
logger.debug("injuries cache hit: %s", cache_key)
|
||||
with open(cache_file, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
headers = {
|
||||
"x-apisports-key": api_key,
|
||||
"x-rapidapi-host": DEFAULT_HOST,
|
||||
}
|
||||
params: dict[str, Any] = {}
|
||||
if date:
|
||||
params["date"] = date
|
||||
if fixture_id:
|
||||
params["fixture"] = fixture_id
|
||||
if league_id:
|
||||
params["league"] = league_id
|
||||
|
||||
url = f"{API_BASE}/injuries"
|
||||
client = get_client()
|
||||
resp = await client.get(url, headers=headers, params=params)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
injuries = data.get("response", [])
|
||||
|
||||
# 写缓存
|
||||
with open(cache_file, "w", encoding="utf-8") as f:
|
||||
json.dump(injuries, ensure_ascii=False, default=str, fp=f)
|
||||
|
||||
return injuries
|
||||
|
||||
|
||||
async def ingest_injuries(db, *, date: str | None = None) -> dict:
|
||||
"""采集伤停数据并入库(injuries 表)。"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.data.team_names import normalize as normalize_name
|
||||
from src.db.models import Injury, Team
|
||||
|
||||
result = {"count": 0, "inserted": 0, "errors": []}
|
||||
|
||||
try:
|
||||
raw_injuries = await fetch_injuries(date=date)
|
||||
except Exception as e:
|
||||
logger.exception("injuries fetch failed")
|
||||
result["errors"].append(f"fetch failed: {e}")
|
||||
return result
|
||||
|
||||
result["count"] = len(raw_injuries)
|
||||
|
||||
# 预加载所有球队(用于按名匹配)
|
||||
teams = (await db.execute(select(Team))).scalars().all()
|
||||
team_by_name = {t.name: t.id for t in teams}
|
||||
|
||||
for raw in raw_injuries:
|
||||
try:
|
||||
player = raw.get("player", {}) or {}
|
||||
team = raw.get("team", {}) or {}
|
||||
fixture = raw.get("fixture", {}) or {}
|
||||
|
||||
player_name = player.get("name", "")
|
||||
team_name = normalize_name(team.get("name", ""))
|
||||
team_id = team_by_name.get(team_name)
|
||||
|
||||
# 解析日期
|
||||
fixture_date = fixture.get("date")
|
||||
injury_date = None
|
||||
if fixture_date:
|
||||
try:
|
||||
dt = datetime.fromisoformat(fixture_date.replace("Z", "+00:00"))
|
||||
injury_date = dt.date()
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
player_id = player.get("id")
|
||||
fixture_id = fixture.get("id")
|
||||
|
||||
# 幂等: 已存在则跳过
|
||||
existing = (
|
||||
await db.execute(
|
||||
select(Injury).where(
|
||||
Injury.player_id == player_id,
|
||||
Injury.fixture_id == fixture_id,
|
||||
Injury.injury_type == player.get("type"),
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if existing is not None:
|
||||
continue
|
||||
|
||||
injury = Injury(
|
||||
player_id=player_id,
|
||||
player_name=player_name,
|
||||
team_id=team_id,
|
||||
fixture_id=fixture_id,
|
||||
league_id=(raw.get("league") or {}).get("id"),
|
||||
injury_type=player.get("type"),
|
||||
reason=player.get("reason"),
|
||||
injury_date=injury_date,
|
||||
)
|
||||
db.add(injury)
|
||||
result["inserted"] += 1
|
||||
except Exception as e:
|
||||
result["errors"].append(f"parse error: {e}")
|
||||
|
||||
await db.commit()
|
||||
logger.info("injuries: fetched %d, inserted %d for %s", result["count"], result["inserted"], date)
|
||||
return result
|
||||
|
||||
|
||||
async def get_injuries_for_match(db, team_id: int, match_date) -> list[Injury]:
|
||||
"""查询某场比赛前某队的伤停名单(比赛日仍缺阵的)。"""
|
||||
from sqlalchemy import and_, or_, select
|
||||
|
||||
from src.db.models import Injury
|
||||
|
||||
if hasattr(match_date, "date"):
|
||||
match_date = match_date.date()
|
||||
|
||||
stmt = (
|
||||
select(Injury)
|
||||
.where(Injury.team_id == team_id)
|
||||
.where(Injury.injury_date <= match_date)
|
||||
.where(or_(Injury.return_date.is_(None), Injury.return_date >= match_date))
|
||||
.order_by(Injury.injury_date.desc())
|
||||
)
|
||||
result = await db.execute(stmt)
|
||||
return list(result.scalars().all())
|
||||
@@ -0,0 +1,40 @@
|
||||
"""比赛匹配辅助函数(多数据源共用)。
|
||||
|
||||
bzzoiro / understat 等数据源在入库时都需要:
|
||||
- 按队名获取或创建球队(get_or_create_team)
|
||||
- 按联赛+主队+客队+日期找已有比赛(find_existing_match)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from src.db.models import Match, Team
|
||||
|
||||
|
||||
async def get_or_create_team(db, name: str) -> Team:
|
||||
"""按名获取球队,不存在则创建。"""
|
||||
stmt = select(Team).where(Team.name == name)
|
||||
team = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if team is None:
|
||||
team = Team(name=name)
|
||||
db.add(team)
|
||||
await db.flush()
|
||||
return team
|
||||
|
||||
|
||||
async def find_existing_match(db, league_id: int, home_name: str, away_name: str, date) -> Match | None:
|
||||
"""按联赛+主队+客队+日期找已有比赛(天级匹配,避免时间精度差异)。"""
|
||||
home_team = (await db.execute(select(Team).where(Team.name == home_name))).scalar_one_or_none()
|
||||
away_team = (await db.execute(select(Team).where(Team.name == away_name))).scalar_one_or_none()
|
||||
if home_team is None or away_team is None:
|
||||
return None
|
||||
|
||||
date_only = date.date() if hasattr(date, "date") else date
|
||||
stmt = (
|
||||
select(Match)
|
||||
.where(Match.league_id == league_id)
|
||||
.where(Match.home_team_id == home_team.id)
|
||||
.where(Match.away_team_id == away_team.id)
|
||||
.where(func.date(Match.match_date) == date_only)
|
||||
)
|
||||
return (await db.execute(stmt)).scalar_one_or_none()
|
||||
@@ -0,0 +1,209 @@
|
||||
"""数据规范化:任意数据源原始记录 → NormalizedMatch。
|
||||
|
||||
迁移自旧项目 app/data/normalize.py,简化:
|
||||
- 去掉 XGBackfill 双轨(不再需要独立回填)
|
||||
- 去掉 PIT 时间契约(无训练集要防泄漏)
|
||||
- 保留核心清洗契约(队名归一、日期解析、数值范围)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
VALID_STATUS = {"finished", "scheduled", "in_play", "paused", "postponed", "cancelled", "suspended"}
|
||||
|
||||
STATUS_MAP = {
|
||||
"finished": "finished", "completed": "finished", "done": "finished", "awarded": "finished",
|
||||
"scheduled": "scheduled", "upcoming": "scheduled",
|
||||
"in_play": "in_play", "live": "in_play",
|
||||
"paused": "paused", "postponed": "postponed",
|
||||
"cancelled": "cancelled", "canceled": "cancelled", "abandoned": "cancelled",
|
||||
"suspended": "suspended",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizedMatch:
|
||||
"""清洗后的统一比赛记录(入库中间格式)。"""
|
||||
|
||||
league_type: str
|
||||
date: datetime
|
||||
home_team: str
|
||||
away_team: str
|
||||
match_status: str = "finished"
|
||||
home_goals: int | None = None
|
||||
away_goals: int | None = None
|
||||
season_label: str = ""
|
||||
home_xg: float | None = None
|
||||
away_xg: float | None = None
|
||||
home_shots: int | None = None
|
||||
away_shots: int | None = None
|
||||
home_shots_on_target: int | None = None
|
||||
away_shots_on_target: int | None = None
|
||||
home_corners: int | None = None
|
||||
away_corners: int | None = None
|
||||
home_possession: float | None = None
|
||||
home_yellow_cards: int | None = None
|
||||
away_yellow_cards: int | None = None
|
||||
home_red_cards: int | None = None
|
||||
away_red_cards: int | None = None
|
||||
home_ht_goals: int | None = None
|
||||
away_ht_goals: int | None = None
|
||||
match_stage: str | None = None
|
||||
|
||||
def validate(self) -> None:
|
||||
"""完整数据契约校验。"""
|
||||
if self.match_status == "finished" and (self.home_goals is None or self.away_goals is None):
|
||||
raise ValueError(f"Finished match must have score: {self.home_team} vs {self.away_team}")
|
||||
|
||||
def _finite(n, v):
|
||||
if v is not None and isinstance(v, float) and not math.isfinite(v):
|
||||
raise ValueError(f"{n} must be finite, got {v}")
|
||||
|
||||
def _range(n, v, lo, hi):
|
||||
if v is not None and not (lo <= v <= hi):
|
||||
raise ValueError(f"{n} out of range [{lo}, {hi}]: {v}")
|
||||
|
||||
for side in ("home", "away"):
|
||||
_finite(f"{side}_goals", getattr(self, f"{side}_goals"))
|
||||
_range(f"{side}_goals", getattr(self, f"{side}_goals"), 0, 30)
|
||||
_finite(f"{side}_xg", getattr(self, f"{side}_xg"))
|
||||
_range(f"{side}_xg", getattr(self, f"{side}_xg"), 0, 20)
|
||||
for fld in ("shots", "shots_on_target", "corners"):
|
||||
_range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 100)
|
||||
for fld in ("yellow_cards", "red_cards"):
|
||||
_range(f"{side}_{fld}", getattr(self, f"{side}_{fld}"), 0, 20)
|
||||
_range("home_possession", self.home_possession, 0, 100)
|
||||
if self.home_ht_goals is not None and self.home_goals is not None and self.home_ht_goals > self.home_goals:
|
||||
raise ValueError(f"home_ht_goals({self.home_ht_goals}) > home_goals({self.home_goals})")
|
||||
if self.away_ht_goals is not None and self.away_goals is not None and self.away_ht_goals > self.away_goals:
|
||||
raise ValueError(f"away_ht_goals({self.away_ht_goals}) > away_goals({self.away_goals})")
|
||||
|
||||
|
||||
def derive_season_label(date: datetime) -> str:
|
||||
y = date.year
|
||||
return f"{y}-{y + 1}" if date.month >= 8 else f"{y - 1}-{y}"
|
||||
|
||||
|
||||
def _parse_date(value) -> datetime | None:
|
||||
"""日期解析 → UTC datetime(带 tzinfo)。"""
|
||||
if value in (None, ""):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return datetime.fromtimestamp(value, tz=timezone.utc)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
iso_s = s[:-1] + "+00:00" if s.endswith("Z") else s
|
||||
try:
|
||||
dt = datetime.fromisoformat(iso_s)
|
||||
if dt.tzinfo is not None:
|
||||
return dt.astimezone(timezone.utc)
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
pass
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d", "%d/%m/%Y", "%d/%m/%y"):
|
||||
try:
|
||||
return datetime.strptime(s[:19], fmt).replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _to_int(v) -> int | None:
|
||||
if v is None or (isinstance(v, str) and v.strip() in ("", "-")):
|
||||
return None
|
||||
if isinstance(v, bool):
|
||||
return None
|
||||
if isinstance(v, int):
|
||||
return v
|
||||
try:
|
||||
f = float(str(v).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if not f.is_integer():
|
||||
return None
|
||||
return int(f)
|
||||
|
||||
|
||||
def _to_float(v) -> float | None:
|
||||
if v is None or (isinstance(v, str) and v.strip() in ("", "-")):
|
||||
return None
|
||||
try:
|
||||
return float(str(v).strip())
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def normalize_bzzoiro(raw: dict, league_type: str) -> NormalizedMatch | None:
|
||||
"""bzzoiro event → NormalizedMatch。"""
|
||||
from src.data.team_names import normalize as normalize_name
|
||||
|
||||
date = _parse_date(raw.get("event_date"))
|
||||
if date is None:
|
||||
return None
|
||||
raw_status = str(raw.get("status", "")).lower()
|
||||
status = STATUS_MAP.get(raw_status)
|
||||
if status is None:
|
||||
return None
|
||||
home = normalize_name(raw.get("home_team", ""))
|
||||
away = normalize_name(raw.get("away_team", ""))
|
||||
if not home or not away or home == away:
|
||||
return None
|
||||
m = NormalizedMatch(
|
||||
league_type=league_type,
|
||||
date=date,
|
||||
home_team=home,
|
||||
away_team=away,
|
||||
match_status=status,
|
||||
season_label=derive_season_label(date),
|
||||
)
|
||||
m.home_goals = _to_int(raw.get("home_score", raw.get("home_goals")))
|
||||
m.away_goals = _to_int(raw.get("away_score", raw.get("away_goals")))
|
||||
m.home_ht_goals = _to_int(raw.get("home_score_ht", raw.get("home_ht_goals")))
|
||||
m.away_ht_goals = _to_int(raw.get("away_score_ht", raw.get("away_ht_goals")))
|
||||
_rn = _to_int(raw.get("round_number"))
|
||||
_rn_name = str(raw.get("round_name") or "").strip()
|
||||
if _rn_name:
|
||||
m.match_stage = _rn_name
|
||||
elif _rn:
|
||||
m.match_stage = f"第 {_rn} 轮"
|
||||
if m.match_status == "finished" and m.home_goals is None:
|
||||
m.match_status = "scheduled"
|
||||
return m
|
||||
|
||||
|
||||
def normalize_understat(raw: dict, league_type: str) -> NormalizedMatch | None:
|
||||
"""understat 单场 → NormalizedMatch(仅 xG)。"""
|
||||
from src.data.team_names import normalize as normalize_name
|
||||
|
||||
dt_str = raw.get("datetime") or raw.get("date")
|
||||
if not dt_str:
|
||||
return None
|
||||
dt = _parse_date(dt_str)
|
||||
if dt is None:
|
||||
return None
|
||||
home_info = raw.get("h", {})
|
||||
away_info = raw.get("a", {})
|
||||
home_name = home_info.get("title", "") if isinstance(home_info, dict) else ""
|
||||
away_name = away_info.get("title", "") if isinstance(away_info, dict) else ""
|
||||
home = normalize_name(home_name)
|
||||
away = normalize_name(away_name)
|
||||
if not home or not away or home == away:
|
||||
return None
|
||||
home_xg = raw.get("xG", {}).get("h") if isinstance(raw.get("xG"), dict) else None
|
||||
away_xg = raw.get("xG", {}).get("a") if isinstance(raw.get("xG"), dict) else None
|
||||
return NormalizedMatch(
|
||||
league_type=league_type,
|
||||
date=dt,
|
||||
home_team=home,
|
||||
away_team=away,
|
||||
match_status="finished",
|
||||
season_label=derive_season_label(dt),
|
||||
home_xg=_to_float(home_xg),
|
||||
away_xg=_to_float(away_xg),
|
||||
)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""数据源协议 + 注册表。
|
||||
|
||||
定义 DataSource 契约,并提供全局注册表供路由层分发。
|
||||
每个比赛数据源实现该协议,注册后即可通过统一入口调度。
|
||||
|
||||
注: injuries 是球员级独立领域(写 Injury 表),不遵循此协议。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Protocol
|
||||
|
||||
from src.db.base import AsyncSession
|
||||
|
||||
|
||||
class DataSource(Protocol):
|
||||
"""比赛数据源契约:抓取 → 规范化 → 入库。"""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""数据源标识名(用于路由/日志)。"""
|
||||
...
|
||||
|
||||
async def ingest(self, db: AsyncSession, **kwargs) -> dict:
|
||||
"""执行完整采集流程,返回统计。"""
|
||||
...
|
||||
|
||||
|
||||
# ── 注册表 ──
|
||||
_SOURCES: dict[str, DataSource] = {}
|
||||
|
||||
|
||||
def register(source: DataSource) -> DataSource:
|
||||
"""装饰器:将数据源注册到全局注册表。"""
|
||||
_SOURCES[source.name] = source
|
||||
return source
|
||||
|
||||
|
||||
def get_source(name: str) -> DataSource:
|
||||
"""按名获取数据源。"""
|
||||
if name not in _SOURCES:
|
||||
raise ValueError(f"未知数据源: {name}")
|
||||
return _SOURCES[name]
|
||||
|
||||
|
||||
def list_sources() -> list[str]:
|
||||
"""列出所有已注册数据源名。"""
|
||||
return list(_SOURCES.keys())
|
||||
|
||||
|
||||
# ── 导入数据源触发 @register ──
|
||||
from src.data.bzzoiro import BzzoiroSource # noqa: E402, F401
|
||||
from src.data.understat import UnderstatSource # noqa: E402, F401
|
||||
@@ -0,0 +1,137 @@
|
||||
"""队名归一化:各源队名 → 统一规范名。
|
||||
|
||||
迁移自旧项目 app/data/team_names.py。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import unicodedata
|
||||
|
||||
NORMALIZE_MAP = {
|
||||
# ---- 英超 ----
|
||||
"Man City": "Manchester City",
|
||||
"Man United": "Manchester United",
|
||||
"Newcastle": "Newcastle United",
|
||||
"Nott'm Forest": "Nottingham Forest",
|
||||
"Wolves": "Wolverhampton Wanderers",
|
||||
"West Ham": "West Ham United",
|
||||
"Tottenham": "Tottenham Hotspur",
|
||||
"Spurs": "Tottenham Hotspur",
|
||||
"Brighton": "Brighton and Hove Albion",
|
||||
"West Brom": "West Bromwich Albion",
|
||||
"Stoke": "Stoke City",
|
||||
"Huddersfield": "Huddersfield Town",
|
||||
"Swansea": "Swansea City",
|
||||
"Hull": "Hull City",
|
||||
"Cardiff": "Cardiff City",
|
||||
"Luton": "Luton Town",
|
||||
"Norwich": "Norwich City",
|
||||
"Bournemouth": "AFC Bournemouth",
|
||||
"Ipswich": "Ipswich Town",
|
||||
"Leicester": "Leicester City",
|
||||
"Leeds": "Leeds United",
|
||||
"Sheffield United": "Sheffield United",
|
||||
"Southampton": "Southampton",
|
||||
"Arsenal": "Arsenal",
|
||||
"Aston Villa": "Aston Villa",
|
||||
"Brentford": "Brentford",
|
||||
"Chelsea": "Chelsea",
|
||||
"Crystal Palace": "Crystal Palace",
|
||||
"Everton": "Everton",
|
||||
"Fulham": "Fulham",
|
||||
"Liverpool": "Liverpool",
|
||||
# ---- 西甲 ----
|
||||
"Atletico Madrid": "Atlético Madrid",
|
||||
"Athletic Club": "Athletic Club",
|
||||
"Real Betis": "Real Betis",
|
||||
"Celta Vigo": "Celta Vigo",
|
||||
"Deportivo Alaves": "Deportivo Alavés",
|
||||
"Girona": "Girona",
|
||||
"Las Palmas": "Las Palmas",
|
||||
"Leganes": "Leganés",
|
||||
"Mallorca": "Mallorca",
|
||||
"Osasuna": "Osasuna",
|
||||
"Rayo Vallecano": "Rayo Vallecano",
|
||||
"Real Sociedad": "Real Sociedad",
|
||||
"Sevilla": "Sevilla",
|
||||
"Valencia": "Valencia",
|
||||
"Villarreal": "Villarreal",
|
||||
"Espanyol": "Espanyol",
|
||||
"Getafe": "Getafe",
|
||||
"Real Madrid": "Real Madrid",
|
||||
"Barcelona": "Barcelona",
|
||||
# ---- 德甲 ----
|
||||
"Bayern Munich": "Bayern München",
|
||||
"FC Koln": "FC Köln",
|
||||
"RB Leipzig": "RB Leipzig",
|
||||
"Borussia Dortmund": "Borussia Dortmund",
|
||||
"Borussia M'gladbach": "Borussia Mönchengladbach",
|
||||
"Bayer Leverkusen": "Bayer Leverkusen",
|
||||
"Eintracht Frankfurt": "Eintracht Frankfurt",
|
||||
"VfB Stuttgart": "VfB Stuttgart",
|
||||
"VfL Wolfsburg": "VfL Wolfsburg",
|
||||
"Werder Bremen": "Werder Bremen",
|
||||
"TSG Hoffenheim": "TSG Hoffenheim",
|
||||
"SC Freiburg": "SC Freiburg",
|
||||
"Union Berlin": "Union Berlin",
|
||||
"Mainz": "Mainz 05",
|
||||
"Augsburg": "FC Augsburg",
|
||||
"Bochum": "VfL Bochum",
|
||||
"Heidenheim": "1. FC Heidenheim",
|
||||
"St. Pauli": "FC St. Pauli",
|
||||
"Holstein Kiel": "Holstein Kiel",
|
||||
# ---- 意甲 ----
|
||||
"AC Milan": "AC Milan",
|
||||
"Inter": "Inter Milan",
|
||||
"Inter Milan": "Inter Milan",
|
||||
"Juventus": "Juventus",
|
||||
"Napoli": "SSC Napoli",
|
||||
"Roma": "AS Roma",
|
||||
"Lazio": "Lazio",
|
||||
"Atalanta": "Atalanta",
|
||||
"Fiorentina": "ACF Fiorentina",
|
||||
"Bologna": "Bologna",
|
||||
"Torino": "Torino",
|
||||
"Monza": "AC Monza",
|
||||
"Udinese": "Udinese",
|
||||
"Sassuolo": "Sassuolo",
|
||||
"Empoli": "Empoli",
|
||||
"Cagliari": "Cagliari",
|
||||
"Genoa": "Genoa",
|
||||
"Lecce": "Lecce",
|
||||
"Hellas Verona": "Hellas Verona",
|
||||
"Parma": "Parma",
|
||||
"Como": "Como",
|
||||
"Venezia": "Venezia",
|
||||
# ---- 法甲 ----
|
||||
"PSG": "Paris Saint-Germain",
|
||||
"Paris Saint-Germain": "Paris Saint-Germain",
|
||||
"Marseille": "Olympique Marseille",
|
||||
"Lyon": "Olympique Lyonnais",
|
||||
"Monaco": "AS Monaco",
|
||||
"Lille": "Lille OSC",
|
||||
"Nice": "OGC Nice",
|
||||
"Rennes": "Stade Rennais",
|
||||
"Lens": "RC Lens",
|
||||
"Strasbourg": "RC Strasbourg",
|
||||
"Brest": "Stade Brestois",
|
||||
"Nantes": "FC Nantes",
|
||||
"Reims": "Stade de Reims",
|
||||
"Toulouse": "Toulouse FC",
|
||||
"Montpellier": "Montpellier HSC",
|
||||
"Le Havre": "Le Havre AC",
|
||||
"Lorient": "FC Lorient",
|
||||
"Saint-Etienne": "AS Saint-Étienne",
|
||||
"Angers": "Angers SCO",
|
||||
"Auxerre": "AJ Auxerre",
|
||||
"Leganes": "Leganés",
|
||||
}
|
||||
|
||||
|
||||
def normalize(name: str) -> str:
|
||||
if not name:
|
||||
return ""
|
||||
# unicode 归一(重音)
|
||||
n = unicodedata.normalize("NFKD", name)
|
||||
n = "".join(c for c in n if not unicodedata.combining(c))
|
||||
n = n.strip()
|
||||
return NORMALIZE_MAP.get(n, n)
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Understat xG 数据源。
|
||||
|
||||
迁移自旧项目 app/data/sources/understat.py,改成 async。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from src.core.http_client import get_client
|
||||
from src.data.config import FDCO_TO_UNDERSTAT, LEAGUE_NAMES
|
||||
from src.data.match_lookup import find_existing_match
|
||||
from src.data.normalize import normalize_understat
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
UNDERSTAT_BASE = "https://understat.com/getLeagueData/{league}/{season}"
|
||||
|
||||
|
||||
async def fetch_understat(league_code: str, season: int) -> list[dict]:
|
||||
"""抓取 understat 单赛季 xG 数据。
|
||||
|
||||
Args:
|
||||
league_code: fdco 风格代码,如 'E0'
|
||||
season: 赛季起始年,如 2025 表示 2025-2026 赛季
|
||||
|
||||
Returns:
|
||||
比赛数组,每项含 datetime/h/a/xG
|
||||
"""
|
||||
understat_league = FDCO_TO_UNDERSTAT.get(league_code)
|
||||
if understat_league is None:
|
||||
raise ValueError(f"未知联赛代码: {league_code}")
|
||||
|
||||
url = UNDERSTAT_BASE.format(league=understat_league, season=season)
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Referer": f"https://understat.com/league/{understat_league}/{season}",
|
||||
}
|
||||
|
||||
client = get_client()
|
||||
resp = await client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
# understat 返回 JS 对象,需要提取 JSON
|
||||
text = resp.text
|
||||
# 匹配 var datesData = JSON.parse('...');
|
||||
match = re.search(r"var\s+datesData\s*=\s*JSON\.parse\('([^']+)'\)", text)
|
||||
if not match:
|
||||
logger.warning("understat 响应格式不符: %s...", text[:200])
|
||||
return []
|
||||
decoded = match.group(1).encode().decode("unicode_escape")
|
||||
data = json.loads(decoded)
|
||||
return data
|
||||
|
||||
|
||||
@register
|
||||
class UnderstatSource:
|
||||
"""understat xG 数据源(实现 DataSource 协议)。"""
|
||||
|
||||
name = "understat"
|
||||
|
||||
async def ingest(self, db, *, league: str, season: int) -> dict:
|
||||
"""采集 understat xG → 回填到现有 Match。只回填 xG 字段,不创建新 Match。"""
|
||||
from sqlalchemy import select
|
||||
|
||||
result = {"updated": 0, "skipped": 0, "unmatched": 0, "errors": []}
|
||||
|
||||
try:
|
||||
raw_matches = await fetch_understat(league, season)
|
||||
except Exception as e:
|
||||
logger.exception("understat fetch failed for %s %s", league, season)
|
||||
result["errors"].append(f"fetch failed: {e}")
|
||||
return result
|
||||
|
||||
# 查联赛
|
||||
stmt = select(League).where(League.code == league)
|
||||
league_obj = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if league_obj is None:
|
||||
result["errors"].append(f"league {league} not found in DB")
|
||||
return result
|
||||
|
||||
for raw in raw_matches:
|
||||
if not raw.get("isResult"):
|
||||
continue
|
||||
try:
|
||||
nm = normalize_understat(raw, league)
|
||||
if nm is None:
|
||||
result["skipped"] += 1
|
||||
continue
|
||||
except Exception as e:
|
||||
result["errors"].append(f"normalize: {e}")
|
||||
continue
|
||||
|
||||
# 匹配已有 Match(天级)
|
||||
existing = await find_existing_match(db, league_obj.id, nm.home_team, nm.away_team, nm.date)
|
||||
if existing is None:
|
||||
result["unmatched"] += 1
|
||||
continue
|
||||
|
||||
# 回填 xG
|
||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
existing.stats = MatchStats(match_id=existing.id)
|
||||
db.add(existing.stats)
|
||||
await db.flush()
|
||||
if existing.stats is not None:
|
||||
if existing.stats.home_xg is None and nm.home_xg is not None:
|
||||
existing.stats.home_xg = nm.home_xg
|
||||
result["updated"] += 1
|
||||
if existing.stats.away_xg is None and nm.away_xg is not None:
|
||||
existing.stats.away_xg = nm.away_xg
|
||||
|
||||
await db.commit()
|
||||
return result
|
||||
Reference in New Issue
Block a user