feat: 足球 LLM 预测服务初始提交
Profeto — 给 LLM 提供数据,让 LLM 预测足球比分。 核心模块: - FastAPI 后端 + PostgreSQL (SQLAlchemy async) - 多 Agent LLM 预测 (5 专家 + 终裁) - 数据采集 (bzzoiro / understat / injuries) - React 前端 (Vite + Tailwind) 包含: - 数据源抽象 (DataSource 协议 + 注册表) - Alembic 数据库迁移 - Prompt 模板 (单/多 Agent) - 核心路径单元测试
This commit is contained in:
shangfangjian
@@ -0,0 +1,226 @@
|
||||
"""Bzzoiro 数据源:抓取 + 入库。
|
||||
|
||||
迁移自旧项目 app/data/sources/bzzoiro/,改成 async + 简化入库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json as _json
|
||||
import logging
|
||||
import time as _time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from src.core.config import settings
|
||||
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
|
||||
from src.data.match_lookup import find_existing_match, get_or_create_team
|
||||
from src.data.normalize import normalize_bzzoiro
|
||||
from src.data.sources import register
|
||||
from src.db.models import League, Match, MatchStats
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _fetch_json_sync(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
|
||||
"""同步 HTTP(bzzoiro 客户端保持同步,在 async 函数里 run_in_executor)。"""
|
||||
base = settings.BZZOIRO_BASE.rstrip("/")
|
||||
url = f"{base}/{path.lstrip('/')}"
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
key = settings.BZZOIRO_KEY
|
||||
if not key:
|
||||
raise RuntimeError("BZZOIRO_KEY 未设置")
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
req.add_header("Authorization", f"Token {key}")
|
||||
req.add_header("Accept", "application/json")
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
return _json.loads(resp.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code == 429:
|
||||
logger.warning("bzzoiro 429, retry %d", attempt + 1)
|
||||
_time.sleep(1)
|
||||
continue
|
||||
raise
|
||||
raise RuntimeError("bzzoiro rate limit exceeded")
|
||||
|
||||
|
||||
async def fetch_bzzoiro_events(
|
||||
league_code: str,
|
||||
*,
|
||||
status: str = "finished",
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> list[dict]:
|
||||
"""抓取 bzzoiro 原始事件(异步包装)。"""
|
||||
league_id = BZZOIRO_LEAGUE_IDS.get(league_code)
|
||||
if league_id is None:
|
||||
raise ValueError(f"未知联赛代码: {league_code}")
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
rows: list[dict] = []
|
||||
offset = 0
|
||||
payload: dict | list = {}
|
||||
while True:
|
||||
params: dict = {
|
||||
"league_id": league_id,
|
||||
"status": status,
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
}
|
||||
if date_from:
|
||||
params["date_from"] = str(date_from)[:10]
|
||||
if date_to:
|
||||
params["date_to"] = str(date_to)[:10]
|
||||
# 显式位置参数,避免 lambda 闭包捕获循环变量
|
||||
payload = await loop.run_in_executor(None, _fetch_json_sync, "/events/", params)
|
||||
batch = payload.get("results") or []
|
||||
if not batch:
|
||||
break
|
||||
rows.extend(batch)
|
||||
total = payload.get("total")
|
||||
offset += limit
|
||||
if total is not None and offset >= total:
|
||||
break
|
||||
if len(batch) < limit:
|
||||
break
|
||||
await asyncio.sleep(REQUEST_INTERVAL)
|
||||
return rows
|
||||
|
||||
|
||||
@register
|
||||
class BzzoiroSource:
|
||||
"""bzzoiro 数据源(实现 DataSource 协议)。"""
|
||||
|
||||
name = "bzzoiro"
|
||||
|
||||
async def ingest(
|
||||
self,
|
||||
db,
|
||||
*,
|
||||
leagues: Iterable[str],
|
||||
date_from: str | None = None,
|
||||
date_to: str | None = None,
|
||||
status: str = "finished",
|
||||
) -> dict:
|
||||
"""采集 bzzoiro → 入库。返回统计。"""
|
||||
result: dict = {"leagues": {}, "total_inserted": 0, "total_updated": 0, "errors": []}
|
||||
|
||||
for code in leagues:
|
||||
league_r: dict = {"inserted": 0, "updated": 0, "errors": []}
|
||||
try:
|
||||
raw_events = await fetch_bzzoiro_events(code, status=status, date_from=date_from, date_to=date_to)
|
||||
except Exception as e:
|
||||
logger.exception("bzzoiro fetch failed for %s", code)
|
||||
league_r["errors"].append(f"fetch failed: {e}")
|
||||
result["leagues"][code] = league_r
|
||||
continue
|
||||
|
||||
# 获取或创建联赛
|
||||
stmt = select(League).where(League.code == code)
|
||||
league = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if league is None:
|
||||
league = League(code=code, name=LEAGUE_NAMES.get(code, code), country=LEAGUE_COUNTRIES.get(code))
|
||||
db.add(league)
|
||||
await db.flush()
|
||||
|
||||
for raw in raw_events:
|
||||
try:
|
||||
nm = normalize_bzzoiro(raw, code)
|
||||
if nm is None:
|
||||
continue
|
||||
nm.validate()
|
||||
except Exception as e:
|
||||
logger.debug("normalize skip: %s", e)
|
||||
league_r["errors"].append(f"normalize: {e}")
|
||||
continue
|
||||
|
||||
# 球队
|
||||
home_team = await get_or_create_team(db, nm.home_team)
|
||||
away_team = await get_or_create_team(db, nm.away_team)
|
||||
|
||||
# 查找已有比赛(天级匹配)
|
||||
existing = await find_existing_match(db, league.id, nm.home_team, nm.away_team, nm.date)
|
||||
|
||||
if existing is None:
|
||||
m = Match(
|
||||
league_id=league.id,
|
||||
season=nm.season_label or None,
|
||||
home_team_id=home_team.id,
|
||||
away_team_id=away_team.id,
|
||||
match_date=nm.date,
|
||||
match_date_date=nm.date.date() if hasattr(nm.date, "date") else nm.date,
|
||||
match_status=nm.match_status,
|
||||
home_goals=nm.home_goals,
|
||||
away_goals=nm.away_goals,
|
||||
home_ht_goals=nm.home_ht_goals,
|
||||
away_ht_goals=nm.away_ht_goals,
|
||||
match_stage=nm.match_stage,
|
||||
)
|
||||
db.add(m)
|
||||
await db.flush()
|
||||
if nm.home_xg is not None or nm.away_xg is not None:
|
||||
stats = MatchStats(
|
||||
match_id=m.id,
|
||||
home_xg=nm.home_xg,
|
||||
away_xg=nm.away_xg,
|
||||
home_shots=nm.home_shots,
|
||||
away_shots=nm.away_shots,
|
||||
home_shots_on_target=nm.home_shots_on_target,
|
||||
away_shots_on_target=nm.away_shots_on_target,
|
||||
home_corners=nm.home_corners,
|
||||
away_corners=nm.away_corners,
|
||||
home_possession=nm.home_possession,
|
||||
home_yellow_cards=nm.home_yellow_cards,
|
||||
away_yellow_cards=nm.away_yellow_cards,
|
||||
home_red_cards=nm.home_red_cards,
|
||||
away_red_cards=nm.away_red_cards,
|
||||
)
|
||||
db.add(stats)
|
||||
league_r["inserted"] += 1
|
||||
else:
|
||||
# 更新(只补空 / 状态升级)
|
||||
changed = False
|
||||
if existing.match_status != nm.match_status and nm.match_status == "finished":
|
||||
existing.match_status = nm.match_status
|
||||
changed = True
|
||||
if existing.home_goals is None and nm.home_goals is not None:
|
||||
existing.home_goals = nm.home_goals
|
||||
existing.away_goals = nm.away_goals
|
||||
existing.home_ht_goals = nm.home_ht_goals
|
||||
existing.away_ht_goals = nm.away_ht_goals
|
||||
changed = True
|
||||
if existing.match_stage is None and nm.match_stage:
|
||||
existing.match_stage = nm.match_stage
|
||||
changed = True
|
||||
# stats 只补空
|
||||
if existing.stats is None and (nm.home_xg is not None or nm.away_xg is not None):
|
||||
existing.stats = MatchStats(match_id=existing.id)
|
||||
db.add(existing.stats)
|
||||
await db.flush()
|
||||
if existing.stats is not None:
|
||||
for fld in ("home_xg", "away_xg", "home_shots", "away_shots",
|
||||
"home_shots_on_target", "away_shots_on_target",
|
||||
"home_corners", "away_corners", "home_possession",
|
||||
"home_yellow_cards", "away_yellow_cards",
|
||||
"home_red_cards", "away_red_cards"):
|
||||
if getattr(existing.stats, fld, None) is None:
|
||||
v = getattr(nm, fld, None)
|
||||
if v is not None:
|
||||
setattr(existing.stats, fld, v)
|
||||
changed = True
|
||||
if changed:
|
||||
league_r["updated"] += 1
|
||||
|
||||
await db.commit()
|
||||
result["leagues"][code] = league_r
|
||||
result["total_inserted"] += league_r["inserted"]
|
||||
result["total_updated"] += league_r["updated"]
|
||||
return result
|
||||
Reference in New Issue
Block a user