shangfangjian/Profeto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix:批量修复了一些问题

Browse Source
This commit is contained in:
shangfangjian
2026-09-19 22:51:35 +08:00
parent 835d7217d0
commit 8e6ad5394e
44 changed files with 4921 additions and 395 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llm/agents/base.py
+1 -1
View File
@@ -35,7 +35,7 @@ def load_agent_prompt(name: str, version: str = "v1") -> str:
@dataclass
class AgentSpec:
"""领域专家 agent 定义。"""
name: str # h2h / form / standings / injuries / xg
name: str # h2h / form / home_away / injuries / stats
system_prompt: str # system message
slice_fn: object # async (header, before) -> str 切片函数
src/llm/agents/orchestrator.py
+21 -5
View File
@@ -97,9 +97,12 @@ class MultiPredictResult:
reasoning: str | None
agent_outputs: list[dict]
agent_weights: dict | None
status: str = "success"
context: str
latency_ms: int | None
raw: dict | None
latency_ms: int | None = None
prompt_tokens: int | None = None
completion_tokens: int | None = None
raw: dict | None = None
async def _agent_provider(agent_id: str, *, tier: str) -> LLMProvider:
@@ -158,7 +161,10 @@ async def run_specialists(
reports: list[AgentReport] = []
for spec, r in zip(SPECIALIST_SPECS, results):
if isinstance(r, Exception):
logger.warning("agent %s raised: %s", spec.name, r)
logger.warning(
"专家调用失败 match=%s agent=%s error=%s",
header.match_id, spec.name, str(r)[:120],
)
reports.append(AgentReport(agent=spec.name, status="error", analysis=str(r)[:200]))
else:
reports.append(r)
@@ -256,8 +262,8 @@ async def predict_match_multi(
else:
# 所有专家无数据/均失败:跳过终裁,标记 degraded
logger.warning(
"match %s: 所有 %d 位专家均无有效数据,跳过终裁,标记 degraded",
match_id, len(reports),
"预测降级 match=%s mode=%s status=degraded experts=%d/%d 均无有效数据",
match_id, "multi", ok_reports, len(reports),
)
# 无有效专家时不调用 aggregator provider,避免多余开销
# model 使用 settings 默认值占位(无实际 LLM 调用)
@@ -337,6 +343,13 @@ async def predict_match_multi(
},
)
logger.info(
"预测完成 match=%s mode=%s status=%s pred=%s:%s (%s) latency=%sms, experts=%d/%d, prediction_id=%s",
match_id, "multi", pred_status,
pred.pred_home_goals, pred.pred_away_goals, pred.pred_1x2,
latency_ms, ok_reports, len(reports), pred.id,
)
return MultiPredictResult(
prediction_id=pred.id,
provider=pred.provider,
@@ -350,9 +363,12 @@ async def predict_match_multi(
pred_1x2=pred.pred_1x2,
subjective_confidence=pred.subjective_confidence,
reasoning=pred.reasoning,
status=pred_status,
agent_outputs=pred.agent_outputs,
agent_weights=agent_weights,
context=_reports_to_json(reports),
latency_ms=latency_ms,
prompt_tokens=pred.prompt_tokens,
completion_tokens=pred.completion_tokens,
raw=final,
)
src/llm/backtest.py
+13
View File
@@ -70,6 +70,8 @@ class BacktestSummary:
"""回测汇总统计。"""
total: int
scored: int
success: int = 0 # status=success 的预测数(有完整比分+1x2)
degraded: int = 0 # status=degraded 的预测数(专家失败/无有效数据)
accuracy_1x2: float | None = None
avg_score_rmse: float | None = None
avg_subjective_confidence: float | None = None
@@ -194,6 +196,17 @@ async def run_backtest(
if r is not None:
summary.results.append(r)
summary.scored += 1
# success:有完整预测比分+1x2;degraded:多专家模式无有效结论
if r.pred_1x2 is not None and r.pred_home is not None and r.pred_away is not None:
summary.success += 1
else:
summary.degraded += 1
logger.info(
"回测汇总 mode=%s total=%d scored=%d success=%d accuracy=%s%%",
mode, summary.total, summary.scored, summary.success,
f"{(sum(1 for r in summary.results if r.correct_1x2) / summary.scored * 100):.1f}" if summary.scored else "n/a",
)
# 汇总统计
if summary.scored > 0:
src/llm/baseline.py
+113
View File
@@ -0,0 +1,113 @@
"""极简基线预测:主客场场均进球估计(不调用 LLM,不产生费用)。
用于与 LLM 预测做 eval 对比。这是最简单的统计基线,仅供研究参考,
文档与 reasoning 均明确标注「非投注建议」。
"""
from __future__ import annotations
import logging
from datetime import datetime
from sqlalchemy import case, func, select
from src.db.base import AsyncSession, AsyncSessionLocal
from src.db.models import Match
logger = logging.getLogger(__name__)
async def _avg_goals(
db: AsyncSession,
*,
team_id: int,
side: str,
league_id: int,
before: datetime | None,
) -> float:
"""某队在该联赛已完赛场次的场均进球(side=home/away)。"""
if side == "home":
goals_col = Match.home_goals
team_col = Match.home_team_id
else:
goals_col = Match.away_goals
team_col = Match.away_team_id
stmt = (
select(func.avg(goals_col).label("avg_goals"), func.count().label("cnt"))
.where(
Match.match_status == "finished",
team_col == team_id,
Match.league_id == league_id,
goals_col.is_not(None),
)
)
if before is not None:
stmt = stmt.where(Match.match_date < before)
row = (await db.execute(stmt)).one()
return float(row.avg_goals) if row.avg_goals is not None and row.cnt > 0 else 0.0
async def predict_baseline(
match_id: int,
*,
backtest: bool = False,
cutoff_at: datetime | None = None,
) -> dict:
"""极简基线预测:主场场均进球 vs 客场场均进球。
返回与 PredictResult 兼容的字典:
provider=model="baseline", 不调用 LLM,latency_ms≈0。
"""
async with AsyncSessionLocal() as db:
match = await db.get(Match, match_id)
if match is None:
raise ValueError(f"match {match_id} not found")
before = None
if backtest and match.match_dt:
from datetime import timedelta
before = match.match_dt - timedelta(days=1)
elif cutoff_at is not None:
before = cutoff_at
home_avg = await _avg_goals(
db, team_id=match.home_team_id, side="home",
league_id=match.league_id, before=before,
)
away_avg = await _avg_goals(
db, team_id=match.away_team_id, side="away",
league_id=match.league_id, before=before,
)
pred_home = max(0, min(10, round(home_avg)))
pred_away = max(0, min(10, round(away_avg)))
# 主场轻微加成(可选,这里保持极简不额外加权)
if pred_home > pred_away:
pred_1x2 = "1"
elif pred_home < pred_away:
pred_1x2 = "2"
else:
pred_1x2 = "X"
return {
"pred_home_goals": float(pred_home),
"pred_away_goals": float(pred_away),
"alt_pred_home_goals": None,
"alt_pred_away_goals": None,
"pred_1x2": pred_1x2,
"subjective_confidence": 0.5,
"prompt_tokens": 0,
"completion_tokens": 0,
"reasoning": (
f"基线估计(非投注建议): 主队主场场均进球 {home_avg:.2f} → 预测 {pred_home}; "
f"客队客场场均进球 {away_avg:.2f} → 预测 {pred_away}。"
),
"provider": "baseline",
"model": "baseline",
"prompt_version": "baseline_v1",
"mode": "baseline",
"status": "success",
"latency_ms": 0,
"raw": {"home_avg": round(home_avg, 2), "away_avg": round(away_avg, 2)},
}
src/llm/eval.py
+43 -9
View File
@@ -25,6 +25,10 @@ async def settle_prediction(prediction_id: int, home_goals: int, away_goals: int
pred.actual_home_goals = home_goals
pred.actual_away_goals = away_goals
pred.settled = True
logger.info(
"结算完成 prediction_id=%s match=%s actual=%s:%s mode=%s",
prediction_id, pred.match_id, home_goals, away_goals, pred.mode or "single",
)
return pred
@@ -109,8 +113,14 @@ async def get_eval_summary(
rows = list((await session.execute(stmt)).scalars().all())
from collections import defaultdict
buckets: dict[tuple[str, str], dict] = defaultdict(lambda: {
buckets: dict[tuple[str, str, str], dict] = defaultdict(lambda: {
"total": 0, "correct_1x2": 0, "score_errors": [], "conf_sum": 0.0, "conf_count": 0,
# 置信度校准分桶(仅 settled 且 pred 完整者计入)
"conf_buckets": {
"low(0-0.5)": {"total": 0, "correct": 0},
"medium(0.5-0.7)": {"total": 0, "correct": 0},
"high(0.7-1)": {"total": 0, "correct": 0},
},
})
evaluated = 0
skipped_incomplete = 0
@@ -118,35 +128,59 @@ async def get_eval_summary(
if (p.pred_home_goals is None or p.pred_away_goals is None or p.pred_1x2 is None):
skipped_incomplete += 1
continue
key = (p.provider, p.model)
key = (p.provider, p.model, p.prompt_version or "")
b = buckets[key]
b["total"] += 1
evaluated += 1
if p.actual_home_goals is None or p.actual_away_goals is None:
continue
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
if p.pred_1x2 == actual:
b["correct_1x2"] += 1
if p.pred_home_goals is not None and p.pred_away_goals is not None:
correct = False
if p.actual_home_goals is not None and p.actual_away_goals is not None:
actual = _actual_1x2(p.actual_home_goals, p.actual_away_goals)
if p.pred_1x2 == actual:
b["correct_1x2"] += 1
correct = True
if (
p.pred_home_goals is not None and p.pred_away_goals is not None
and p.actual_home_goals is not None and p.actual_away_goals is not None
):
err = ((p.pred_home_goals - p.actual_home_goals) ** 2 +
(p.pred_away_goals - p.actual_away_goals) ** 2) ** 0.5
b["score_errors"].append(err)
if p.subjective_confidence is not None:
b["conf_sum"] += p.subjective_confidence
b["conf_count"] += 1
# 仅当有实际结果可用于校准时,才落入置信度分桶
if p.actual_home_goals is not None and p.actual_away_goals is not None:
conf = p.subjective_confidence
if conf < 0.5:
bucket = "low(0-0.5)"
elif conf < 0.7:
bucket = "medium(0.5-0.7)"
else:
bucket = "high(0.7-1)"
b["conf_buckets"][bucket]["total"] += 1
if correct:
b["conf_buckets"][bucket]["correct"] += 1
summary = []
for (prov, model), b in sorted(buckets.items()):
for (prov, model, ver), b in sorted(buckets.items()):
acc = (b["correct_1x2"] / b["total"] * 100) if b["total"] else 0
avg_err = (sum(b["score_errors"]) / len(b["score_errors"])) if b["score_errors"] else None
avg_conf = (b["conf_sum"] / b["conf_count"]) if b["conf_count"] else None
# 校准分桶 → 命中率
calibration = {}
for name, cb in b["conf_buckets"].items():
hit_rate = round(cb["correct"] / cb["total"] * 100, 1) if cb["total"] else None
calibration[name] = {"total": cb["total"], "hit_rate": hit_rate}
summary.append({
"provider": prov,
"model": model,
"prompt_version": ver or None,
"total": b["total"],
"accuracy_1x2": round(acc, 1),
"avg_score_rmse": round(avg_err, 2) if avg_err is not None else None,
"avg_subjective_confidence": round(avg_conf, 2) if avg_conf is not None else None,
"calibration": calibration,
})
return {
"summary": summary,
src/llm/predict.py
+21 -6
View File
@@ -101,8 +101,9 @@ class PredictResult:
subjective_confidence: float | None
reasoning: str | None
context: str
latency_ms: int | None
raw: dict | None
status: str = "success"
latency_ms: int | None = None
raw: dict | None = None
async def _upsert_prediction(
@@ -158,15 +159,22 @@ async def predict_match(
backtest: bool = False,
cutoff_at=None,
) -> "PredictResult | MultiPredictResult":
"""预测入口。mode=multi(默认)走多 agent;mode=single 走单次调用。
"""预测入口。mode=multi(默认)走多 agent;mode=single 走单次调用;mode=baseline 走无 LLM 基线。
Args:
use_cache: 是否允许返回进程内缓存结果。回测必须传 False——
mode: multi(默认,5 专家+终裁) / single(单次) / baseline(极简统计基线,不调用 LLM)。
use_cache:是否允许返回进程内缓存结果。回测必须传 False——
缓存命中不会新建 prediction 行,调用方会对同一个 prediction_id
反复 settle,把不同比赛的真实比分覆盖到同一条记录上。
backtest: 是否回测模式。True 时 cutoff 自动设为 match_date-1天。
cutoff_at: 显式截止时间,优先级高于 backtest 自动计算。
backtest:是否回测模式。True 时 cutoff 自动设为 match_date-1天。
cutoff_at:显式截止时间,优先级高于 backtest 自动计算。
"""
if mode == "baseline":
from src.llm.baseline import predict_baseline
return await predict_baseline(
match_id, backtest=backtest, cutoff_at=cutoff_at,
)
if mode == "single":
return await _predict_single(
match_id,
@@ -300,6 +308,7 @@ async def _predict_single(
pred_1x2=pred.pred_1x2,
subjective_confidence=pred.subjective_confidence,
reasoning=pred.reasoning,
status=pred.status,
context=ctx.text,
latency_ms=resp.latency_ms,
raw=resp.raw,
@@ -308,4 +317,10 @@ async def _predict_single(
# 5. 写入缓存(仅当允许缓存时)
if use_cache:
_set_cached(match_id, settings.LLM_PROVIDER, provider.model, version, tpl_hash, result)
logger.info(
"预测完成 match=%s mode=%s status=%s pred=%s:%s (%s) latency=%sms",
match_id, "single", "success",
validated.pred_home_goals, validated.pred_away_goals, validated.pred_1x2,
resp.latency_ms,
)
return result
src/llm/prompts/match_prediction_v2.md
-1
View File
@@ -6,7 +6,6 @@
1. 主客队近期状态差异
2. 主客场因素
3. 历史交锋心理优势
4. 联赛排名差距
严格按此 JSON 输出,不要其他内容:
```json