feat: 核心模块增强 — 加密 + 运行时配置 + 日志缓冲

- crypto.py: API Key 加密/解密工具
- runtime_config.py: 运行时动态配置管理
- log_buffer.py: 内存日志缓冲区
- config.py: 新增加密配置项
- http_client.py: 增强重试和错误处理
This commit is contained in:
shangfangjian
2026-09-19 11:58:03 +08:00
parent b3e2c52b49
commit 786f10aa11
57 changed files with 3178 additions and 488 deletions
+20 -9
View File
@@ -14,10 +14,13 @@ from datetime import datetime, timezone
from sqlalchemy import select
from src.core.config import settings
import httpx
from src.core.runtime_config import get_runtime_value
from src.core.http_client import get_client
from src.data.config import BZZOIRO_LEAGUE_IDS, LEAGUE_COUNTRIES, LEAGUE_NAMES, REQUEST_INTERVAL
from src.data.normalize import normalize_bzzoiro
from src.data.team_names_zh import zh_name
from src.data.sources import register
from src.db.models import League, Match, MatchStats, Team
@@ -46,9 +49,9 @@ def _match_key(home_team_id: int, away_team_id: int, match_date) -> tuple[int, i
async def _fetch_json_async(path: str, params: dict | None = None, max_retries: int = 3) -> dict | list:
"""异步 HTTP(bzzoiro 使用 httpx,不再阻塞事件循环线程池)。"""
base = settings.BZZOIRO_BASE.rstrip("/")
base = (await get_runtime_value("BZZOIRO_BASE")).rstrip("/")
url = f"{base}/{path.lstrip('/')}"
key = settings.BZZOIRO_KEY
key = await get_runtime_value("BZZOIRO_KEY")
if not key:
raise RuntimeError("BZZOIRO_KEY 未设置")
@@ -61,7 +64,14 @@ async def _fetch_json_async(path: str, params: dict | None = None, max_retries:
for attempt in range(max_retries):
try:
client = get_client()
resp = await client.get(url, headers=headers, params=params, timeout=30)
# 整请求兜底: httpx 无 total 超时,用 wait_for 防「滴水式」限速挂死
resp = await asyncio.wait_for(
client.get(
url, headers=headers, params=params,
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
),
timeout=60.0,
)
resp.raise_for_status()
return resp.json()
except Exception as e:
@@ -199,7 +209,8 @@ class BzzoiroSource:
# 避免加载联赛全部历史比赛到内存(多赛季采集时内存溢出)
if normalized_matches:
from datetime import timedelta
dates = [nm.date for nm in normalized_matches if nm.date is not None]
# normalized_matches 存的是 (nm, raw) 元组,遍历需解包
dates = [nm.date for nm, _raw in normalized_matches if nm.date is not None]
if dates:
min_dt = min(dates) - timedelta(days=30)
max_dt = max(dates) + timedelta(days=30)
@@ -219,7 +230,7 @@ class BzzoiroSource:
# 球队: 内存查找 + 按需创建
home_team_id = team_name_to_id.get(nm.home_team)
if home_team_id is None:
home = Team(name=nm.home_team)
home = Team(name=nm.home_team, name_zh=zh_name(nm.home_team))
db.add(home)
await db.flush()
home_team_id = home.id
@@ -227,7 +238,7 @@ class BzzoiroSource:
away_team_id = team_name_to_id.get(nm.away_team)
if away_team_id is None:
away = Team(name=nm.away_team)
away = Team(name=nm.away_team, name_zh=zh_name(nm.away_team))
db.add(away)
await db.flush()
away_team_id = away.id
@@ -273,7 +284,7 @@ class BzzoiroSource:
home_red_cards=nm.home_red_cards,
away_red_cards=nm.away_red_cards,
source="bzzoiro",
source_event_id=str(raw.get("id", "")),
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
)
@@ -299,7 +310,7 @@ class BzzoiroSource:
existing_match.stats = MatchStats(
match_id=existing_match.id,
source="bzzoiro",
source_event_id=str(raw.get("id", "")),
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
)
+1 -1
View File
@@ -43,4 +43,4 @@ LEAGUE_COUNTRIES: dict[str, str] = {
"EL": "Europe",
}
REQUEST_INTERVAL = 1.2 # bzzoiro 限速(秒)
REQUEST_INTERVAL = 2.0 # bzzoiro 限速(秒);上游限速严厉时宁可慢一点
+9 -3
View File
@@ -16,7 +16,7 @@ from typing import Any
import httpx
from src.core.config import settings
from src.core.runtime_config import get_runtime_value
from src.core.http_client import get_client
logger = logging.getLogger(__name__)
@@ -39,7 +39,7 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
Returns:
伤停记录列表
"""
api_key = settings.API_FOOTBALL_KEY
api_key = await get_runtime_value("API_FOOTBALL_KEY")
if not api_key:
raise RuntimeError("API_FOOTBALL_KEY 未设置")
@@ -77,7 +77,13 @@ async def fetch_injuries(*, date: str | None = None, fixture_id: int | None = No
for attempt in range(3):
try:
client = get_client()
resp = await client.get(url, headers=headers, params=params, timeout=30)
resp = await asyncio.wait_for(
client.get(
url, headers=headers, params=params,
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
),
timeout=60.0,
)
resp.raise_for_status()
break
except Exception as e:
+3 -3
View File
@@ -18,7 +18,7 @@ VALID_STATUS = {"finished", "scheduled", "in_play", "paused", "postponed", "canc
STATUS_MAP = {
"finished": "finished", "completed": "finished", "done": "finished", "awarded": "finished",
"scheduled": "scheduled", "upcoming": "scheduled",
"scheduled": "scheduled", "upcoming": "scheduled", "notstarted": "scheduled", "not_started": "scheduled",
"in_play": "in_play", "live": "in_play",
"paused": "paused", "postponed": "postponed",
"cancelled": "cancelled", "canceled": "cancelled", "abandoned": "cancelled",
@@ -200,8 +200,8 @@ def normalize_understat(raw: dict, league_type: str) -> NormalizedMatch | None:
away = normalize_name(away_name)
if not home or not away or home == away:
return None
home_xg = raw.get("xG", {}).get("h") if isinstance(raw.get("xG"), dict) else None
away_xg = raw.get("xG", {}).get("a") if isinstance(raw.get("xG"), dict) else None
home_xg = _to_float(raw["xG"].get("h")) if isinstance(raw.get("xG"), dict) else None
away_xg = _to_float(raw["xG"].get("a")) if isinstance(raw.get("xG"), dict) else None
return NormalizedMatch(
league_type=league_type,
date=dt,
+7 -3
View File
@@ -29,9 +29,13 @@ class DataSource(Protocol):
_SOURCES: dict[str, DataSource] = {}
def register(source: DataSource) -> DataSource:
"""装饰器:将数据源注册到全局注册表。"""
_SOURCES[source.name] = source
def register(source):
"""装饰器:将数据源注册到全局注册表。
兼容类注册与实例注册:类会被实例化后存入(保证 get_source 返回实例)。
"""
obj = source() if isinstance(source, type) else source
_SOURCES[obj.name] = obj
return source
+168
View File
@@ -0,0 +1,168 @@
"""球队中文译名表: 规范化英文名 → 中文。
来源: 手工整理(五大联赛全部 + 欧战常客)。
未收录的球队保持英文显示(前端回落),新队采集入库时自动查此表。
"""
TEAM_NAME_ZH: dict[str, str] = {
# ── 英格兰 ──
"Arsenal": "阿森纳", "Aston Villa": "阿斯顿维拉", "Chelsea": "切尔西",
"Liverpool FC": "利物浦", "Liverpool": "利物浦",
"Manchester City": "曼城", "Manchester United": "曼联",
"Tottenham Hotspur": "托特纳姆热刺", "Newcastle United": "纽卡斯尔联",
"West Ham United": "西汉姆联", "Everton": "埃弗顿", "Fulham": "富勒姆",
"Crystal Palace": "水晶宫", "Brentford": "布伦特福德",
"Brighton & Hove Albion": "布莱顿", "Brighton and Hove Albion": "布莱顿",
"Wolverhampton": "狼队", "Wolverhampton Wanderers": "狼队",
"Nottingham Forest": "诺丁汉森林", "AFC Bournemouth": "伯恩茅斯",
"Leeds United": "利兹联", "Leicester City": "莱斯特城",
"Ipswich Town": "伊普斯维奇", "Southampton": "南安普顿",
"Norwich City": "诺维奇城", "Sheffield United": "谢菲尔德联",
"Sheffield Wednesday": "谢周三", "Stoke City": "斯托克城",
"Sunderland": "桑德兰", "Burnley": "伯恩利", "Watford": "沃特福德",
"Hull City": "赫尔城", "Huddersfield Town": "哈德斯菲尔德",
"Luton Town": "卢顿", "Cardiff City": "加的夫城", "Swansea City": "斯旺西",
"West Bromwich Albion": "西布罗姆维奇", "Birmingham City": "伯明翰",
"Blackburn Rovers": "布莱克本", "Bolton Wanderers": "博尔顿",
"Barnsley": "巴恩斯利", "Blackpool": "布莱克浦",
"Bradford City": "布拉德福德", "Charlton Athletic": "查尔顿竞技",
"Coventry City": "考文垂", "Derby County": "德比郡",
"Middlesbrough": "米德尔斯堡", "Milton Keynes Dons": "米尔顿凯恩斯",
"Oldham Athletic": "奥尔德姆竞技", "Portsmouth": "朴茨茅斯",
"Queens Park Rangers": "女王公园巡游者", "Reading": "雷丁",
"Swindon Town": "斯温登", "Wigan Athletic": "维冈竞技",
# ── 苏格兰/爱尔兰 ──
"Celtic": "凯尔特人", "Rangers": "流浪者", "Aberdeen": "阿伯丁",
"Heart of Midlothian": "哈茨", "Hibernian": "希伯尼安",
"Derry City": "德里城", "Shelbourne": "谢尔本", "Larne FC": "拉恩",
"Linfield FC": "林斯菲尔德", "Shamrock Rovers": "沙姆罗克流浪者",
# ── 西班牙 ──
"Real Madrid": "皇家马德里", "FC Barcelona": "巴塞罗那",
"Atlético Madrid": "马德里竞技", "Athletic Club": "毕尔巴鄂竞技",
"Real Sociedad": "皇家社会", "Villarreal": "比利亚雷亚尔",
"Real Betis": "皇家贝蒂斯", "Sevilla": "塞维利亚", "Valencia": "瓦伦西亚",
"Celta Vigo": "塞尔塔", "Osasuna": "奥萨苏纳", "Getafe": "赫塔菲",
"Rayo Vallecano": "巴列卡诺", "Mallorca": "马略卡", "Girona FC": "赫罗纳",
"Girona": "赫罗纳", "Espanyol": "西班牙人", "UD Las Palmas": "拉斯帕尔马斯",
"Las Palmas": "拉斯帕尔马斯", "Deportivo Alavés": "阿拉维斯",
"Leganés": "莱加内斯", "Elche": "埃尔切", "Levante UD": "莱万特",
"Malaga CF": "马拉加", "Deportivo de A Coruna": "拉科鲁尼亚",
"Real Oviedo": "皇家奥维耶多", "Real Racing Club": "桑坦德竞技",
"Real Valladolid": "巴利亚多利德",
# ── 意大利 ──
"Juventus": "尤文图斯", "AC Milan": "AC米兰", "Inter Milan": "国际米兰",
"SSC Napoli": "那不勒斯", "AS Roma": "罗马", "Lazio": "拉齐奥",
"Atalanta": "亚特兰大", "ACF Fiorentina": "佛罗伦萨", "Bologna": "博洛尼亚",
"Torino": "都灵", "Udinese": "乌迪内斯", "Genoa": "热那亚",
"Cagliari": "卡利亚里", "Hellas Verona": "维罗纳", "Lecce": "莱切",
"Empoli": "恩波利", "Parma": "帕尔马", "Como": "科莫", "Venezia": "威尼斯",
"Pisa": "比萨", "Cremonese": "克雷莫纳", "AC Monza": "蒙扎",
"Frosinone": "弗罗西诺内", "Sassuolo": "萨索洛",
# ── 德国 ──
"FC Bayern Munchen": "拜仁慕尼黑", "Borussia Dortmund": "多特蒙德",
"Bayer 04 Leverkusen": "勒沃库森", "RB Leipzig": "莱比锡红牛",
"Borussia Mönchengladbach": "门兴格拉德巴赫", "VfB Stuttgart": "斯图加特",
"Eintracht Frankfurt": "法兰克福", "VfL Wolfsburg": "沃尔夫斯堡",
"SC Freiburg": "弗赖堡", "TSG Hoffenheim": "霍芬海姆",
"1. FC Union Berlin": "柏林联合", "1. FC Koln": "科隆",
"1. FSV Mainz 05": "美因茨", "FC Augsburg": "奥格斯堡",
"SV Werder Bremen": "云达不来梅", "VfL Bochum 1848": "波鸿",
"1. FC Heidenheim": "海登海姆", "FC St. Pauli": "圣保利",
"Holstein Kiel": "荷尔斯泰因基尔", "FC Schalke 04": "沙尔克04",
"Hamburger SV": "汉堡", "SC Paderborn 07": "帕德博恩",
"SV 07 Elversberg": "埃弗斯贝格",
# ── 法国 ──
"Paris Saint-Germain": "巴黎圣日耳曼", "Olympique de Marseille": "马赛",
"Olympique Lyonnais": "里昂", "AS Monaco": "摩纳哥", "Lille OSC": "里尔",
"OGC Nice": "尼斯", "RC Lens": "朗斯", "Stade Rennais": "雷恩",
"RC Strasbourg": "斯特拉斯堡", "Stade Brestois": "布雷斯特",
"Stade de Reims": "兰斯", "FC Nantes": "南特", "Toulouse FC": "图卢兹",
"Montpellier HSC": "蒙彼利埃", "AS Saint-Étienne": "圣埃蒂安",
"AJ Auxerre": "欧塞尔", "Le Havre AC": "勒阿弗尔", "FC Lorient": "洛里昂",
"Metz": "梅斯", "Angers SCO": "昂热", "Guingamp": "甘冈", "Troyes": "特鲁瓦",
"Le Mans": "勒芒", "Paris FC": "巴黎FC", "USL Dunkerque": "敦刻尔克",
"Rodez AF": "罗德兹", "Red Star FC": "巴黎红星",
# ── 荷兰/比利时 ──
"AFC Ajax": "阿贾克斯", "PSV Eindhoven": "埃因霍温", "Feyenoord": "费耶诺德",
"AZ Alkmaar": "阿尔克马尔", "FC Twente": "特温特", "FC Utrecht": "乌得勒支",
"NEC Nijmegen": "奈梅亨", "Go Ahead Eagles": "前进之鹰",
"Club Brugge KV": "布鲁日", "RSC Anderlecht": "安德莱赫特",
"KRC Genk": "亨克", "Royale Union Saint-Gilloise": "圣吉罗斯联合",
"Sint-Truidense VV": "圣特鲁伊登",
# ── 葡萄牙 ──
"FC Porto": "波尔图", "Benfica": "本菲卡", "Sporting CP": "里斯本竞技",
"Sporting Braga": "布拉加", "Torreense": "托雷恩塞",
# ── 土耳其 ──
"Galatasaray": "加拉塔萨雷", "Fenerbahce": "费内巴切",
"Besiktas JK": "贝西克塔斯", "Trabzonspor": "特拉布宗体育",
"Samsunspor": "萨姆松体育",
# ── 北欧 ──
"Bodø/Glimt": "博多闪耀", "Viking FK": "维京", "Tromsø IL": "特罗姆瑟",
"SK Brann": "布兰", "Lillestrøm SK": "利勒斯特罗姆",
"Malmo FF": "马尔默", "IF Elfsborg": "埃尔夫斯堡", "BK Hacken": "哈肯",
"Hammarby IF": "哈马比", "Fredrikstad FK": "腓特烈斯塔",
"Mjallby AIF": "米亚尔比", "AGF": "奥胡斯", "FC Midtjylland": "中日德兰",
"FC København": "哥本哈根", "Klaksvikar Itrottarfelag": "克拉克斯维克",
"Vikingur Gøta": "戈塔维京人", "Vikingur Reykjavik": "雷克雅未克维京人",
"Breidablik Kopavogur": "布雷达布利克", "IF Vestri": "韦斯特里",
"Kuopion Palloseura": "库奥皮奥", "Ilves": "伊尔维斯",
# ── 瑞士/奥地利 ──
"Basel": "巴塞尔", "BSC Young Boys": "伯尔尼年轻人", "FC Lugano": "卢加诺",
"Servette FC": "塞尔维特", "FC Thun": "图恩",
"FC St. Gallen 1879": "圣加仑", "LASK": "林茨", "SK Sturm Graz": "格拉茨风暴",
"Red Bull Salzburg": "萨尔茨堡红牛", "Wolfsberger AC": "沃尔夫斯贝格",
# ── 中东欧 ──
"Shakhtar Donetsk": "顿涅茨克矿工", "Dynamo Kyiv": "基辅迪纳摩",
"Dinamo Minsk": "明斯克迪纳摩", "ML Vitebsk": "维捷布斯克",
"Legia Warszawa": "华沙莱吉亚", "Lech Poznan": "波兹南莱赫",
"Jagiellonia Białystok": "比亚韦斯托克亚盖隆尼亚",
"Gornik Zabrze": "扎布热矿工", "MSK Zilina": "日利纳",
"SK Slovan Bratislava": "布拉迪斯拉发斯洛万",
"FC Spartak Trnava": "特尔纳瓦斯巴达", "SK Slavia Praha": "布拉格斯拉维亚",
"AC Sparta Praha": "布拉格斯巴达", "FC Viktoria Plzen": "比尔森胜利",
"SK Sigma Olomouc": "奥洛莫茨西格玛", "FC Hradec Kralove": "赫拉德茨克拉洛韦",
"Banik Ostrava": "俄斯特拉发矿工", "Ferencvaros TC": "费伦茨瓦罗斯",
"Paksi FC": "帕克斯", "ETO FC Gyor": "杰尔", "CFR 1907 Cluj": "克卢日",
"FCSB": "布加勒斯特星", "FC Universitatea Cluj": "克卢日大学",
"Universitatea Craiova": "克拉约瓦大学", "Ludogorets": "卢多戈雷茨",
"Levski Sofia": "索非亚列夫斯基", "CSKA Sofia": "索非亚中央陆军",
"GNK Dinamo Zagreb": "萨格勒布迪纳摩", "HNK Hajduk Split": "斯普利特海杜克",
"HNK Rijeka": "里耶卡", "NK Olimpija Ljubljana": "卢布尔雅那奥林匹亚",
"NK Celje": "采列", "NK Aluminij Kidricevo": "阿卢米尼",
"FK Partizan": "贝尔格莱德游击队", "FK Vojvodina": "伏伊伏丁那",
"FK Crvena Zvezda": "贝尔格莱德红星", "FK Crvena zvezda": "贝尔格莱德红星",
"FK Borac Banja Luka": "巴尼亚卢卡战士", "HSK Zrinjski Mostar": "莫斯塔尔兹林斯基",
"FK Buducnost Podgorica": "波德戈里察未来",
"FK Sutjeska Niksic": "苏捷斯卡", "Sheriff Tiraspol": "谢里夫",
"FC Petrocub Hincesti": "佩特罗库布", "FC Milsami Orhei": "米尔萨米",
# ── 希腊/塞浦路斯/以色列 ──
"Olympiacos FC": "奥林匹亚科斯", "Panathinaikos FC": "帕纳辛纳科斯",
"PAOK": "塞萨洛尼基PAOK", "AEK Athens": "雅典AEK", "OFI Crete": "克里特OFI",
"Omonia Nicosia": "尼科西亚奥莫尼亚", "AEK Larnaca": "拉纳卡AEK",
"Pafos FC": "帕福斯", "Hapoel Be'er Sheva": "贝尔谢巴夏普尔",
"Maccabi Tel Aviv": "特拉维夫马卡比",
# ── 东南欧/高加索/中亚 ──
"Qarabag FK": "卡拉巴赫", "Sabah FK": "萨巴赫",
"FC Ararat-Armenia": "亚美尼亚阿拉拉特", "FC Noah": "诺亚",
"FK Aktobe": "阿克托别", "Kairat Almaty": "阿拉木图凯拉特",
"FC Kairat Almaty": "阿拉木图凯拉特",
"FK Vardar Skopje": "瓦尔达尔斯科普里", "KF Shkendija": "什肯迪贾",
"KF Egnatia": "埃格纳蒂亚", "FK Zalgiris": "萨尔吉里斯",
"FK Kauno Zalgiris": "考那斯萨尔吉里斯", "Riga FC": "里加",
"RFS": "里加足球学校", "FCI Levadia Tallinn": "塔林列瓦迪亚",
"Flora Tallinn": "塔林弗洛拉",
# ── 小联赛/外围 ──
"The New Saints": "新圣徒", "Lincoln Red Imps": "林肯红魔",
"Ħamrun Spartans FC": "哈姆伦斯巴达", "Floriana FC": "弗洛里亚纳",
"SP Tre Fiori": "特雷菲奥里", "SS Virtus": "维尔图斯",
"Inter Club d'Escaldes": "埃斯卡尔德斯", "Differdange FC 03": "迪费尔当",
"Atert Bissen": "比森", "FC Drita": "德里塔", "FC Prishtina": "普里什蒂纳",
"FC Iberia 1999": "伊比利亚1999", "FK Buducnost": "波德戈里察未来",
}
def zh_name(name: str | None) -> str | None:
"""查中文译名;未收录返回 None(由调用方回落英文)。"""
if not name:
return None
return TEAM_NAME_ZH.get(name.strip())
+17 -4
View File
@@ -52,7 +52,13 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
for attempt in range(3):
try:
client = get_client()
resp = await client.get(url, headers=headers, timeout=30)
resp = await asyncio.wait_for(
client.get(
url, headers=headers,
timeout=httpx.Timeout(connect=10.0, read=30.0, write=10.0, pool=10.0),
),
timeout=60.0,
)
resp.raise_for_status()
break
except Exception as e:
@@ -65,9 +71,16 @@ async def fetch_understat(league_code: str, season: int) -> list[dict]:
else:
raise RuntimeError(f"understat fetch failed: {last_exc}")
# understat 返回 JS 对象,需要提取 JSON
# 优先按 JSON 响应解析(getLeagueData 接口返回 {teams, players, dates})
try:
data = resp.json()
except Exception:
data = None
if isinstance(data, dict) and isinstance(data.get("dates"), list):
return data["dates"]
# 兼容旧版联赛页面:内嵌 var datesData = JSON.parse('...')
text = resp.text
# 匹配 var datesData = JSON.parse('...');
match = re.search(r"var\s+datesData\s*=\s*JSON\.parse\('([^']+)'\)", text)
if not match:
logger.warning("understat 响应格式不符: %s...", text[:200])
@@ -187,7 +200,7 @@ class UnderstatSource:
existing.stats = MatchStats(
match_id=existing.id,
source="understat",
source_event_id=str(raw.get("id", "")),
source_record_id=str(raw.get("id", "")),
retrieved_at=now,
available_at=now,
)