feat: 核心模块增强 — 加密 + 运行时配置 + 日志缓冲
- crypto.py: API Key 加密/解密工具 - runtime_config.py: 运行时动态配置管理 - log_buffer.py: 内存日志缓冲区 - config.py: 新增加密配置项 - http_client.py: 增强重试和错误处理
This commit is contained in:
shangfangjian
+14
-3
@@ -45,10 +45,21 @@ class Settings(BaseSettings):
|
||||
DB_POOL_RECYCLE: int = 1800
|
||||
|
||||
# --- 管理接口鉴权 ---
|
||||
# 采集 / 回测等高成本或写入型接口需要此 Key(请求头 X-API-Key)。
|
||||
# 留空表示「未启用鉴权」(本地开发默认),生产环境必须设置。
|
||||
# 见审查报告 P2-7:ingest/backtest 无鉴权可被任意调用并烧掉 LLM 额度。
|
||||
# 管理后台登录密码(POST /api/v1/auth/login),登录后颁发 HttpOnly Cookie 会话。
|
||||
# 采集 / 回测等高成本或写入型接口同样需要此密码或下方 API Key。
|
||||
# 两者均留空表示「未启用鉴权」(本地开发默认),生产环境必须至少设置一项。
|
||||
# 注意:.env 中的 ADMIN_PASSWORD 是初始值;后台修改密码后以数据库中的
|
||||
# scrypt 哈希为准,建议随后删除此明文项。
|
||||
ADMIN_PASSWORD: str = ""
|
||||
ADMIN_API_KEY: str = ""
|
||||
# 管理后台会话有效期(小时)
|
||||
ADMIN_SESSION_TTL_HOURS: int = 168
|
||||
|
||||
# --- 加密主密钥 ---
|
||||
# 敏感配置(数据源/LLM 的 API Key)入库加密、会话签名都由它派生。
|
||||
# 只存于部署机 .env,切勿入库或提交代码。生成: openssl rand -base64 32
|
||||
# 变更后已加密配置将无法解密(需在后台重新保存)。
|
||||
SECRET_KEY: str = ""
|
||||
|
||||
|
||||
settings = Settings()
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
"""安全原语:对称加密(Fernet/AES)与密码哈希(scrypt)。
|
||||
|
||||
- API Key 等需要原文调用的敏感值:入库前用 SECRET_KEY 派生的 Fernet 密钥加密,
|
||||
存储格式 `enc:v1:<token>`;读取时解密。SECRET_KEY 只存于部署机 .env,不入库。
|
||||
- 管理员密码:只存 scrypt 哈希(单向,不可逆),验证用,永远不需要还原原文。
|
||||
|
||||
`enc:v1:` 前缀 + 透传设计使旧明文数据无需停机即可共存,由启动迁移一次性加密。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac as _hmac
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
|
||||
from src.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ENC_PREFIX = "enc:v1:"
|
||||
|
||||
# scrypt 参数(OWASP 推荐: n=2^17 更强,取 n=2^15 平衡 NAS CPU)
|
||||
_SCRYPT_N = 2**15
|
||||
_SCRYPT_R = 8
|
||||
_SCRYPT_P = 1
|
||||
# OpenSSL 默认 maxmem 限制约 32MB,显式放宽到 128MB
|
||||
_SCRYPT_MAXMEM = 128 * 1024 * 1024
|
||||
|
||||
|
||||
def _fernet() -> Fernet:
|
||||
"""由 SECRET_KEY 确定性派生 Fernet 密钥(任意字符串输入均可)。
|
||||
|
||||
SECRET_KEY 未配置时回落派生自 DATABASE_URL(仅为不让开发环境崩溃;
|
||||
生产必须显式配置,否则加密强度受限 —— 启动时会打 warning)。
|
||||
"""
|
||||
raw = settings.SECRET_KEY
|
||||
if not raw:
|
||||
logger.warning(
|
||||
"SECRET_KEY 未设置,加密密钥回落派生自 DATABASE_URL。"
|
||||
"请在 .env 配置强随机 SECRET_KEY(openssl rand -base64 32)。"
|
||||
)
|
||||
raw = f"fallback:{settings.DATABASE_URL}"
|
||||
digest = hashlib.sha256(raw.encode()).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(digest))
|
||||
|
||||
|
||||
def encrypt_value(plaintext: str) -> str:
|
||||
"""加密敏感值,带版本前缀;空值原样返回。"""
|
||||
if not plaintext:
|
||||
return plaintext
|
||||
token = _fernet().encrypt(plaintext.encode()).decode()
|
||||
return f"{_ENC_PREFIX}{token}"
|
||||
|
||||
|
||||
def decrypt_value(stored: str) -> str:
|
||||
"""解密 `enc:v1:` 前缀的值;无前缀(旧明文)原样返回,便于平滑迁移。"""
|
||||
if not stored or not stored.startswith(_ENC_PREFIX):
|
||||
return stored
|
||||
token = stored[len(_ENC_PREFIX):]
|
||||
try:
|
||||
return _fernet().decrypt(token.encode()).decode()
|
||||
except InvalidToken:
|
||||
# 密钥不匹配(通常是 SECRET_KEY 变了):报错而非静默返回错误数据
|
||||
raise ValueError(
|
||||
"敏感配置解密失败:SECRET_KEY 与加密时不一致。"
|
||||
"恢复原 SECRET_KEY 或在后台重新保存对应配置项。"
|
||||
) from None
|
||||
|
||||
|
||||
def is_encrypted(stored: str) -> bool:
|
||||
return bool(stored) and stored.startswith(_ENC_PREFIX)
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
"""scrypt 哈希,存储格式 scrypt$N$r$p$salt_hex$dk_hex。"""
|
||||
salt = secrets.token_bytes(16)
|
||||
dk = hashlib.scrypt(
|
||||
password.encode(), salt=salt, n=_SCRYPT_N, r=_SCRYPT_R, p=_SCRYPT_P,
|
||||
dklen=32, maxmem=_SCRYPT_MAXMEM,
|
||||
)
|
||||
return f"scrypt${_SCRYPT_N}${_SCRYPT_R}${_SCRYPT_P}${salt.hex()}${dk.hex()}"
|
||||
|
||||
|
||||
def verify_password(password: str, stored: str) -> bool:
|
||||
"""校验密码与存储的 scrypt 哈希是否匹配。"""
|
||||
try:
|
||||
algo, n, r, p, salt_hex, dk_hex = stored.split("$")
|
||||
if algo != "scrypt":
|
||||
return False
|
||||
dk = hashlib.scrypt(
|
||||
password.encode(),
|
||||
salt=bytes.fromhex(salt_hex),
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(bytes.fromhex(dk_hex)),
|
||||
maxmem=_SCRYPT_MAXMEM,
|
||||
)
|
||||
return _hmac.compare_digest(dk, bytes.fromhex(dk_hex))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
@@ -0,0 +1,82 @@
|
||||
"""内存日志缓冲:供后台「系统日志」页查看应用运行日志。
|
||||
|
||||
把应用日志(stdout)同时捕获到进程内环形缓冲(deque),提供级别/关键字/条数
|
||||
过滤查询。缓冲在进程重启后清零;需要持久化的审计请另行落库。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from collections import deque
|
||||
|
||||
_BUFFER: deque[dict] = deque(maxlen=2000)
|
||||
_LOCK = threading.Lock()
|
||||
|
||||
_LEVEL_ORDER = {"DEBUG": 10, "INFO": 20, "WARNING": 30, "ERROR": 40, "CRITICAL": 50}
|
||||
|
||||
|
||||
class MemoryLogHandler(logging.Handler):
|
||||
"""把日志记录写入内存环形缓冲。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
# format() 会在有 exc_info 时自动附带异常堆栈文本
|
||||
self.setFormatter(logging.Formatter("%(message)s"))
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
try:
|
||||
entry = {
|
||||
"ts": record.created,
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": self.format(record),
|
||||
}
|
||||
with _LOCK:
|
||||
_BUFFER.append(entry)
|
||||
except Exception: # noqa: BLE001 日志采集绝不影响业务
|
||||
self.handleError(record)
|
||||
|
||||
|
||||
class _SQLNoiseFilter(logging.Filter):
|
||||
"""过滤 SQLAlchemy 的 DEBUG/INFO 回显(只留警告以上)。"""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
return not (
|
||||
record.name.startswith("sqlalchemy.") and record.levelno < logging.WARNING
|
||||
)
|
||||
|
||||
|
||||
def get_entries(
|
||||
min_level: str | None = None,
|
||||
keyword: str | None = None,
|
||||
limit: int = 200,
|
||||
) -> list[dict]:
|
||||
"""按条件查询缓冲日志,最新在前。"""
|
||||
min_no = _LEVEL_ORDER.get((min_level or "").upper(), 0)
|
||||
kw = (keyword or "").strip().lower()
|
||||
with _LOCK:
|
||||
items = list(_BUFFER)
|
||||
items.reverse()
|
||||
out: list[dict] = []
|
||||
for e in items:
|
||||
if _LEVEL_ORDER.get(e["level"], 0) < min_no:
|
||||
continue
|
||||
if kw and kw not in e["message"].lower() and kw not in e["logger"].lower():
|
||||
continue
|
||||
out.append(e)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def setup_memory_logging(level: str = "INFO") -> None:
|
||||
"""挂载内存 handler 到 root logger(幂等),并确保 root 级别不低于 INFO。"""
|
||||
root = logging.getLogger()
|
||||
if any(isinstance(h, MemoryLogHandler) for h in root.handlers):
|
||||
return
|
||||
handler = MemoryLogHandler()
|
||||
handler.setLevel(logging.INFO)
|
||||
handler.addFilter(_SQLNoiseFilter())
|
||||
root.addHandler(handler)
|
||||
if root.level == logging.NOTSET or root.level > logging.INFO:
|
||||
root.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
@@ -0,0 +1,259 @@
|
||||
"""运行时配置:数据库优先,回落 .env。
|
||||
|
||||
后台「数据源」页可在线修改的配置项存 app_settings 表;
|
||||
读取时 DB 有值用 DB,否则回落同名环境变量(pydantic settings)。
|
||||
DB 读取失败时也回落环境变量,保证采集不因管理表故障而中断。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from src.core import crypto
|
||||
from src.core.config import settings
|
||||
from src.db.base import AsyncSessionLocal
|
||||
from src.db.models import AppSetting
|
||||
|
||||
# 管理员密码哈希在 app_settings 中的键(不进 SETTING_DEFS 白名单:
|
||||
# 只能走专门的改密接口 —— 需验证当前密码,不能被通用配置接口绕过)
|
||||
ADMIN_PASSWORD_HASH_KEY = "ADMIN_PASSWORD_HASH"
|
||||
# .env 明文密码的键名(仅作为初始值;后台改密后以哈希为准)
|
||||
_ADMIN_ENV_KEY = "ADMIN_PASSWORD"
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SettingDef:
|
||||
key: str
|
||||
label: str
|
||||
description: str
|
||||
sensitive: bool
|
||||
|
||||
|
||||
# 允许在后台查看/修改的配置项白名单(之外的 key 一律拒绝读写)
|
||||
SETTING_DEFS: dict[str, SettingDef] = {
|
||||
"BZZOIRO_KEY": SettingDef(
|
||||
"BZZOIRO_KEY", "Bzzoiro API Key", "比赛赛程 / 比分数据源凭证", sensitive=True,
|
||||
),
|
||||
"BZZOIRO_BASE": SettingDef(
|
||||
"BZZOIRO_BASE", "Bzzoiro API 地址", "Bzzoiro 接口基础地址", sensitive=False,
|
||||
),
|
||||
"API_FOOTBALL_KEY": SettingDef(
|
||||
"API_FOOTBALL_KEY", "API-Football Key", "伤停数据源凭证(api-sports)", sensitive=True,
|
||||
),
|
||||
"LLM_API_KEY": SettingDef(
|
||||
"LLM_API_KEY", "LLM API Key", "大模型服务凭证(OpenAI 兼容接口)", sensitive=True,
|
||||
),
|
||||
"LLM_BASE_URL": SettingDef(
|
||||
"LLM_BASE_URL", "LLM 接口地址", "如 https://api.deepseek.com/v1", sensitive=False,
|
||||
),
|
||||
"LLM_MODEL": SettingDef(
|
||||
"LLM_MODEL", "LLM 模型", "如 deepseek-chat / gpt-4o", sensitive=False,
|
||||
),
|
||||
}
|
||||
|
||||
# ── 按角色独立配置 LLM 的键(5 专家 + 终裁) ──
|
||||
# 每个角色可独立覆盖 模型 / 接口地址 / API Key;留空继承分层默认(见 orchestrator._agent_provider)。
|
||||
AGENT_META: list[dict] = [
|
||||
{"id": "form", "label": "近期状态分析专家"},
|
||||
{"id": "stats", "label": "攻防数据分析专家"},
|
||||
{"id": "home_away", "label": "主客因素分析专家"},
|
||||
{"id": "injuries", "label": "阵容完整性分析专家"},
|
||||
{"id": "h2h", "label": "历史交锋分析专家"},
|
||||
{"id": "aggregator", "label": "终裁分析专家"},
|
||||
]
|
||||
|
||||
for _agent in AGENT_META:
|
||||
_u = _agent["id"].upper()
|
||||
SETTING_DEFS[f"AGENT_{_u}_MODEL"] = SettingDef(
|
||||
f"AGENT_{_u}_MODEL", f"{_agent['label']} 模型", "留空继承默认(专家层/全局)", sensitive=False,
|
||||
)
|
||||
SETTING_DEFS[f"AGENT_{_u}_BASE_URL"] = SettingDef(
|
||||
f"AGENT_{_u}_BASE_URL", f"{_agent['label']} 接口地址", "留空继承全局 LLM_BASE_URL", sensitive=False,
|
||||
)
|
||||
SETTING_DEFS[f"AGENT_{_u}_API_KEY"] = SettingDef(
|
||||
f"AGENT_{_u}_API_KEY", f"{_agent['label']} API Key", "留空继承全局 LLM_API_KEY", sensitive=True,
|
||||
)
|
||||
|
||||
|
||||
def mask_value(value: str, sensitive: bool) -> str:
|
||||
"""脱敏展示:敏感值只留末 4 位;非敏感值原样返回。"""
|
||||
if not value:
|
||||
return ""
|
||||
if not sensitive:
|
||||
return value
|
||||
return f"****{value[-4:]}" if len(value) >= 8 else "****"
|
||||
|
||||
|
||||
async def get_runtime_value(key: str) -> str:
|
||||
"""读运行时配置:DB 覆盖值 → .env 默认值 → 空串。
|
||||
|
||||
敏感项入库时是密文,读出后自动解密;旧明文(迁移前)由 decrypt_value 透传。
|
||||
"""
|
||||
defn = SETTING_DEFS.get(key)
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
row = await db.get(AppSetting, key)
|
||||
if row and row.value:
|
||||
value = crypto.decrypt_value(row.value) if defn and defn.sensitive else row.value
|
||||
if value:
|
||||
return value
|
||||
except Exception:
|
||||
logger.warning("读取运行时配置 %s 失败,回落环境变量", key)
|
||||
return getattr(settings, key, "") or ""
|
||||
|
||||
|
||||
async def set_runtime_value(key: str, value: str) -> None:
|
||||
"""写入/更新 DB 覆盖值(调用方需先校验 key 在白名单内)。
|
||||
|
||||
敏感项(SETTING_DEFS.sensitive)以 Fernet 加密存储,库里不落明文。
|
||||
"""
|
||||
defn = SETTING_DEFS.get(key)
|
||||
stored = crypto.encrypt_value(value) if defn and defn.sensitive else value
|
||||
async with AsyncSessionLocal() as db:
|
||||
stmt = pg_insert(AppSetting).values(key=key, value=stored)
|
||||
stmt = stmt.on_conflict_do_update(index_elements=["key"], set_={"value": stored})
|
||||
await db.execute(stmt)
|
||||
await db.commit()
|
||||
logger.info("运行时配置 %s 已更新", key)
|
||||
|
||||
|
||||
async def clear_runtime_value(key: str) -> None:
|
||||
"""删除 DB 覆盖值,回落 .env(调用方需先校验 key 在白名单内)。"""
|
||||
async with AsyncSessionLocal() as db:
|
||||
row = await db.get(AppSetting, key)
|
||||
if row is not None:
|
||||
await db.delete(row)
|
||||
await db.commit()
|
||||
logger.info("运行时配置 %s 已清除覆盖", key)
|
||||
|
||||
|
||||
async def get_setting_origin(key: str) -> tuple[str, str]:
|
||||
"""返回 (origin, 当前生效值)。origin ∈ db / env / none。"""
|
||||
defn = SETTING_DEFS.get(key)
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
row = await db.get(AppSetting, key)
|
||||
if row and row.value:
|
||||
value = crypto.decrypt_value(row.value) if defn and defn.sensitive else row.value
|
||||
return "db", value
|
||||
except Exception:
|
||||
logger.warning("读取运行时配置 %s 来源失败,按环境变量处理", key)
|
||||
env_value = getattr(settings, key, "") or ""
|
||||
return ("env", env_value) if env_value else ("none", "")
|
||||
|
||||
|
||||
async def migrate_plaintext_sensitive_settings() -> int:
|
||||
"""一次性迁移:把库中仍是明文的敏感项加密(幂等,启动时执行)。
|
||||
|
||||
返回加密的条数。
|
||||
"""
|
||||
migrated = 0
|
||||
sensitive_keys = {k for k, d in SETTING_DEFS.items() if d.sensitive}
|
||||
async with AsyncSessionLocal() as db:
|
||||
rows = (await db.execute(select(AppSetting))).scalars().all()
|
||||
for row in rows:
|
||||
if row.key not in sensitive_keys or crypto.is_encrypted(row.value):
|
||||
continue
|
||||
row.value = crypto.encrypt_value(row.value)
|
||||
migrated += 1
|
||||
await db.commit()
|
||||
if migrated:
|
||||
logger.info("已加密迁移 %d 条明文敏感配置", migrated)
|
||||
return migrated
|
||||
|
||||
|
||||
# ── 管理员密码:只存 scrypt 哈希,永不存明文 ──────────────────────
|
||||
|
||||
|
||||
async def get_admin_password_hash() -> str:
|
||||
"""库中管理员密码哈希;无则空串。"""
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
row = await db.get(AppSetting, ADMIN_PASSWORD_HASH_KEY)
|
||||
return row.value if row else ""
|
||||
except Exception:
|
||||
logger.warning("读取管理员密码哈希失败")
|
||||
return ""
|
||||
|
||||
|
||||
async def set_admin_password_hash(hash_str: str) -> None:
|
||||
async with AsyncSessionLocal() as db:
|
||||
stmt = pg_insert(AppSetting).values(key=ADMIN_PASSWORD_HASH_KEY, value=hash_str)
|
||||
stmt = stmt.on_conflict_do_update(index_elements=["key"], set_={"value": hash_str})
|
||||
await db.execute(stmt)
|
||||
await db.commit()
|
||||
logger.info("管理员密码哈希已更新")
|
||||
|
||||
|
||||
async def verify_admin_password(candidate: str) -> bool:
|
||||
"""校验管理员密码:优先哈希;哈希不存在时回落 .env 明文(未迁移的旧部署)。"""
|
||||
stored_hash = await get_admin_password_hash()
|
||||
if stored_hash:
|
||||
return crypto.verify_password(candidate, stored_hash)
|
||||
env_pw = getattr(settings, _ADMIN_ENV_KEY, "") or ""
|
||||
return bool(env_pw) and secrets.compare_digest(candidate, env_pw)
|
||||
|
||||
|
||||
async def get_admin_credential_fingerprint() -> str:
|
||||
"""管理员凭证指纹(作为会话签名密钥的输入)。
|
||||
|
||||
用密码哈希而非密码本身:凭证变化 → 指纹变化 → 全部会话失效。
|
||||
"""
|
||||
stored_hash = await get_admin_password_hash()
|
||||
if stored_hash:
|
||||
return f"hash:{stored_hash}"
|
||||
env_pw = getattr(settings, _ADMIN_ENV_KEY, "") or ""
|
||||
return f"env:{env_pw}" if env_pw else ""
|
||||
|
||||
|
||||
async def _get_raw_setting(key: str) -> str:
|
||||
async with AsyncSessionLocal() as db:
|
||||
row = await db.get(AppSetting, key)
|
||||
return row.value if row else ""
|
||||
|
||||
|
||||
async def _delete_setting(key: str) -> None:
|
||||
async with AsyncSessionLocal() as db:
|
||||
row = await db.get(AppSetting, key)
|
||||
if row is not None:
|
||||
await db.delete(row)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def ensure_admin_password_hashed() -> bool:
|
||||
"""启动迁移:确保管理员密码只以 scrypt 哈希存在(幂等)。
|
||||
|
||||
迁移来源优先级:
|
||||
1. 库中旧版明文 ADMIN_PASSWORD 行(旧代码写入的当前密码,迁移后删除该明文行)
|
||||
2. .env 的 ADMIN_PASSWORD 初始值
|
||||
"""
|
||||
if await get_admin_password_hash():
|
||||
# 哈希已存在:清除旧版可能残留的明文行
|
||||
if await _get_raw_setting(_ADMIN_ENV_KEY):
|
||||
await _delete_setting(_ADMIN_ENV_KEY)
|
||||
logger.info("已删除遗留的明文 ADMIN_PASSWORD 行(哈希已存在)")
|
||||
return False
|
||||
|
||||
legacy_plain = await _get_raw_setting(_ADMIN_ENV_KEY)
|
||||
if legacy_plain:
|
||||
await set_admin_password_hash(crypto.hash_password(legacy_plain))
|
||||
await _delete_setting(_ADMIN_ENV_KEY)
|
||||
logger.info("已将库中明文管理员密码迁移为 scrypt 哈希,明文行已删除")
|
||||
return True
|
||||
|
||||
env_pw = getattr(settings, _ADMIN_ENV_KEY, "") or ""
|
||||
if not env_pw:
|
||||
return False
|
||||
await set_admin_password_hash(crypto.hash_password(env_pw))
|
||||
logger.info(
|
||||
"已将 .env 中的明文管理员密码迁移为 scrypt 哈希。"
|
||||
"建议现在从 .env 中删除 ADMIN_PASSWORD 明文行。"
|
||||
)
|
||||
return True
|
||||
Reference in New Issue
Block a user