feature:基本框架
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
"""四库统一注册表:异步生命周期编排。
|
||||
|
||||
启动:逐库 init,失败按指数退避重试;DB_STRICT_STARTUP=true 时重试耗尽直接抛错阻止启动。
|
||||
关闭:逆序 dispose。健康:check_ready_detail 返回带耗时明细,供 /health 与监控面板使用。
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
|
||||
from . import milvus, mysql, neo4j, redis
|
||||
from config.settings import settings
|
||||
|
||||
logger = logging.getLogger("config.database")
|
||||
|
||||
# 关闭顺序与设计一致:mysql → redis → neo4j → milvus
|
||||
_DB_MODULES = (mysql, redis, neo4j, milvus)
|
||||
|
||||
|
||||
def _name(m) -> str:
|
||||
return m.__name__.rsplit(".", 1)[-1]
|
||||
|
||||
|
||||
async def _init_with_retry(m, strict: bool) -> bool:
|
||||
name = _name(m)
|
||||
retries = settings.db.conn_retries
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
await m.init_db()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"[%s] init failed (attempt %d/%d): %s: %s",
|
||||
name, attempt + 1, retries, type(e).__name__, e,
|
||||
)
|
||||
if attempt < retries - 1:
|
||||
await asyncio.sleep(settings.db.retry_backoff_sec * (2**attempt))
|
||||
if strict:
|
||||
raise RuntimeError(f"[{name}] 初始化失败({retries} 次尝试后)")
|
||||
return False
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
"""手动预热(应用启动不自动调用,会话懒创建):asyncio.run(database.init_db())"""
|
||||
for m in _DB_MODULES:
|
||||
await _init_with_retry(m, settings.db.strict_startup)
|
||||
|
||||
|
||||
async def dispose() -> None:
|
||||
for m in _DB_MODULES:
|
||||
try:
|
||||
await m.dispose()
|
||||
except Exception as e:
|
||||
logger.warning("[%s] dispose failed: %s: %s", _name(m), type(e).__name__, e)
|
||||
|
||||
|
||||
async def check_ready_detail() -> dict[str, dict]:
|
||||
"""含耗时明细:{db: {status, ms}}。任一失败不影响其它库探测。"""
|
||||
out: dict[str, dict] = {}
|
||||
for m in _DB_MODULES:
|
||||
name = _name(m)
|
||||
t0 = time.perf_counter()
|
||||
try:
|
||||
await m.check_health()
|
||||
status = "ok"
|
||||
except Exception as e:
|
||||
status = f"down: {type(e).__name__}"
|
||||
out[name] = {"status": status, "ms": round((time.perf_counter() - t0) * 1000, 1)}
|
||||
return out
|
||||
|
||||
|
||||
async def check_ready() -> dict[str, str]:
|
||||
"""兼容旧契约:{db: status_str}。"""
|
||||
return {k: v["status"] for k, v in (await check_ready_detail()).items()}
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Milvus 会话:全局单例 AsyncMilvusClient。
|
||||
|
||||
timeout 入参是数据操作默认超时(用 MILVUS_TIMEOUT);
|
||||
构造为急切连接、由注册表 init_db 做指数退避重试 + 严格模式兜底。
|
||||
"""
|
||||
from pymilvus import AsyncMilvusClient
|
||||
|
||||
from config.settings import settings
|
||||
|
||||
_client: AsyncMilvusClient | None = None
|
||||
|
||||
|
||||
def client() -> AsyncMilvusClient:
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = AsyncMilvusClient(
|
||||
uri=settings.milvus.uri,
|
||||
token=settings.milvus.token,
|
||||
user=settings.milvus.user,
|
||||
password=settings.milvus.password,
|
||||
db_name=settings.milvus.db_name,
|
||||
timeout=settings.milvus.timeout,
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
client() # 急切建连;失败由注册表重试(config/database/__init__.py)
|
||||
|
||||
|
||||
async def dispose() -> None:
|
||||
global _client
|
||||
if _client is not None:
|
||||
await _client.close()
|
||||
_client = None
|
||||
|
||||
|
||||
async def check_health() -> None:
|
||||
await client().get_server_version()
|
||||
@@ -0,0 +1,51 @@
|
||||
"""MySQL 会话(懒):首次访问才创建 AsyncEngine + 连接池,不随应用启动初始化。
|
||||
|
||||
引擎/会话工厂都是函数式访问,第一次调用时建出单例;dispose 清理已建对象。
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from config.settings import settings
|
||||
|
||||
_engine = None
|
||||
_sessionmaker = None
|
||||
|
||||
|
||||
def get_engine():
|
||||
global _engine
|
||||
if _engine is None:
|
||||
_engine = create_async_engine(
|
||||
settings.mysql.url,
|
||||
pool_size=settings.mysql.pool_size,
|
||||
max_overflow=settings.mysql.max_overflow,
|
||||
pool_recycle=settings.mysql.pool_recycle,
|
||||
pool_pre_ping=True, # 取/还连接时探测,自动剔除死连接
|
||||
pool_timeout=settings.mysql.pool_timeout,
|
||||
connect_args={"connect_timeout": settings.mysql.connect_timeout},
|
||||
)
|
||||
return _engine
|
||||
|
||||
|
||||
def get_session_factory():
|
||||
global _sessionmaker
|
||||
if _sessionmaker is None:
|
||||
_sessionmaker = async_sessionmaker(bind=get_engine(), expire_on_commit=False)
|
||||
return _sessionmaker
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
"""手动预热入口(registry.init_db 逐库调用;不随应用启动自动执行)。"""
|
||||
get_engine()
|
||||
|
||||
|
||||
async def dispose() -> None:
|
||||
global _engine, _sessionmaker
|
||||
if _engine is not None:
|
||||
await _engine.dispose()
|
||||
_engine = None
|
||||
_sessionmaker = None
|
||||
|
||||
|
||||
async def check_health() -> None:
|
||||
async with get_engine().connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Neo4j 会话:全局单例 AsyncDriver(内部自带连接池与事务重试)。"""
|
||||
from neo4j import AsyncGraphDatabase
|
||||
from config.settings import settings
|
||||
|
||||
_driver = None
|
||||
|
||||
|
||||
def client():
|
||||
global _driver
|
||||
if _driver is None:
|
||||
_driver = AsyncGraphDatabase.driver(
|
||||
settings.neo4j.uri,
|
||||
auth=(settings.neo4j.user, settings.neo4j.password),
|
||||
max_connection_pool_size=settings.neo4j.max_connection_pool_size,
|
||||
connection_acquisition_timeout=settings.neo4j.connection_acquisition_timeout,
|
||||
connection_timeout=settings.neo4j.connection_timeout,
|
||||
max_transaction_retry_time=settings.neo4j.max_transaction_retry_time,
|
||||
)
|
||||
return _driver
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
client() # 构造为懒连接,仅建出 driver 单例
|
||||
|
||||
|
||||
async def get_session():
|
||||
"""每次操作取一个轻量 pool 内 session;execute_read/write 自带事务重试。"""
|
||||
async with client().session() as s:
|
||||
yield s
|
||||
|
||||
|
||||
async def dispose() -> None:
|
||||
global _driver
|
||||
if _driver is not None:
|
||||
await _driver.close()
|
||||
_driver = None
|
||||
|
||||
|
||||
async def check_health() -> None:
|
||||
await client().verify_connectivity()
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Redis 会话:内置连接池的 asyncio 客户端单例。"""
|
||||
import redis.asyncio as aioredis
|
||||
from redis.backoff import FullJitterBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
from config.settings import settings
|
||||
|
||||
_client: aioredis.Redis | None = None
|
||||
|
||||
|
||||
def client() -> aioredis.Redis:
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = aioredis.Redis(
|
||||
host=settings.redis.host,
|
||||
port=settings.redis.port,
|
||||
db=settings.redis.db,
|
||||
password=settings.redis.password,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout,
|
||||
socket_timeout=settings.redis.socket_timeout,
|
||||
socket_keepalive=True, # 长连接保活,防代理/防火墙断连
|
||||
retry_on_timeout=True, # 读超时自动重试,吸收瞬时抖动
|
||||
health_check_interval=settings.redis.health_check_interval,
|
||||
retry=Retry(FullJitterBackoff(base=1, cap=10), retries=3),
|
||||
max_connections=settings.redis.max_connections,
|
||||
decode_responses=True,
|
||||
)
|
||||
return _client
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
await client() # 构造是同步懒加载,启动时只需确保单例被建出
|
||||
|
||||
|
||||
async def dispose() -> None:
|
||||
global _client
|
||||
if _client is not None:
|
||||
await _client.aclose()
|
||||
_client = None
|
||||
|
||||
|
||||
async def check_health() -> None:
|
||||
assert await client().ping()
|
||||
Reference in New Issue
Block a user