324 lines
13 KiB
Python
324 lines
13 KiB
Python
|
"""NL2SQL 管理接口。"""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import json
|
||
|
|
import time
|
||
|
|
from datetime import datetime, timedelta, timezone
|
||
|
|
|
||
|
|
from fastapi import APIRouter, Depends
|
||
|
|
from fastapi.responses import PlainTextResponse
|
||
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
|
|
||
|
|
from api.deps import get_current_user
|
||
|
|
from config.deps import get_db, get_redis
|
||
|
|
from model.sys_user import SysUser
|
||
|
|
from nl2sql.audit import write_nl2sql_audit_safely
|
||
|
|
from nl2sql.metrics import load_history_metrics_safely, query_metrics, render_prometheus
|
||
|
|
from nl2sql.runtime_config import runtime_config
|
||
|
|
from nl2sql.job_history import list_job_history, record_job_history_safely
|
||
|
|
from nl2sql.jobs import (
|
||
|
|
run_consistency_check,
|
||
|
|
run_history_cleanup,
|
||
|
|
run_metadata_sync,
|
||
|
|
run_vector_cleanup,
|
||
|
|
)
|
||
|
|
from nl2sql.semantics import get_semantic_catalog_info, refresh_semantic_catalog
|
||
|
|
from repositories.nl2sql_permission import Nl2SqlPermissionRepo
|
||
|
|
from schemas.nl2sql_admin import (
|
||
|
|
ColumnPermissionCreateReq,
|
||
|
|
ColumnPermissionUpdateReq,
|
||
|
|
RoleCreateReq,
|
||
|
|
RoleUpdateReq,
|
||
|
|
SensitiveFieldCreateReq,
|
||
|
|
SensitiveFieldUpdateReq,
|
||
|
|
TablePermissionCreateReq,
|
||
|
|
TablePermissionUpdateReq,
|
||
|
|
MaintenanceJobReq,
|
||
|
|
RuntimeConfigUpdateReq,
|
||
|
|
)
|
||
|
|
from service.nl2sql.admin_service import (
|
||
|
|
column_permission_payload,
|
||
|
|
create_role,
|
||
|
|
role_payload,
|
||
|
|
sensitive_field_payload,
|
||
|
|
table_permission_payload,
|
||
|
|
validate_mask_type,
|
||
|
|
validate_table_permission,
|
||
|
|
)
|
||
|
|
from utils.exceptions import NotFoundError, ParamError
|
||
|
|
from utils.request_id import get_request_id, new_request_id
|
||
|
|
from utils.response import success
|
||
|
|
|
||
|
|
from api.routers.nl2sql import ensure_query_admin
|
||
|
|
|
||
|
|
router = APIRouter()
|
||
|
|
|
||
|
|
|
||
|
|
async def _audit(
|
||
|
|
db,
|
||
|
|
user: SysUser,
|
||
|
|
action: str,
|
||
|
|
target: str,
|
||
|
|
detail: dict | None = None,
|
||
|
|
status: str = "成功",
|
||
|
|
):
|
||
|
|
await write_nl2sql_audit_safely(
|
||
|
|
db,
|
||
|
|
user_id=user.id,
|
||
|
|
username=user.username,
|
||
|
|
action=action,
|
||
|
|
target=target,
|
||
|
|
trace_id=get_request_id() or new_request_id(),
|
||
|
|
detail=detail,
|
||
|
|
status=status,
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/roles")
|
||
|
|
async def list_roles(user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
rows = await Nl2SqlPermissionRepo(db).list_roles(include_inactive=True)
|
||
|
|
return success([role_payload(row) for row in rows])
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/nl2sql/admin/roles")
|
||
|
|
async def add_role(body: RoleCreateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
try:
|
||
|
|
role = await create_role(db, body.model_dump())
|
||
|
|
except ValueError as exc:
|
||
|
|
raise ParamError(str(exc)) from exc
|
||
|
|
await _audit(db, user, "permission_role_create", str(role.id), {"role_code": role.role_code})
|
||
|
|
return success(role_payload(role))
|
||
|
|
|
||
|
|
|
||
|
|
@router.patch("/nl2sql/admin/roles/{role_id}")
|
||
|
|
async def edit_role(role_id: int, body: RoleUpdateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
role = await Nl2SqlPermissionRepo(db).update_role(role_id, **body.model_dump(exclude_none=True))
|
||
|
|
if role is None:
|
||
|
|
raise NotFoundError("NL2SQL 角色不存在")
|
||
|
|
await _audit(db, user, "permission_role_update", str(role_id))
|
||
|
|
return success(role_payload(role))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/roles/{role_id}/tables")
|
||
|
|
async def list_tables(role_id: int, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
rows = await Nl2SqlPermissionRepo(db).list_role_table_permissions(role_id, include_inactive=True)
|
||
|
|
return success([table_permission_payload(row) for row in rows])
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/nl2sql/admin/roles/{role_id}/tables")
|
||
|
|
async def add_table(role_id: int, body: TablePermissionCreateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
if await Nl2SqlPermissionRepo(db).get_role(role_id) is None:
|
||
|
|
raise NotFoundError("NL2SQL 角色不存在")
|
||
|
|
try:
|
||
|
|
values = validate_table_permission({**body.model_dump(), "role_id": role_id})
|
||
|
|
item = await Nl2SqlPermissionRepo(db).add_table_permission(role_id=role_id, **values)
|
||
|
|
except ValueError as exc:
|
||
|
|
raise ParamError(str(exc)) from exc
|
||
|
|
await _audit(db, user, "permission_table_create", str(item.id), {"table_name": item.table_name})
|
||
|
|
return success(table_permission_payload(item))
|
||
|
|
|
||
|
|
|
||
|
|
@router.patch("/nl2sql/admin/table-permissions/{permission_id}")
|
||
|
|
async def edit_table(permission_id: int, body: TablePermissionUpdateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
item = await Nl2SqlPermissionRepo(db).update_table_permission(permission_id, **body.model_dump(exclude_none=True))
|
||
|
|
if item is None:
|
||
|
|
raise NotFoundError("NL2SQL 表权限不存在")
|
||
|
|
await _audit(db, user, "permission_table_update", str(permission_id))
|
||
|
|
return success(table_permission_payload(item))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/roles/{role_id}/columns")
|
||
|
|
async def list_columns(role_id: int, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
rows = await Nl2SqlPermissionRepo(db).list_role_column_permissions(role_id, include_inactive=True)
|
||
|
|
return success([column_permission_payload(row) for row in rows])
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/nl2sql/admin/roles/{role_id}/columns")
|
||
|
|
async def add_column(role_id: int, body: ColumnPermissionCreateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
if await Nl2SqlPermissionRepo(db).get_role(role_id) is None:
|
||
|
|
raise NotFoundError("NL2SQL 角色不存在")
|
||
|
|
if body.access_mode == "mask":
|
||
|
|
try:
|
||
|
|
validate_mask_type(body.mask_type)
|
||
|
|
except ValueError as exc:
|
||
|
|
raise ParamError(str(exc)) from exc
|
||
|
|
item = await Nl2SqlPermissionRepo(db).add_column_permission(role_id=role_id, **body.model_dump())
|
||
|
|
await _audit(db, user, "permission_column_create", str(item.id), {"table_name": item.table_name})
|
||
|
|
return success(column_permission_payload(item))
|
||
|
|
|
||
|
|
|
||
|
|
@router.patch("/nl2sql/admin/column-permissions/{permission_id}")
|
||
|
|
async def edit_column(permission_id: int, body: ColumnPermissionUpdateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
values = body.model_dump(exclude_none=True)
|
||
|
|
if "mask_type" in values:
|
||
|
|
try:
|
||
|
|
validate_mask_type(values["mask_type"])
|
||
|
|
except ValueError as exc:
|
||
|
|
raise ParamError(str(exc)) from exc
|
||
|
|
item = await Nl2SqlPermissionRepo(db).update_column_permission(permission_id, **values)
|
||
|
|
if item is None:
|
||
|
|
raise NotFoundError("NL2SQL 字段权限不存在")
|
||
|
|
await _audit(db, user, "permission_column_update", str(permission_id))
|
||
|
|
return success(column_permission_payload(item))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/sensitive-fields")
|
||
|
|
async def list_sensitive(user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
rows = await Nl2SqlPermissionRepo(db).list_sensitive_fields_admin(include_inactive=True)
|
||
|
|
return success([sensitive_field_payload(row) for row in rows])
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/nl2sql/admin/sensitive-fields")
|
||
|
|
async def add_sensitive(body: SensitiveFieldCreateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
item = await Nl2SqlPermissionRepo(db).add_sensitive_field(**body.model_dump())
|
||
|
|
await _audit(db, user, "permission_sensitive_create", str(item.id), {"table_name": item.table_name})
|
||
|
|
return success(sensitive_field_payload(item))
|
||
|
|
|
||
|
|
|
||
|
|
@router.patch("/nl2sql/admin/sensitive-fields/{field_id}")
|
||
|
|
async def edit_sensitive(field_id: int, body: SensitiveFieldUpdateReq, user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
item = await Nl2SqlPermissionRepo(db).update_sensitive_field(field_id, **body.model_dump(exclude_none=True))
|
||
|
|
if item is None:
|
||
|
|
raise NotFoundError("NL2SQL 敏感字段不存在")
|
||
|
|
await _audit(db, user, "permission_sensitive_update", str(field_id))
|
||
|
|
return success(sensitive_field_payload(item))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/metrics")
|
||
|
|
async def admin_metrics(user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
ensure_query_admin(user)
|
||
|
|
return success({
|
||
|
|
"runtime": query_metrics.snapshot(),
|
||
|
|
"history": await load_history_metrics_safely(db),
|
||
|
|
})
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/runtime-config")
|
||
|
|
async def get_runtime_config(user: SysUser = Depends(get_current_user)):
|
||
|
|
"""管理员查看当前进程内 NL2SQL 运行参数。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
return success(runtime_config.model_dump())
|
||
|
|
|
||
|
|
|
||
|
|
@router.patch("/nl2sql/admin/runtime-config")
|
||
|
|
async def update_runtime_config(
|
||
|
|
body: RuntimeConfigUpdateReq,
|
||
|
|
user: SysUser = Depends(get_current_user),
|
||
|
|
db: AsyncSession = Depends(get_db),
|
||
|
|
):
|
||
|
|
"""管理员更新当前进程内 NL2SQL 非敏感运行参数。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
try:
|
||
|
|
values = runtime_config.update(**body.model_dump(exclude_none=True))
|
||
|
|
except ValueError as exc:
|
||
|
|
raise ParamError("运行参数不合法") from exc
|
||
|
|
await _audit(db, user, "runtime_config_update", "nl2sql", {"fields": sorted(body.model_dump(exclude_none=True))})
|
||
|
|
return success(values)
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/metrics/prometheus", response_class=PlainTextResponse)
|
||
|
|
async def admin_metrics_prometheus(user: SysUser = Depends(get_current_user)):
|
||
|
|
"""管理员读取聚合 Prometheus 指标，不返回查询明细。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
return render_prometheus()
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/nl2sql/admin/jobs")
|
||
|
|
async def run_admin_job(
|
||
|
|
body: MaintenanceJobReq,
|
||
|
|
user: SysUser = Depends(get_current_user),
|
||
|
|
db: AsyncSession = Depends(get_db),
|
||
|
|
redis=Depends(get_redis),
|
||
|
|
):
|
||
|
|
"""管理员手动执行一个幂等运维任务并保存执行历史。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
started = time.perf_counter()
|
||
|
|
if body.task == "metadata_sync":
|
||
|
|
result = await run_metadata_sync(redis=redis)
|
||
|
|
elif body.task == "vector_cleanup":
|
||
|
|
result = await run_vector_cleanup(redis=redis)
|
||
|
|
elif body.task == "consistency_check":
|
||
|
|
from scripts.check_nl2sql_consistency import collect_consistency
|
||
|
|
|
||
|
|
result = await run_consistency_check(redis=redis, worker=collect_consistency)
|
||
|
|
else:
|
||
|
|
before = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=body.before_days)
|
||
|
|
result = await run_history_cleanup(db, before, redis=redis)
|
||
|
|
await record_job_history_safely(
|
||
|
|
db,
|
||
|
|
result,
|
||
|
|
elapsed_ms=(time.perf_counter() - started) * 1000,
|
||
|
|
parameter_summary={"before_days": body.before_days} if body.task == "history_cleanup" else {},
|
||
|
|
)
|
||
|
|
await _audit(db, user, "maintenance_job_run", result.name, {"status": result.status})
|
||
|
|
return success({
|
||
|
|
"name": result.name,
|
||
|
|
"status": result.status,
|
||
|
|
"attempts": result.attempts,
|
||
|
|
"detail": result.detail,
|
||
|
|
"error_type": result.error_type,
|
||
|
|
})
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/jobs/history")
|
||
|
|
async def admin_job_history(
|
||
|
|
page: int = 1,
|
||
|
|
page_size: int = 20,
|
||
|
|
status: str | None = None,
|
||
|
|
user: SysUser = Depends(get_current_user),
|
||
|
|
db: AsyncSession = Depends(get_db),
|
||
|
|
):
|
||
|
|
"""管理员查询运维任务执行历史摘要。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
return success(await list_job_history(db, page=page, page_size=page_size, status=status))
|
||
|
|
|
||
|
|
|
||
|
|
@router.get("/nl2sql/admin/semantics")
|
||
|
|
async def admin_semantics(user: SysUser = Depends(get_current_user)):
|
||
|
|
"""管理员查看当前语义目录版本和规模摘要。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
return success(get_semantic_catalog_info())
|
||
|
|
|
||
|
|
|
||
|
|
@router.post("/nl2sql/admin/semantics/refresh")
|
||
|
|
async def refresh_semantics(user: SysUser = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||
|
|
"""管理员刷新语义目录缓存，目录无效时继续使用旧缓存。"""
|
||
|
|
ensure_query_admin(user)
|
||
|
|
try:
|
||
|
|
result = refresh_semantic_catalog()
|
||
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||
|
|
await _audit(
|
||
|
|
db,
|
||
|
|
user,
|
||
|
|
"semantic_catalog_refresh",
|
||
|
|
"default",
|
||
|
|
{"error_type": type(exc).__name__},
|
||
|
|
status="失败",
|
||
|
|
)
|
||
|
|
raise ParamError("语义目录刷新失败") from exc
|
||
|
|
await _audit(
|
||
|
|
db,
|
||
|
|
user,
|
||
|
|
"semantic_catalog_refresh",
|
||
|
|
"default",
|
||
|
|
{
|
||
|
|
"version": result["version"],
|
||
|
|
"previous_version": result["previous_version"],
|
||
|
|
"changed": result["changed"],
|
||
|
|
"digest": result["digest"],
|
||
|
|
},
|
||
|
|
)
|
||
|
|
return success(result)
|