297 lines
13 KiB
Python
297 lines
13 KiB
Python
|
"""知识库管理服务（Task 11）：上传 / 查询 / 删除文档。
|
|||
|
|
|
|||
|
|
## 为什么单独一个 Service，而不是塞进 `KnowledgeIngestService`
|
|||
|
|
|
|||
|
|
`KnowledgeIngestService.ingest(...)` 是**切分与入库的活动链**（一个 chunk 一行知识 + 一条
|
|||
|
|
向量同步事件），但它**不 commit**（事务归调用方）、不认识权限、不认识删除语义。接口层需要的
|
|||
|
|
是"一个文档维度的管理用例"：鉴权 → 开事务 → 复用入库链 → 提交。那一层就是本模块，
|
|||
|
|
`KnowledgeIngestService` 一行不改（它的 `LAST_INSERT_ID()` 成对约束与事件形状都保持在原处）。
|
|||
|
|
|
|||
|
|
## 事务边界（谁 commit）
|
|||
|
|
|
|||
|
|
本模块**拥有事务**：`async with session.begin()` 包住整段写入并提交；入库链中途抛错
|
|||
|
|
（不支持的文件类型、非法 knowledge_type）时事务整体回滚，不会留下半截知识行。
|
|||
|
|
`ingest` 内部投的 `knowledge.vector_sync_requested` 与知识行同事务，因此"知识入库了但
|
|||
|
|
向量同步事件丢了"这个中间态在接口层同样不成立。
|
|||
|
|
|
|||
|
|
## 删除语义（老师原文：标记 + 投删除事件）
|
|||
|
|
|
|||
|
|
一次删除在**同一个事务**里做两件事：`fin_knowledge_meta.status = 'expired'` 与投
|
|||
|
|
`knowledge.vector_delete_requested`（消费侧见 `app/worker/knowledge_vector_worker.py`
|
|||
|
|
的 `remove`）。**不做物理删除**：知识行是审计与对账的锚点，删掉就再也说不清"这条知识
|
|||
|
|
什么时候被谁下线过"。文件字节的归档（`LocalDocumentStorage.archive()`）在提交之后做，
|
|||
|
|
且**尽力而为**：归档失败只记日志，不回滚已经提交的删除 —— 否则会出现"库里还是 active、
|
|||
|
|
存储里已经归档"的不一致，比"库已 expired、文件还在"危险得多（后者只浪费磁盘）。
|
|||
|
|
|
|||
|
|
## 权限
|
|||
|
|
|
|||
|
|
三个端点统一要求 `knowledge:manage`（admin 角色持有；权限码在 `tools/seed_test_rbac.py`
|
|||
|
|
的 `PERMISSIONS` 里，id 9019）。复用既有 `knowledge:query`（9018）会让**客户角色**看见
|
|||
|
|
整库文档清单与正文摘要，知识库的后台维护面不该向客户开放。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import base64
|
|||
|
|
import binascii
|
|||
|
|
import logging
|
|||
|
|
from collections.abc import Callable
|
|||
|
|
from datetime import UTC, datetime
|
|||
|
|
from pathlib import Path
|
|||
|
|
from typing import Any
|
|||
|
|
from uuid import uuid4
|
|||
|
|
|
|||
|
|
from sqlalchemy import select, update
|
|||
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|||
|
|
|
|||
|
|
from app.core.contracts import RequestContext
|
|||
|
|
from app.core.errors import GenericResourceNotFoundError, ValidationAgentError
|
|||
|
|
from app.infrastructure.db import SessionFactory
|
|||
|
|
from app.model.knowledge import KnowledgeMeta
|
|||
|
|
from app.model.platform import DomainEventOutbox
|
|||
|
|
from app.service.authorization_service import AuthorizationService
|
|||
|
|
from app.service.document_parser import DocumentParser
|
|||
|
|
from app.service.knowledge_ingest_service import (
|
|||
|
|
ALLOWED_KNOWLEDGE_TYPES,
|
|||
|
|
KnowledgeIngestService,
|
|||
|
|
)
|
|||
|
|
from app.worker.knowledge_vector_worker import (
|
|||
|
|
KNOWLEDGE_AGGREGATE_TYPE,
|
|||
|
|
VECTOR_DELETE_EVENT,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
logger = logging.getLogger(__name__)
|
|||
|
|
|
|||
|
|
#: 管理面的唯一权限码（见模块 docstring）。
|
|||
|
|
REQUIRED_PERMISSION = "knowledge:manage"
|
|||
|
|
|
|||
|
|
#: 删除后的状态取值。`expired` 表示已过期/已下线的知识，与
|
|||
|
|
#: `KnowledgeReferenceService` 的 `ACTIVE_STATUS = "active"` 相互排斥。
|
|||
|
|
EXPIRED_STATUS = "expired"
|
|||
|
|
|
|||
|
|
#: 列表返回的正文预览长度：管理面需要能分辨同名文档，但接口不该整篇回吐正文
|
|||
|
|
#: （正文是检索侧素材，全量回吐等于绕开引用接口的脱敏口径）。
|
|||
|
|
CONTENT_PREVIEW_LENGTH = 200
|
|||
|
|
|
|||
|
|
#: 上传时 `filename` 的最大长度（`fin_knowledge_meta.source_file` 是 `String(256)`）。
|
|||
|
|
FILENAME_MAX_LENGTH = 256
|
|||
|
|
|
|||
|
|
#: 按"当次事务的 session"构造入库链的工厂：事务归本服务，入库链**不持有** session，
|
|||
|
|
#: 否则一个 Service 实例被两次请求复用时会串事务。
|
|||
|
|
IngestFactory = Callable[[AsyncSession], Any]
|
|||
|
|
|
|||
|
|
#: 本地文档存储根目录（MinIO 无实例时的一期实现，见 `app/infrastructure/document_storage.py`）。
|
|||
|
|
#: 只影响字节落盘位置，不影响 `fin_knowledge_meta.minio_path` 里记录的相对 key。
|
|||
|
|
KNOWLEDGE_STORAGE_ROOT = Path("data") / "knowledge_documents"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def decode_content(content_base64: str) -> bytes:
|
|||
|
|
"""严格 base64 解码：非法输入直接失败，不做"忽略错误字符"的宽容解码。
|
|||
|
|
|
|||
|
|
宽容解码（`validate=False`）会把 `!!!` 之类解码成空字节，最终表现为
|
|||
|
|
"上传成功但文档入库为空"——错误被推到很远的地方才暴露。
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
return base64.b64decode(content_base64, validate=True)
|
|||
|
|
except (binascii.Error, ValueError) as exc:
|
|||
|
|
raise ValidationAgentError("content_base64 不是合法的 base64 内容") from exc
|
|||
|
|
|
|||
|
|
|
|||
|
|
class KnowledgeManagementService:
|
|||
|
|
"""知识文档管理用例：上传（切分+入库）、列表查询、删除（标记+投递向量删除）。"""
|
|||
|
|
|
|||
|
|
def __init__(
|
|||
|
|
self,
|
|||
|
|
*,
|
|||
|
|
ingest_factory: IngestFactory,
|
|||
|
|
session_factory: Callable[[], Any] | None = None,
|
|||
|
|
storage: Any = None,
|
|||
|
|
archive_on_delete: bool = True,
|
|||
|
|
) -> None:
|
|||
|
|
self._ingest_factory = ingest_factory
|
|||
|
|
self._session_factory: Callable[[], Any] = session_factory or SessionFactory
|
|||
|
|
self._storage = storage
|
|||
|
|
self._archive_on_delete = archive_on_delete
|
|||
|
|
|
|||
|
|
async def upload(
|
|||
|
|
self,
|
|||
|
|
context: RequestContext,
|
|||
|
|
*,
|
|||
|
|
filename: str,
|
|||
|
|
content: bytes,
|
|||
|
|
knowledge_type: str,
|
|||
|
|
) -> dict[str, Any]:
|
|||
|
|
"""上传一份文档并自动入库，返回本次产生的 `knowledge_id` 列表。
|
|||
|
|
|
|||
|
|
`created_by` **只**来自认证上下文（`context.user_id`），方法不提供该参数：
|
|||
|
|
允许调用方传就等于允许伪造导入人，而 `fin_knowledge_meta.reviewer_id` 是
|
|||
|
|
"谁导入了这条知识"的唯一线索。
|
|||
|
|
"""
|
|||
|
|
await AuthorizationService.require(context, REQUIRED_PERMISSION)
|
|||
|
|
self._assert_filename(filename)
|
|||
|
|
async with self._session_factory() as session, session.begin():
|
|||
|
|
knowledge_ids = await self._ingest_factory(session).ingest(
|
|||
|
|
filename=filename,
|
|||
|
|
content=content,
|
|||
|
|
knowledge_type=knowledge_type,
|
|||
|
|
created_by=int(context.user_id),
|
|||
|
|
)
|
|||
|
|
return {
|
|||
|
|
"knowledge_ids": list(knowledge_ids),
|
|||
|
|
"filename": filename,
|
|||
|
|
"knowledge_type": knowledge_type,
|
|||
|
|
"created_by": context.user_id,
|
|||
|
|
"chunk_count": len(knowledge_ids),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async def list_documents(
|
|||
|
|
self,
|
|||
|
|
context: RequestContext,
|
|||
|
|
*,
|
|||
|
|
limit: int,
|
|||
|
|
offset: int,
|
|||
|
|
knowledge_type: str | None = None,
|
|||
|
|
) -> dict[str, Any]:
|
|||
|
|
"""列出未过期（`status != 'expired'`）的知识行，按 id 倒序。
|
|||
|
|
|
|||
|
|
默认口径是**只返回未过期行**：被删除的知识仍在库里（删除是标记而非物理删除），
|
|||
|
|
管理面默认视图不应该再列出它们。
|
|||
|
|
"""
|
|||
|
|
await AuthorizationService.require(context, REQUIRED_PERMISSION)
|
|||
|
|
if knowledge_type is not None and knowledge_type not in ALLOWED_KNOWLEDGE_TYPES:
|
|||
|
|
raise ValidationAgentError(f"knowledge_type 非法：{knowledge_type}")
|
|||
|
|
statement = (
|
|||
|
|
select(KnowledgeMeta)
|
|||
|
|
.where(KnowledgeMeta.status != EXPIRED_STATUS)
|
|||
|
|
.order_by(KnowledgeMeta.id.desc())
|
|||
|
|
.limit(limit)
|
|||
|
|
.offset(offset)
|
|||
|
|
)
|
|||
|
|
if knowledge_type is not None:
|
|||
|
|
statement = statement.where(KnowledgeMeta.knowledge_type == knowledge_type)
|
|||
|
|
async with self._session_factory() as session:
|
|||
|
|
rows = list((await session.scalars(statement)).all())
|
|||
|
|
return {"items": [self._to_item(row) for row in rows], "count": len(rows)}
|
|||
|
|
|
|||
|
|
async def delete_document(
|
|||
|
|
self, context: RequestContext, knowledge_id: int
|
|||
|
|
) -> dict[str, Any]:
|
|||
|
|
"""删除一份文档：标记 `expired` + 投递向量删除事件（+ 尽力归档文件字节）。
|
|||
|
|
|
|||
|
|
不存在的 id（以及已经删除过的 id）一律抛 `SESSION_NOT_FOUND` 语义的 404，
|
|||
|
|
**不静默成功**：静默成功会让调用方以为向量已清理，而实际上什么都没发生。
|
|||
|
|
"""
|
|||
|
|
await AuthorizationService.require(context, REQUIRED_PERMISSION)
|
|||
|
|
async with self._session_factory() as session, session.begin():
|
|||
|
|
row = await session.get(KnowledgeMeta, knowledge_id)
|
|||
|
|
if row is None or row.status == EXPIRED_STATUS:
|
|||
|
|
raise GenericResourceNotFoundError("知识文档不存在")
|
|||
|
|
storage_key = row.minio_path
|
|||
|
|
now = datetime.now(UTC).replace(tzinfo=None)
|
|||
|
|
await session.execute(
|
|||
|
|
update(KnowledgeMeta)
|
|||
|
|
.where(KnowledgeMeta.id == knowledge_id)
|
|||
|
|
.values(status=EXPIRED_STATUS, updated_at=now)
|
|||
|
|
)
|
|||
|
|
self._enqueue_vector_delete(
|
|||
|
|
session, knowledge_id, trace_id=context.trace_id, now=now
|
|||
|
|
)
|
|||
|
|
await self._archive(storage_key)
|
|||
|
|
return {
|
|||
|
|
"knowledge_id": knowledge_id,
|
|||
|
|
"status": EXPIRED_STATUS,
|
|||
|
|
"vector_delete_event": VECTOR_DELETE_EVENT,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# --- 内部 ---------------------------------------------------------------
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _assert_filename(filename: str) -> None:
|
|||
|
|
"""文件名不得为空、不得超列宽、不得带路径：`source_file` 只存文件名本身。
|
|||
|
|
|
|||
|
|
带路径的文件名会被 `KnowledgeIngestService._storage_key` 剥成 basename 落存储，
|
|||
|
|
但 `source_file` 列的取值同样要干净，否则管理面列表里会出现调用方的本地路径。
|
|||
|
|
"""
|
|||
|
|
if not filename.strip():
|
|||
|
|
raise ValidationAgentError("filename 不得为空")
|
|||
|
|
if len(filename) > FILENAME_MAX_LENGTH:
|
|||
|
|
raise ValidationAgentError(f"filename 超过 {FILENAME_MAX_LENGTH} 字符")
|
|||
|
|
if filename != Path(filename).name or "\\" in filename:
|
|||
|
|
raise ValidationAgentError("filename 只能是文件名，不得包含路径")
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _enqueue_vector_delete(
|
|||
|
|
session: AsyncSession, knowledge_id: int, *, trace_id: str, now: datetime
|
|||
|
|
) -> None:
|
|||
|
|
"""投一条向量删除事件；`aggregate_id` 与 payload 都是 `str(knowledge_id)`（投递同口径）。
|
|||
|
|
|
|||
|
|
`event_id` 是唯一键，显式给 uuid4；`aggregate_type` 沿用投递侧常量，
|
|||
|
|
保证消费侧按同一聚合维度幂等覆盖。
|
|||
|
|
"""
|
|||
|
|
session.add(DomainEventOutbox(
|
|||
|
|
event_id=str(uuid4()),
|
|||
|
|
event_type=VECTOR_DELETE_EVENT,
|
|||
|
|
aggregate_type=KNOWLEDGE_AGGREGATE_TYPE,
|
|||
|
|
aggregate_id=str(knowledge_id),
|
|||
|
|
trace_id=trace_id or str(uuid4()),
|
|||
|
|
payload={"knowledge_id": str(knowledge_id)},
|
|||
|
|
status="pending",
|
|||
|
|
retry_count=0,
|
|||
|
|
occurred_at=now,
|
|||
|
|
created_at=now,
|
|||
|
|
updated_at=now,
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
async def _archive(self, storage_key: str | None) -> None:
|
|||
|
|
"""提交之后把文件字节移入 `archive/`；失败只记日志（见模块 docstring 的取舍）。"""
|
|||
|
|
if not self._archive_on_delete or not storage_key or self._storage is None:
|
|||
|
|
return
|
|||
|
|
try:
|
|||
|
|
await self._storage.archive(key=storage_key)
|
|||
|
|
except Exception: # noqa: BLE001 - 归档尽力而为，任何失败都不影响已提交的删除
|
|||
|
|
logger.warning("知识文档归档失败，已提交的删除不回滚 storage_key=%s", storage_key)
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _to_item(row: KnowledgeMeta) -> dict[str, Any]:
|
|||
|
|
content = row.content_text or ""
|
|||
|
|
tags = row.tags if isinstance(row.tags, dict) else {}
|
|||
|
|
return {
|
|||
|
|
"knowledge_id": row.id,
|
|||
|
|
"knowledge_type": row.knowledge_type,
|
|||
|
|
"title": row.title,
|
|||
|
|
"source_file": row.source_file,
|
|||
|
|
"collection": row.milvus_collection,
|
|||
|
|
"version": row.version,
|
|||
|
|
"status": row.status,
|
|||
|
|
"review_status": row.review_status,
|
|||
|
|
"created_by": row.reviewer_id,
|
|||
|
|
"tags": tags,
|
|||
|
|
"content_preview": content[:CONTENT_PREVIEW_LENGTH],
|
|||
|
|
"content_length": len(content),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_knowledge_management_service() -> KnowledgeManagementService:
|
|||
|
|
"""组合根：把接口层需要的依赖一次装好（Controller 只调它，不自己拼装配）。
|
|||
|
|
|
|||
|
|
`embedder` / `endpoint_resolver` 在 `KnowledgeIngestService` 里是**给后续对账自愈预留的**
|
|||
|
|
依赖（入库链自己不调用 embedding，向量化由 Outbox Worker 做），因此直接用组装层
|
|||
|
|
同一套工厂函数（`app.service.agent.bootstrap`）构造，不另写一套模型网关装配。
|
|||
|
|
"""
|
|||
|
|
from app.infrastructure.document_storage import LocalDocumentStorage
|
|||
|
|
from app.service.agent.bootstrap import get_memory_embedding_service
|
|||
|
|
from app.service.model_gateway import DatabaseModelEndpointResolver
|
|||
|
|
|
|||
|
|
def ingest_for(session: AsyncSession) -> KnowledgeIngestService:
|
|||
|
|
return KnowledgeIngestService(
|
|||
|
|
session=session,
|
|||
|
|
parser=DocumentParser(),
|
|||
|
|
storage=LocalDocumentStorage(KNOWLEDGE_STORAGE_ROOT),
|
|||
|
|
embedder=get_memory_embedding_service(),
|
|||
|
|
endpoint_resolver=DatabaseModelEndpointResolver(),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return KnowledgeManagementService(
|
|||
|
|
ingest_factory=ingest_for,
|
|||
|
|
session_factory=SessionFactory,
|
|||
|
|
storage=LocalDocumentStorage(KNOWLEDGE_STORAGE_ROOT),
|
|||
|
|
)
|