merge: integrate ZSY customer service and profile capabilities
This commit is contained in:
张胜宇
@@ -0,0 +1,74 @@
|
||||
from typing import Any
|
||||
|
||||
from app.core.errors import ForbiddenAgentError, RecoverableAgentError
|
||||
from app.core.knowledge_contracts import ALLOWED_KNOWLEDGE_COLLECTIONS
|
||||
|
||||
|
||||
class MilvusKnowledgeClient:
|
||||
def __init__(self, uri: str, token: str | None = None) -> None:
|
||||
self._uri = uri
|
||||
self._token = token
|
||||
self._client: Any | None = None
|
||||
|
||||
async def _ensure_client(self) -> Any:
|
||||
if self._client is None:
|
||||
from pymilvus import AsyncMilvusClient # type: ignore[import-untyped]
|
||||
|
||||
self._client = AsyncMilvusClient(uri=self._uri, token=self._token)
|
||||
return self._client
|
||||
|
||||
async def search(
|
||||
self, collection: str, vector: list[float], top_k: int
|
||||
) -> list[dict[str, Any]]:
|
||||
if collection not in ALLOWED_KNOWLEDGE_COLLECTIONS:
|
||||
raise ForbiddenAgentError("未授权的知识集合")
|
||||
if len(vector) != 1024 or not 1 <= top_k <= 20:
|
||||
raise RecoverableAgentError("知识检索参数无效")
|
||||
try:
|
||||
client = await self._ensure_client()
|
||||
# Lite 重启后集合默认未加载；远程 Milvus 对重复加载保持幂等。
|
||||
load_collection = getattr(client, "load_collection", None)
|
||||
if load_collection is not None:
|
||||
await load_collection(collection_name=collection)
|
||||
batches = await client.search(
|
||||
collection_name=collection,
|
||||
data=[vector],
|
||||
limit=top_k,
|
||||
output_fields=["knowledge_id", "title", "snippet", "tags", "version"],
|
||||
search_params={"metric_type": "COSINE"},
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RecoverableAgentError("知识检索不可用") from exc
|
||||
return [
|
||||
normalized
|
||||
for batch in batches
|
||||
for hit in batch
|
||||
if (normalized := self._normalize_hit(hit)) is not None
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_hit(hit: Any) -> dict[str, Any] | None:
|
||||
"""统一 Milvus SDK 的平铺与 entity 包装命中格式。"""
|
||||
raw = dict(hit)
|
||||
entity = raw.get("entity")
|
||||
fields = entity if isinstance(entity, dict) else raw
|
||||
knowledge_id = fields.get("knowledge_id")
|
||||
snippet = fields.get("snippet")
|
||||
score = raw.get("score", raw.get("distance", fields.get("score")))
|
||||
if (
|
||||
not isinstance(knowledge_id, str)
|
||||
or not isinstance(snippet, str)
|
||||
or not isinstance(score, (int, float))
|
||||
or isinstance(score, bool)
|
||||
):
|
||||
return None
|
||||
normalized: dict[str, Any] = {
|
||||
"knowledge_id": knowledge_id,
|
||||
"snippet": snippet,
|
||||
"score": float(score),
|
||||
}
|
||||
for field in ("title", "tags", "version"):
|
||||
value = fields.get(field)
|
||||
if value is not None:
|
||||
normalized[field] = value
|
||||
return normalized
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Milvus 长期记忆投影适配器。
|
||||
|
||||
只写入已经审核的 `memory_sources`，不接受画像快照整体冒充单条记忆。
|
||||
"""
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.conversation_privacy import sanitize_customer_service_message
|
||||
from app.core.errors import RecoverableAgentError
|
||||
|
||||
PROFILE_COLLECTION = "user_long_term_memory_v1"
|
||||
VECTOR_DIM = 1024
|
||||
|
||||
|
||||
class MilvusProfileClient(Protocol):
|
||||
async def query(self, **kwargs: Any) -> list[dict[str, Any]]: ...
|
||||
|
||||
async def upsert(self, **kwargs: Any) -> Any: ...
|
||||
|
||||
|
||||
EmbeddingProvider = Callable[[str], Awaitable[list[float]]]
|
||||
|
||||
|
||||
class MilvusProfileProjection:
|
||||
"""按记忆 UUID 幂等写入长期记忆向量。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: MilvusProfileClient,
|
||||
embed: EmbeddingProvider,
|
||||
*,
|
||||
collection: str = PROFILE_COLLECTION,
|
||||
) -> None:
|
||||
self._client = client
|
||||
self._embed = embed
|
||||
self._collection = collection
|
||||
|
||||
async def upsert(self, payload: dict[str, Any]) -> None:
|
||||
customer_id, profile_version, sources = self._normalize(payload)
|
||||
load_collection = getattr(self._client, "load_collection", None)
|
||||
if load_collection is not None:
|
||||
await load_collection(collection_name=self._collection)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for source in sources:
|
||||
vector = await self._embed(source["content"])
|
||||
if len(vector) != VECTOR_DIM:
|
||||
raise RecoverableAgentError("画像向量维度不一致")
|
||||
existing = await self._client.query(
|
||||
collection_name=self._collection,
|
||||
filter=f'memory_uuid == "{source["memory_uuid"]}"',
|
||||
output_fields=["memory_uuid", "version", "customer_id"],
|
||||
)
|
||||
if existing and int(existing[0].get("version", 0)) > source["version"]:
|
||||
continue
|
||||
rows.append({
|
||||
"memory_uuid": source["memory_uuid"],
|
||||
"customer_id": customer_id,
|
||||
"content": source["content"],
|
||||
"embedding": vector,
|
||||
"memory_type": source["memory_type"],
|
||||
"memory_key": source["memory_key"],
|
||||
"confidence": source["confidence"],
|
||||
"version": source["version"],
|
||||
"status": "active",
|
||||
"valid_until_ts": source["valid_until_ts"],
|
||||
"updated_at_ts": source["updated_at_ts"],
|
||||
})
|
||||
if rows:
|
||||
await self._client.upsert(collection_name=self._collection, data=rows)
|
||||
|
||||
@staticmethod
|
||||
def _normalize(
|
||||
payload: dict[str, Any],
|
||||
) -> tuple[int, int, list[dict[str, Any]]]:
|
||||
customer_id = payload.get("customer_id")
|
||||
profile_version = payload.get("profile_version")
|
||||
sources = payload.get("memory_sources")
|
||||
if not isinstance(customer_id, int) or customer_id <= 0:
|
||||
raise ValueError("customer_id is invalid")
|
||||
if not isinstance(profile_version, int) or profile_version <= 0:
|
||||
raise ValueError("profile_version is invalid")
|
||||
if not isinstance(sources, list):
|
||||
raise ValueError("memory_sources is invalid")
|
||||
normalized: list[dict[str, Any]] = []
|
||||
now = int(datetime.now(UTC).timestamp())
|
||||
for source in sources:
|
||||
if not isinstance(source, dict):
|
||||
raise ValueError("memory source is invalid")
|
||||
required = [source.get(name) for name in (
|
||||
"memory_uuid", "memory_key", "content", "memory_type"
|
||||
)]
|
||||
if not all(isinstance(value, str) and value.strip() for value in required):
|
||||
raise ValueError("memory source fields are invalid")
|
||||
try:
|
||||
memory_uuid = str(UUID(str(source["memory_uuid"])))
|
||||
except ValueError as exc:
|
||||
raise ValueError("memory_uuid is invalid") from exc
|
||||
memory_key = str(source["memory_key"]).strip()
|
||||
if not (memory_key.startswith("preference:") or memory_key.startswith("goal:")):
|
||||
raise ValueError("memory key is not projectable")
|
||||
confidence = source.get("confidence")
|
||||
version = source.get("version")
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
raise ValueError("memory confidence is invalid")
|
||||
if not isinstance(version, int) or version <= 0:
|
||||
raise ValueError("memory version is invalid")
|
||||
valid_until = source.get("valid_until")
|
||||
valid_until_ts = None
|
||||
if isinstance(valid_until, str) and valid_until:
|
||||
try:
|
||||
valid_until_ts = int(datetime.fromisoformat(valid_until).timestamp())
|
||||
except ValueError as exc:
|
||||
raise ValueError("memory valid_until is invalid") from exc
|
||||
normalized.append({
|
||||
"memory_uuid": memory_uuid,
|
||||
"memory_key": memory_key,
|
||||
"content": sanitize_customer_service_message(str(source["content"])).strip(),
|
||||
"memory_type": str(source["memory_type"]).strip(),
|
||||
"confidence": float(confidence),
|
||||
"version": version,
|
||||
"valid_until_ts": valid_until_ts,
|
||||
"updated_at_ts": now,
|
||||
})
|
||||
return customer_id, profile_version, normalized
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Neo4j 客户画像最小投影适配器。
|
||||
|
||||
该模块只接受已审核画像快照的结构化来源，不接受模型生成的 Cypher 或关系名称。
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
from app.core.conversation_privacy import sanitize_customer_service_message
|
||||
|
||||
|
||||
class Neo4jQueryDriver(Protocol):
|
||||
async def execute_query(self, *args: Any, **kwargs: Any) -> Any: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProjectionResult:
|
||||
"""一次画像投影结果；`applied=False` 表示版本已被更新版本覆盖。"""
|
||||
|
||||
applied: bool
|
||||
reason: str = ""
|
||||
|
||||
|
||||
_CUSTOMER_QUERY = """
|
||||
MERGE (c:Customer {customer_id: $customer_id})
|
||||
WITH c, coalesce(c.profile_version, 0) AS current_version
|
||||
WHERE current_version < $profile_version
|
||||
SET c.profile_version = $profile_version, c.updated_at = $updated_at
|
||||
RETURN true AS applied
|
||||
"""
|
||||
|
||||
_PREFERENCE_QUERY = """
|
||||
UNWIND $items AS item
|
||||
MERGE (p:Preference {customer_id: $customer_id, key: item.memory_key})
|
||||
WITH p, item
|
||||
WHERE coalesce(p.version, 0) <= $profile_version
|
||||
SET p.value = item.content, p.memory_uuid = item.memory_uuid,
|
||||
p.version = item.version, p.confidence = item.confidence
|
||||
WITH p, item
|
||||
MATCH (c:Customer {customer_id: $customer_id})
|
||||
MERGE (c)-[r:PREFERS {memory_uuid: item.memory_uuid}]->(p)
|
||||
SET r.confidence = item.confidence, r.version = item.version,
|
||||
r.valid_from = item.valid_from, r.valid_until = item.valid_until
|
||||
RETURN count(p) AS projected
|
||||
"""
|
||||
|
||||
_GOAL_QUERY = """
|
||||
UNWIND $items AS item
|
||||
MERGE (g:Goal {customer_id: $customer_id, key: item.memory_key})
|
||||
WITH g, item
|
||||
WHERE coalesce(g.version, 0) <= $profile_version
|
||||
SET g.value = item.content, g.memory_uuid = item.memory_uuid,
|
||||
g.version = item.version, g.confidence = item.confidence
|
||||
WITH g, item
|
||||
MATCH (c:Customer {customer_id: $customer_id})
|
||||
MERGE (c)-[r:HAS_GOAL {memory_uuid: item.memory_uuid}]->(g)
|
||||
SET r.confidence = item.confidence, r.version = item.version,
|
||||
r.valid_from = item.valid_from, r.valid_until = item.valid_until
|
||||
RETURN count(g) AS projected
|
||||
"""
|
||||
|
||||
|
||||
class Neo4jProfileProjection:
|
||||
"""把已审核画像来源投影为受控 Neo4j 节点和关系。"""
|
||||
|
||||
def __init__(self, driver: Neo4jQueryDriver) -> None:
|
||||
self._driver = driver
|
||||
|
||||
async def upsert(self, payload: dict[str, Any]) -> ProjectionResult:
|
||||
customer_id, profile_version, updated_at, sources = self._normalize(payload)
|
||||
customer_result = await self._driver.execute_query(
|
||||
_CUSTOMER_QUERY,
|
||||
customer_id=customer_id,
|
||||
profile_version=profile_version,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
if not getattr(customer_result, "records", None):
|
||||
return ProjectionResult(False, "newer_profile_version_exists")
|
||||
grouped = {
|
||||
"preference": [item for item in sources if item["kind"] == "preference"],
|
||||
"goal": [item for item in sources if item["kind"] == "goal"],
|
||||
}
|
||||
for kind, items in grouped.items():
|
||||
if not items:
|
||||
continue
|
||||
query = _PREFERENCE_QUERY if kind == "preference" else _GOAL_QUERY
|
||||
await self._driver.execute_query(
|
||||
query,
|
||||
customer_id=customer_id,
|
||||
profile_version=profile_version,
|
||||
items=items,
|
||||
)
|
||||
return ProjectionResult(True, "applied")
|
||||
|
||||
@staticmethod
|
||||
def _normalize(
|
||||
payload: dict[str, Any],
|
||||
) -> tuple[int, int, str, list[dict[str, Any]]]:
|
||||
customer_id = payload.get("customer_id")
|
||||
profile_version = payload.get("profile_version")
|
||||
profile_uuid = payload.get("profile_uuid")
|
||||
sources = payload.get("memory_sources")
|
||||
if not isinstance(customer_id, int) or customer_id <= 0:
|
||||
raise ValueError("customer_id is invalid")
|
||||
if not isinstance(profile_version, int) or profile_version <= 0:
|
||||
raise ValueError("profile_version is invalid")
|
||||
if not isinstance(profile_uuid, str) or not profile_uuid.strip():
|
||||
raise ValueError("profile_uuid is invalid")
|
||||
if not isinstance(sources, list):
|
||||
raise ValueError("memory_sources is invalid")
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for source in sources:
|
||||
if not isinstance(source, dict):
|
||||
raise ValueError("memory source is invalid")
|
||||
memory_uuid = source.get("memory_uuid")
|
||||
memory_key = source.get("memory_key")
|
||||
content = source.get("content")
|
||||
memory_type = source.get("memory_type")
|
||||
if not isinstance(memory_uuid, str) or not memory_uuid.strip():
|
||||
raise ValueError("memory source fields are invalid")
|
||||
if not isinstance(memory_key, str) or not memory_key.strip():
|
||||
raise ValueError("memory source fields are invalid")
|
||||
if not isinstance(content, str) or not content.strip():
|
||||
raise ValueError("memory source fields are invalid")
|
||||
if not isinstance(memory_type, str) or not memory_type.strip():
|
||||
raise ValueError("memory source fields are invalid")
|
||||
if memory_key.startswith("preference:"):
|
||||
kind = "preference"
|
||||
elif memory_key.startswith("goal:"):
|
||||
kind = "goal"
|
||||
else:
|
||||
raise ValueError("memory key is not projectable")
|
||||
confidence = source.get("confidence")
|
||||
version = source.get("version")
|
||||
if not isinstance(confidence, (int, float)) or not 0 <= confidence <= 1:
|
||||
raise ValueError("memory confidence is invalid")
|
||||
if not isinstance(version, int) or version <= 0:
|
||||
raise ValueError("memory version is invalid")
|
||||
normalized.append({
|
||||
"kind": kind,
|
||||
"memory_uuid": memory_uuid.strip(),
|
||||
"memory_key": memory_key.strip(),
|
||||
"content": sanitize_customer_service_message(content).strip(),
|
||||
"memory_type": memory_type.strip(),
|
||||
"confidence": float(confidence),
|
||||
"version": version,
|
||||
"valid_until": source.get("valid_until"),
|
||||
"valid_from": source.get("valid_from"),
|
||||
})
|
||||
updated_at = str(payload.get("updated_at") or datetime.now(UTC).isoformat())
|
||||
return customer_id, profile_version, updated_at, normalized
|
||||
Reference in New Issue
Block a user