fix(memory-projection): 订正 outbox 取值口径并接通画像投影链路
背景：memory_sync_outbox 这条链此前**完全没有消费者**，且生产端照 docs/00 §6.4.6
写成大写 MILVUS/NEO4J + 中文「待处理」，而消费端按 target_store 的**值**分派 handler、
且只领 status in {pending, failed} —— 两个条件都不满足，事件任何消费者都领不到、
永久滞留且不报错（唯一键 (event_uuid, target_store) 对大小写无约束，MySQL 也不报错）。
根因是代码与测试都硬编码字面量，所以测试跟着一起错、谁也没拦住。
订正
- profile_generation_service：取值改为全仓一致的小写（milvus/neo4j/upsert/pending）
- 测试改为引用常量并断言消费端契约，不再硬编码（硬编码是本次跑偏的直接原因）
- 新增契约回归测试：断言大写值分派不到 handler、会进死信，谁改回大写立刻红
- 新增 tools/normalize_memory_sync_outbox.py：订正历史脏行（默认 dry-run、幂等）
接通投影链路（此前零消费者）
- 新增 Milvus 集合 user_long_term_memory_v1 及建集合工具（幂等、不覆盖已有集合）
- 新增 MilvusProfileProjection / MilvusProfileVectorClient，并修掉移植带来的两处必炸点：
customer_id 由「必须 int」放宽为接受数字字符串（本仓所有生产者都写 str，
不放宽则每个事件必然失败）；不可投影的 memory_key 由「整批 raise」改为跳过留痕
（否则一条 constraint: 记忆毒死该客户整批，而受控词表 13 个键里有 7 个不满足前缀）
- 新增 MemorySyncOutboxWorker（领取/指数退避/死信骨架保留原样）并接入 WorkerRuntime
- milvus → 向量投影；neo4j → 复用主干 ProfileGraphProjectionService（方案 A，
不引入第二套投影，避免同一事实在图中两种说法、违反主干既有的只投影已确认事实的不变式）
- 生产端从 memory_unit(status=active) 组装 memory_sources，随事件带上确定快照
- 前置移植 conversation_privacy：写外部存储前脱敏手机号/证件号/银行卡等
验证
- 新增 17 个单测；全量 2 failed, 1307 passed, 2 skipped
（2 个失败为既有环境项：断言请求体中文原文而 httpx 序列化成 \uXXXX，非本次引入）
- mypy app → 0 错（227 文件）；audit_schema → 89 张业务表无缺失/意外，未改动表结构
- 真机：真实 embedding(1024 维) + 真实 Milvus 写入并回读通过
- 整合链路（测试记忆 → 生产端组装 → outbox → 消费端投递 → Milvus 回读）通过，
且 MySQL 已回滚、Milvus 无残留
文档
- 新增 docs/32-记忆投影链路实现说明.md：真实口径、根因、契约与验证证据（供接手）
- AGENTS.md：新增该易错点；新增 Windows 中文输出乱码的正确命令（-X utf8）；
校正测试基线与 mypy 文件数
未做：未改 docs/00 基线、未动数据库迁移、未改投顾线代码、未启动常驻 Worker。
遗留：投顾线两处生产者的 payload 缺 memory_sources，会被消费至死信，待架构师确认是否投影。
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
"""`MilvusProfileProjection` 的定向测试。
|
||||
|
||||
前 4 个用例移植自同事 `ZSY_develop` 的
|
||||
`tests/unit/infrastructure/test_milvus_profile_projection.py`；
|
||||
后 4 个覆盖本仓对其做的**两处契约放宽**（`customer_id` 兼容字符串、
|
||||
不可投影键跳过而非整批失败）与脱敏，这些是移植时必须钉住的差异点。
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import RecoverableAgentError
|
||||
from app.infrastructure.milvus_profile_projection import MilvusProfileProjection
|
||||
|
||||
|
||||
class FakeMilvus:
|
||||
def __init__(self, existing: list[dict[str, object]] | None = None) -> None:
|
||||
self.existing = existing or []
|
||||
self.queries: list[dict[str, object]] = []
|
||||
self.upserts: list[dict[str, object]] = []
|
||||
|
||||
async def query(self, **kwargs: object) -> list[dict[str, object]]:
|
||||
self.queries.append(kwargs)
|
||||
return self.existing
|
||||
|
||||
async def upsert(self, **kwargs: object) -> None:
|
||||
self.upserts.append(kwargs)
|
||||
|
||||
|
||||
def payload() -> dict[str, object]:
|
||||
return {
|
||||
"customer_id": 7,
|
||||
"profile_version": 1,
|
||||
"memory_sources": [{
|
||||
"memory_uuid": str(uuid4()),
|
||||
"memory_key": "preference:risk_level",
|
||||
"content": "稳健型",
|
||||
"memory_type": "preference",
|
||||
"confidence": 0.9,
|
||||
"version": 2,
|
||||
"valid_until": None,
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def _source(data: dict[str, object]) -> dict[str, object]:
|
||||
sources = data["memory_sources"]
|
||||
assert isinstance(sources, list)
|
||||
source = sources[0]
|
||||
assert isinstance(source, dict)
|
||||
return source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_writes_schema_fields_and_vector() -> None:
|
||||
client = FakeMilvus()
|
||||
projection = MilvusProfileProjection(client, _embed)
|
||||
|
||||
await projection.upsert(payload())
|
||||
|
||||
assert len(client.upserts) == 1
|
||||
row = client.upserts[0]["data"][0]
|
||||
assert row["customer_id"] == 7
|
||||
assert row["status"] == "active"
|
||||
assert len(row["embedding"]) == 1024
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lower_memory_version_is_not_overwritten() -> None:
|
||||
data = payload()
|
||||
memory_uuid = _source(data)["memory_uuid"]
|
||||
client = FakeMilvus(existing=[{
|
||||
"memory_uuid": memory_uuid, "customer_id": 7, "version": 3,
|
||||
}])
|
||||
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert client.upserts == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_dimension_is_enforced() -> None:
|
||||
with pytest.raises(RecoverableAgentError, match="维度"):
|
||||
await MilvusProfileProjection(client=FakeMilvus(), embed=_embed_short).upsert(
|
||||
payload()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_uuid_memory_id_is_rejected() -> None:
|
||||
data = payload()
|
||||
_source(data)["memory_uuid"] = "unsafe\" or true"
|
||||
|
||||
with pytest.raises(ValueError, match="memory_uuid"):
|
||||
await MilvusProfileProjection(FakeMilvus(), _embed).upsert(data)
|
||||
|
||||
|
||||
# --- 本仓放宽的契约（移植差异点） -------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_string_customer_id_is_accepted() -> None:
|
||||
"""本仓三处生产者写的都是 `str(customer_id)`；不接受字符串则事件必然全部失败。"""
|
||||
data = payload()
|
||||
data["customer_id"] = "9102"
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert client.upserts[0]["data"][0]["customer_id"] == 9102
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_numeric_customer_id_is_rejected() -> None:
|
||||
"""放宽不等于不校验：uuid 之类的非数字串必须拒绝，不能当成客户号写进向量库。"""
|
||||
data = payload()
|
||||
data["customer_id"] = "957c0552-7fa2-4f2a-924d-d2d2e133b245"
|
||||
|
||||
with pytest.raises(ValueError, match="customer_id"):
|
||||
await MilvusProfileProjection(FakeMilvus(), _embed).upsert(data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_version_key_fallback_is_supported() -> None:
|
||||
"""本仓生产端 payload 用 `version`；适配器契约用 `profile_version`。两个都要认。"""
|
||||
data = payload()
|
||||
del data["profile_version"]
|
||||
data["version"] = 5
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert len(client.upserts) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_projectable_memory_key_is_skipped_not_fatal() -> None:
|
||||
"""`constraint:*` / `profile:*` 不在可投影前缀内。
|
||||
|
||||
关键：一条不可投影的键**不得**毒死同一客户其余可投影记忆。
|
||||
"""
|
||||
data = payload()
|
||||
good = _source(data)
|
||||
data["memory_sources"] = [
|
||||
{
|
||||
"memory_uuid": str(uuid4()),
|
||||
"memory_key": "constraint:liquidity",
|
||||
"content": "半年内需要流动性",
|
||||
"memory_type": "constraint",
|
||||
"confidence": 0.8,
|
||||
"version": 1,
|
||||
"valid_until": None,
|
||||
},
|
||||
good,
|
||||
]
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert len(client.upserts) == 1
|
||||
rows = client.upserts[0]["data"]
|
||||
assert [row["memory_key"] for row in rows] == ["preference:risk_level"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_keys_non_projectable_writes_nothing() -> None:
|
||||
"""全部不可投影时不写 Milvus，但也不报错（不是失败，是无需投影）。"""
|
||||
data = payload()
|
||||
_source(data)["memory_key"] = "profile:occupation"
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert client.upserts == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sensitive_credentials_are_sanitized_before_write() -> None:
|
||||
"""落外部存储前必须脱敏：手机号不得原样写进向量库。"""
|
||||
data = payload()
|
||||
_source(data)["content"] = "我的手机号是 15936583816，稳健型"
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
content = client.upserts[0]["data"][0]["content"]
|
||||
assert "15936583816" not in content
|
||||
assert "[手机号已隐藏]" in content
|
||||
|
||||
|
||||
def _vector(size: int = 1024) -> list[float]:
|
||||
return [0.0] * size
|
||||
|
||||
|
||||
async def _embed(_: str) -> list[float]:
|
||||
return _vector()
|
||||
|
||||
|
||||
async def _embed_short(_: str) -> list[float]:
|
||||
return _vector(3)
|
||||
Reference in New Issue
Block a user