fix(memory-projection): 订正 outbox 取值口径并接通画像投影链路
背景：memory_sync_outbox 这条链此前**完全没有消费者**，且生产端照 docs/00 §6.4.6
写成大写 MILVUS/NEO4J + 中文「待处理」，而消费端按 target_store 的**值**分派 handler、
且只领 status in {pending, failed} —— 两个条件都不满足，事件任何消费者都领不到、
永久滞留且不报错（唯一键 (event_uuid, target_store) 对大小写无约束，MySQL 也不报错）。
根因是代码与测试都硬编码字面量，所以测试跟着一起错、谁也没拦住。
订正
- profile_generation_service：取值改为全仓一致的小写（milvus/neo4j/upsert/pending）
- 测试改为引用常量并断言消费端契约，不再硬编码（硬编码是本次跑偏的直接原因）
- 新增契约回归测试：断言大写值分派不到 handler、会进死信，谁改回大写立刻红
- 新增 tools/normalize_memory_sync_outbox.py：订正历史脏行（默认 dry-run、幂等）
接通投影链路（此前零消费者）
- 新增 Milvus 集合 user_long_term_memory_v1 及建集合工具（幂等、不覆盖已有集合）
- 新增 MilvusProfileProjection / MilvusProfileVectorClient，并修掉移植带来的两处必炸点：
customer_id 由「必须 int」放宽为接受数字字符串（本仓所有生产者都写 str，
不放宽则每个事件必然失败）；不可投影的 memory_key 由「整批 raise」改为跳过留痕
（否则一条 constraint: 记忆毒死该客户整批，而受控词表 13 个键里有 7 个不满足前缀）
- 新增 MemorySyncOutboxWorker（领取/指数退避/死信骨架保留原样）并接入 WorkerRuntime
- milvus → 向量投影；neo4j → 复用主干 ProfileGraphProjectionService（方案 A，
不引入第二套投影，避免同一事实在图中两种说法、违反主干既有的只投影已确认事实的不变式）
- 生产端从 memory_unit(status=active) 组装 memory_sources，随事件带上确定快照
- 前置移植 conversation_privacy：写外部存储前脱敏手机号/证件号/银行卡等
验证
- 新增 17 个单测；全量 2 failed, 1307 passed, 2 skipped
（2 个失败为既有环境项：断言请求体中文原文而 httpx 序列化成 \uXXXX，非本次引入）
- mypy app → 0 错（227 文件）；audit_schema → 89 张业务表无缺失/意外，未改动表结构
- 真机：真实 embedding(1024 维) + 真实 Milvus 写入并回读通过
- 整合链路（测试记忆 → 生产端组装 → outbox → 消费端投递 → Milvus 回读）通过，
且 MySQL 已回滚、Milvus 无残留
文档
- 新增 docs/32-记忆投影链路实现说明.md：真实口径、根因、契约与验证证据（供接手）
- AGENTS.md：新增该易错点；新增 Windows 中文输出乱码的正确命令（-X utf8）；
校正测试基线与 mypy 文件数
未做：未改 docs/00 基线、未动数据库迁移、未改投顾线代码、未启动常驻 Worker。
遗留：投顾线两处生产者的 payload 缺 memory_sources，会被消费至死信，待架构师确认是否投影。
This commit is contained in:
@@ -0,0 +1,201 @@
|
||||
"""`MilvusProfileProjection` 的定向测试。
|
||||
|
||||
前 4 个用例移植自同事 `ZSY_develop` 的
|
||||
`tests/unit/infrastructure/test_milvus_profile_projection.py`；
|
||||
后 4 个覆盖本仓对其做的**两处契约放宽**（`customer_id` 兼容字符串、
|
||||
不可投影键跳过而非整批失败）与脱敏，这些是移植时必须钉住的差异点。
|
||||
"""
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import RecoverableAgentError
|
||||
from app.infrastructure.milvus_profile_projection import MilvusProfileProjection
|
||||
|
||||
|
||||
class FakeMilvus:
|
||||
def __init__(self, existing: list[dict[str, object]] | None = None) -> None:
|
||||
self.existing = existing or []
|
||||
self.queries: list[dict[str, object]] = []
|
||||
self.upserts: list[dict[str, object]] = []
|
||||
|
||||
async def query(self, **kwargs: object) -> list[dict[str, object]]:
|
||||
self.queries.append(kwargs)
|
||||
return self.existing
|
||||
|
||||
async def upsert(self, **kwargs: object) -> None:
|
||||
self.upserts.append(kwargs)
|
||||
|
||||
|
||||
def payload() -> dict[str, object]:
|
||||
return {
|
||||
"customer_id": 7,
|
||||
"profile_version": 1,
|
||||
"memory_sources": [{
|
||||
"memory_uuid": str(uuid4()),
|
||||
"memory_key": "preference:risk_level",
|
||||
"content": "稳健型",
|
||||
"memory_type": "preference",
|
||||
"confidence": 0.9,
|
||||
"version": 2,
|
||||
"valid_until": None,
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
def _source(data: dict[str, object]) -> dict[str, object]:
|
||||
sources = data["memory_sources"]
|
||||
assert isinstance(sources, list)
|
||||
source = sources[0]
|
||||
assert isinstance(source, dict)
|
||||
return source
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_writes_schema_fields_and_vector() -> None:
|
||||
client = FakeMilvus()
|
||||
projection = MilvusProfileProjection(client, _embed)
|
||||
|
||||
await projection.upsert(payload())
|
||||
|
||||
assert len(client.upserts) == 1
|
||||
row = client.upserts[0]["data"][0]
|
||||
assert row["customer_id"] == 7
|
||||
assert row["status"] == "active"
|
||||
assert len(row["embedding"]) == 1024
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_lower_memory_version_is_not_overwritten() -> None:
|
||||
data = payload()
|
||||
memory_uuid = _source(data)["memory_uuid"]
|
||||
client = FakeMilvus(existing=[{
|
||||
"memory_uuid": memory_uuid, "customer_id": 7, "version": 3,
|
||||
}])
|
||||
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert client.upserts == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_embedding_dimension_is_enforced() -> None:
|
||||
with pytest.raises(RecoverableAgentError, match="维度"):
|
||||
await MilvusProfileProjection(client=FakeMilvus(), embed=_embed_short).upsert(
|
||||
payload()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_uuid_memory_id_is_rejected() -> None:
|
||||
data = payload()
|
||||
_source(data)["memory_uuid"] = "unsafe\" or true"
|
||||
|
||||
with pytest.raises(ValueError, match="memory_uuid"):
|
||||
await MilvusProfileProjection(FakeMilvus(), _embed).upsert(data)
|
||||
|
||||
|
||||
# --- 本仓放宽的契约（移植差异点） -------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_string_customer_id_is_accepted() -> None:
|
||||
"""本仓三处生产者写的都是 `str(customer_id)`；不接受字符串则事件必然全部失败。"""
|
||||
data = payload()
|
||||
data["customer_id"] = "9102"
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert client.upserts[0]["data"][0]["customer_id"] == 9102
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_numeric_customer_id_is_rejected() -> None:
|
||||
"""放宽不等于不校验：uuid 之类的非数字串必须拒绝，不能当成客户号写进向量库。"""
|
||||
data = payload()
|
||||
data["customer_id"] = "957c0552-7fa2-4f2a-924d-d2d2e133b245"
|
||||
|
||||
with pytest.raises(ValueError, match="customer_id"):
|
||||
await MilvusProfileProjection(FakeMilvus(), _embed).upsert(data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_version_key_fallback_is_supported() -> None:
|
||||
"""本仓生产端 payload 用 `version`；适配器契约用 `profile_version`。两个都要认。"""
|
||||
data = payload()
|
||||
del data["profile_version"]
|
||||
data["version"] = 5
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert len(client.upserts) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_projectable_memory_key_is_skipped_not_fatal() -> None:
|
||||
"""`constraint:*` / `profile:*` 不在可投影前缀内。
|
||||
|
||||
关键：一条不可投影的键**不得**毒死同一客户其余可投影记忆。
|
||||
"""
|
||||
data = payload()
|
||||
good = _source(data)
|
||||
data["memory_sources"] = [
|
||||
{
|
||||
"memory_uuid": str(uuid4()),
|
||||
"memory_key": "constraint:liquidity",
|
||||
"content": "半年内需要流动性",
|
||||
"memory_type": "constraint",
|
||||
"confidence": 0.8,
|
||||
"version": 1,
|
||||
"valid_until": None,
|
||||
},
|
||||
good,
|
||||
]
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert len(client.upserts) == 1
|
||||
rows = client.upserts[0]["data"]
|
||||
assert [row["memory_key"] for row in rows] == ["preference:risk_level"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_keys_non_projectable_writes_nothing() -> None:
|
||||
"""全部不可投影时不写 Milvus，但也不报错（不是失败，是无需投影）。"""
|
||||
data = payload()
|
||||
_source(data)["memory_key"] = "profile:occupation"
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
assert client.upserts == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sensitive_credentials_are_sanitized_before_write() -> None:
|
||||
"""落外部存储前必须脱敏：手机号不得原样写进向量库。"""
|
||||
data = payload()
|
||||
_source(data)["content"] = "我的手机号是 15936583816，稳健型"
|
||||
|
||||
client = FakeMilvus()
|
||||
await MilvusProfileProjection(client, _embed).upsert(data)
|
||||
|
||||
content = client.upserts[0]["data"][0]["content"]
|
||||
assert "15936583816" not in content
|
||||
assert "[手机号已隐藏]" in content
|
||||
|
||||
|
||||
def _vector(size: int = 1024) -> list[float]:
|
||||
return [0.0] * size
|
||||
|
||||
|
||||
async def _embed(_: str) -> list[float]:
|
||||
return _vector()
|
||||
|
||||
|
||||
async def _embed_short(_: str) -> list[float]:
|
||||
return _vector(3)
|
||||
@@ -21,6 +21,8 @@ from app.core.errors import ValidationAgentError
|
||||
from app.core.profile_projection import project_profile
|
||||
from app.service.profile_generation_service import (
|
||||
REQUIRED_SNAPSHOT_FIELDS,
|
||||
SYNC_OPERATION_UPSERT,
|
||||
SYNC_STATUS_PENDING,
|
||||
SYNC_TARGETS,
|
||||
ProfileGenerationService,
|
||||
build_snapshot,
|
||||
@@ -45,11 +47,13 @@ class FakeRepository:
|
||||
assessment: dict[str, Any] | None = None,
|
||||
current: dict[str, Any] | None = None,
|
||||
next_version: int = 2,
|
||||
memories: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self._profile = profile
|
||||
self._assessment = assessment
|
||||
self._current = current
|
||||
self._next_version = next_version
|
||||
self._memories = memories or []
|
||||
self.executed: list[str] = []
|
||||
self.inserted_snapshot: dict[str, Any] | None = None
|
||||
self.sync_events: list[dict[str, Any]] = []
|
||||
@@ -66,6 +70,10 @@ class FakeRepository:
|
||||
self.executed.append("current_snapshot")
|
||||
return self._current
|
||||
|
||||
async def active_memories(self, _cid: int) -> list[dict[str, Any]]:
|
||||
self.executed.append("active_memories")
|
||||
return self._memories
|
||||
|
||||
async def next_version(self, _cid: int) -> int:
|
||||
self.executed.append("next_version")
|
||||
return self._next_version
|
||||
@@ -179,7 +187,13 @@ async def test_first_generation_writes_two_events_and_clears_old_current(
|
||||
# `aggregate_type='profile'` 由仓储层固定写入（不在 kwargs 里），此处断言服务传入的实体标识
|
||||
assert len({e["aggregate_uuid"] for e in repo.sync_events}) == 1
|
||||
assert all(e["aggregate_uuid"] == result.profile_uuid for e in repo.sync_events)
|
||||
assert all(e["operation"] == "UPSERT" for e in repo.sync_events)
|
||||
# 断言取值本身，且**与消费端领取条件对齐**：outbox worker 只领 `pending`/`failed`，
|
||||
# 写成别的取值事件就永远没人消费。这里不再硬编码字面量（硬编码正是当初跑偏的原因）。
|
||||
assert all(e["operation"] == SYNC_OPERATION_UPSERT for e in repo.sync_events)
|
||||
assert all(e["status"] == SYNC_STATUS_PENDING for e in repo.sync_events)
|
||||
assert SYNC_OPERATION_UPSERT == "upsert"
|
||||
assert SYNC_STATUS_PENDING == "pending"
|
||||
assert set(SYNC_TARGETS) == {"milvus", "neo4j"}
|
||||
# 顺序：清旧当前标记必须在插新版本之前（否则撞唯一键）
|
||||
assert repo.executed.index("clear_current") < repo.executed.index("insert_snapshot")
|
||||
# 新版本标为当前
|
||||
@@ -187,6 +201,58 @@ async def test_first_generation_writes_two_events_and_clears_old_current(
|
||||
assert repo.inserted_snapshot["version"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_payload_carries_memory_sources_for_projection(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""payload 必须带 `memory_sources` 与 `profile_version`。
|
||||
|
||||
这是 Milvus 长期记忆投影的输入契约：适配器要 `memory_sources` 才知道往向量库
|
||||
写什么，要 `profile_version`（或 `version`）才认得出这一批属于哪个画像版本。
|
||||
缺了它，事件能被领取、却什么也投影不出来——属于"静默空转"，必须由测试挡住。
|
||||
"""
|
||||
memories = [{
|
||||
"memory_uuid": "11111111-2222-3333-4444-555555555555",
|
||||
"memory_key": "preference:risk_level",
|
||||
"content": "稳健型",
|
||||
"memory_type": "preference",
|
||||
"confidence": 0.9,
|
||||
"version": 1,
|
||||
"valid_until": None,
|
||||
}]
|
||||
repo = FakeRepository(
|
||||
profile=profile_row(), assessment=assessment_row(), current=None, memories=memories
|
||||
)
|
||||
|
||||
result = await service(monkeypatch, repo).generate(9102, now=NOW)
|
||||
|
||||
for sync_event in repo.sync_events:
|
||||
payload = sync_event["payload"]
|
||||
assert payload["profile_version"] == result.version
|
||||
assert payload["version"] == result.version
|
||||
sources = payload["memory_sources"]
|
||||
assert len(sources) == 1
|
||||
assert sources[0]["memory_uuid"] == memories[0]["memory_uuid"]
|
||||
assert sources[0]["memory_key"] == "preference:risk_level"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_payload_memory_sources_is_empty_without_active_memories(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""没有有效记忆时给**空列表**（而不是省略该键）。
|
||||
|
||||
省略键会让适配器的 `memory_sources is invalid` 报错、事件反复重试直至死信；
|
||||
空列表是"确实没有要投影的记忆"，语义不同。这里把这个区别钉住。
|
||||
"""
|
||||
repo = FakeRepository(profile=profile_row(), assessment=assessment_row(), current=None)
|
||||
|
||||
await service(monkeypatch, repo).generate(9102, now=NOW)
|
||||
|
||||
for sync_event in repo.sync_events:
|
||||
assert sync_event["payload"]["memory_sources"] == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_each_version_gets_a_fresh_profile_uuid(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`profile_uuid` 有唯一键，**每个版本必须用新 uuid**（实测撞过 Duplicate entry）。"""
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""`MemorySyncOutboxWorker` 的定向测试。
|
||||
|
||||
4 个用例移植自同事 `ZSY_develop` 的
|
||||
`tests/unit/worker/test_memory_sync_outbox_worker.py`；最后一个用例是本仓新增的
|
||||
**契约回归测试**——它是这次整条链故障的根因所在，必须有人守着。
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
import pytest
|
||||
|
||||
from app.model.memory import MemorySyncOutbox
|
||||
from app.worker.memory_sync_outbox_worker import MemorySyncOutboxWorker
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, event: MemorySyncOutbox | None) -> None:
|
||||
self.event = event
|
||||
self.commits = 0
|
||||
self.rollbacks = 0
|
||||
|
||||
async def __aenter__(self) -> "FakeSession":
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *args: object) -> None:
|
||||
return None
|
||||
|
||||
async def scalar(self, statement: object) -> MemorySyncOutbox | None:
|
||||
del statement
|
||||
return self.event
|
||||
|
||||
async def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
async def rollback(self) -> None:
|
||||
self.rollbacks += 1
|
||||
|
||||
|
||||
def event(*, target: str = "neo4j", retry_count: int = 0) -> MemorySyncOutbox:
|
||||
return MemorySyncOutbox(
|
||||
id=1, event_uuid="event-1", aggregate_type="profile",
|
||||
aggregate_uuid="profile-1", aggregate_version=1, target_store=target,
|
||||
operation="upsert", payload={"customer_id": 7}, status="pending",
|
||||
retry_count=retry_count, next_retry_at=None, last_error=None,
|
||||
created_at=datetime(2026, 1, 1), processed_at=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_marks_event_processed() -> None:
|
||||
item = event()
|
||||
session = FakeSession(item)
|
||||
seen: list[dict[str, object]] = []
|
||||
|
||||
async def handler(payload: dict[str, object]) -> None:
|
||||
seen.append(payload)
|
||||
|
||||
worker = MemorySyncOutboxWorker({"neo4j": handler}, session_factory=lambda: session)
|
||||
assert await worker.run_once() is True
|
||||
assert seen == [{"customer_id": 7}]
|
||||
assert item.status == "processed"
|
||||
assert item.processed_at is not None
|
||||
assert session.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handler_failure_uses_backoff_and_keeps_event() -> None:
|
||||
item = event()
|
||||
session = FakeSession(item)
|
||||
|
||||
async def handler(payload: dict[str, object]) -> None:
|
||||
del payload
|
||||
raise TimeoutError
|
||||
|
||||
worker = MemorySyncOutboxWorker({"neo4j": handler}, session_factory=lambda: session)
|
||||
assert await worker.run_once() is True
|
||||
assert item.status == "failed"
|
||||
assert item.retry_count == 1
|
||||
assert item.next_retry_at is not None
|
||||
assert item.last_error == "TimeoutError"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fifth_failure_enters_dead_state() -> None:
|
||||
item = event(retry_count=4)
|
||||
session = FakeSession(item)
|
||||
|
||||
async def handler(payload: dict[str, object]) -> None:
|
||||
del payload
|
||||
raise RuntimeError
|
||||
|
||||
worker = MemorySyncOutboxWorker({"neo4j": handler}, session_factory=lambda: session)
|
||||
await worker.run_once()
|
||||
assert item.status == "dead"
|
||||
assert item.retry_count == 5
|
||||
assert item.next_retry_at is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_handler_enters_dead_state_without_external_call() -> None:
|
||||
item = event(target="milvus")
|
||||
session = FakeSession(item)
|
||||
worker = MemorySyncOutboxWorker({"neo4j": lambda _: None}, session_factory=lambda: session)
|
||||
|
||||
assert await worker.run_once() is True
|
||||
assert item.status == "dead"
|
||||
assert item.last_error == "target_handler_not_configured"
|
||||
|
||||
|
||||
# --- 本仓新增：契约回归 ------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_handler_is_registered_for_uppercase_target_store() -> None:
|
||||
"""契约回归：`target_store` 只能是小写 `milvus`/`neo4j`。
|
||||
|
||||
背景（这条链真实故障的根因）：`profile_generation_service` 曾照 `docs/00` §6.4.6
|
||||
写作大写 `MILVUS`/`NEO4J` + 中文状态 `待处理`，而本 worker 按 `target_store` 的**值**
|
||||
分派 handler、且只领 `{"pending","failed"}`。结果该事件两个条件都不满足，
|
||||
**任何消费者都领不到，永久滞留且不报错**（唯一键 `(event_uuid, target_store)`
|
||||
对大小写没有约束，所以静默）。
|
||||
|
||||
本用例把"大写分派不到"这一事实钉住：将来谁把取值改回大写，这里会立刻红。
|
||||
"""
|
||||
item = event(target="MILVUS", retry_count=0)
|
||||
session = FakeSession(item)
|
||||
called: list[object] = []
|
||||
|
||||
async def handler(payload: dict[str, object]) -> None:
|
||||
called.append(payload)
|
||||
|
||||
worker = MemorySyncOutboxWorker({"milvus": handler}, session_factory=lambda: session)
|
||||
assert await worker.run_once() is True
|
||||
|
||||
assert called == [] # 大写键分派不到 milvus handler
|
||||
assert item.status == "dead"
|
||||
assert item.last_error == "target_handler_not_configured"
|
||||
Reference in New Issue
Block a user