456 lines
18 KiB
Python
456 lines
18 KiB
Python
"""客服 Agent（Task 9）单测：安全路由先行 + 知识检索三档置信度 + 契约自洽。
|
||
|
||
全部用 fake：不连 Milvus / Redis / 真实模型（Milvus 与 Redis 当前不可用），
|
||
工具与模型都用 `AsyncMock` 注入，端点解析用 monkeypatch 顶掉。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
from unittest.mock import AsyncMock
|
||
|
||
import pytest
|
||
|
||
from app.core.contracts import AgentRequest, RequestContext
|
||
from app.core.customer_service_rules import (
|
||
COMPLIANCE_REPLY,
|
||
P0_REPLY,
|
||
P1_REPLY,
|
||
P2_REPLY,
|
||
P4_REPLY,
|
||
)
|
||
from app.service.agent.bootstrap import get_agent_factory
|
||
from app.service.agent.implementations import customer_service as cs
|
||
from app.service.agent.implementations.customer_service import CustomerServiceAgent
|
||
|
||
CONTEXT = RequestContext(
|
||
user_id="9001", trace_id="trace-task9", roles=("customer",), permissions=("agent:run",),
|
||
)
|
||
|
||
FAQ_SNIPPET = "基金申购后，一般在 T+1 个交易日确认份额，请以基金公告为准。"
|
||
|
||
|
||
def make_request(message: str) -> AgentRequest:
|
||
return AgentRequest(
|
||
agent_type="customer_service", message=message, session_id="session-task9",
|
||
idempotency_key="k" * 16,
|
||
)
|
||
|
||
|
||
def hit(snippet: str, *, score: float | None = 0.9, intent: str | None = None,
|
||
collection: str = "fin_faq_collection", knowledge_id: str = "101") -> dict[str, Any]:
|
||
return {
|
||
"knowledge_id": knowledge_id, "collection": collection, "title": "客服 FAQ",
|
||
"snippet": snippet, "score": score, "tags": [], "version": "v1", "intent": intent,
|
||
}
|
||
|
||
|
||
def build_agent(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
*,
|
||
output: object,
|
||
endpoints: list[Any] | None = None,
|
||
model_text: str = "合并后的标准答复。",
|
||
) -> tuple[CustomerServiceAgent, AsyncMock, AsyncMock]:
|
||
"""构造 Agent 并注入工具/模型/端点三处 fake（不触达任何真实依赖）。"""
|
||
agent = CustomerServiceAgent(CustomerServiceAgent.definition)
|
||
call_tool = AsyncMock(return_value=output)
|
||
generate = AsyncMock(return_value=type("Execution", (), {"text": model_text})())
|
||
agent.call_tool = call_tool # type: ignore[method-assign]
|
||
agent.generate_with_model = generate # type: ignore[method-assign]
|
||
monkeypatch.setattr(
|
||
cs, "resolve_chat_endpoints",
|
||
AsyncMock(return_value=list(endpoints if endpoints is not None else [])),
|
||
)
|
||
return agent, call_tool, generate
|
||
|
||
|
||
def endpoint() -> Any:
|
||
return type("Endpoint", (), {"endpoint_code": "primary", "timeout_ms": 1000})()
|
||
|
||
|
||
# --- ① 安全路由命中：直返固定话术且不走检索 -------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("message", "reply", "intent", "transfer_required", "transfer_reason"),
|
||
[
|
||
("有人打电话让我提供短信验证码，是不是骗子", P0_REPLY, "transfer_human", True,
|
||
"P0_safety_risk"),
|
||
("有什么年化5%以上的理财推荐吗", COMPLIANCE_REPLY, "product_inquiry", False, None),
|
||
("帮我查一下我的持仓和收益", P1_REPLY, "transfer_human", False, None),
|
||
("帮我下单买1000块的那只基金", P2_REPLY, "transfer_human", True, "P2_human_requested"),
|
||
],
|
||
)
|
||
@pytest.mark.asyncio
|
||
async def test_safety_route_short_circuits_without_retrieval(
|
||
monkeypatch: pytest.MonkeyPatch, message: str, reply: str, intent: str,
|
||
transfer_required: bool, transfer_reason: str | None,
|
||
) -> None:
|
||
agent, call_tool, generate = build_agent(
|
||
monkeypatch, output={"hits": [hit(FAQ_SNIPPET)], "degraded": False}
|
||
)
|
||
result = await agent.handle(make_request(message), CONTEXT)
|
||
|
||
assert result.text == reply
|
||
assert result.intent is not None
|
||
assert result.intent.intent == intent
|
||
assert result.intent.confidence == 1.0
|
||
assert result.transfer_required is transfer_required
|
||
assert result.transfer_reason == transfer_reason
|
||
# 安全关键路径不依赖检索/模型：两者都必须没被调用。
|
||
call_tool.assert_not_awaited()
|
||
generate.assert_not_awaited()
|
||
assert result.source_references == ()
|
||
|
||
|
||
# --- ② 高置信：faq 命中 1 条 → 直返原文且不调模型 -------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_single_hit_returns_original_text_without_model(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, call_tool, generate = build_agent(
|
||
monkeypatch, output={"hits": [hit(FAQ_SNIPPET, score=0.87)], "degraded": False},
|
||
endpoints=[endpoint()],
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认"), CONTEXT)
|
||
|
||
assert result.text == FAQ_SNIPPET
|
||
assert result.intent is not None
|
||
assert result.intent.intent == "faq"
|
||
assert result.intent.confidence == pytest.approx(0.87)
|
||
assert result.transfer_required is False
|
||
generate.assert_not_awaited()
|
||
call_tool.assert_awaited_once()
|
||
assert call_tool.await_args.args[0] == cs.TOOL_NAME
|
||
assert call_tool.await_args.kwargs["intent"] == cs.TOOL_WHITELIST_INTENT
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_tool_arguments_route_collections_and_cap_top_k(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, call_tool, _ = build_agent(
|
||
monkeypatch, output={"hits": [hit(FAQ_SNIPPET)], "degraded": False}
|
||
)
|
||
await agent.handle(make_request("基金申购后多久确认"), CONTEXT)
|
||
|
||
arguments = call_tool.await_args.args[1]
|
||
assert arguments["query"] == "基金申购后多久确认"
|
||
assert tuple(arguments["intents"]) == cs.RETRIEVAL_INTENTS
|
||
assert arguments["top_k"] == cs.TOP_K
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_chitchat_hit_reports_chitchat_intent_with_full_confidence(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, _, _ = build_agent(monkeypatch, output={
|
||
"hits": [hit("你好呀，我是奶龙基金智能助手。", score=0.72, intent="chitchat")],
|
||
"degraded": False,
|
||
})
|
||
result = await agent.handle(make_request("你好呀"), CONTEXT)
|
||
|
||
assert result.intent is not None
|
||
assert result.intent.intent == "chitchat"
|
||
assert result.intent.confidence == 1.0
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_degraded_single_hit_appends_incompleteness_note(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, _, generate = build_agent(
|
||
monkeypatch, output={"hits": [hit(FAQ_SNIPPET, score=None)], "degraded": True}
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认"), CONTEXT)
|
||
|
||
assert FAQ_SNIPPET in result.text
|
||
assert "以上信息可能不完整" in result.text
|
||
assert result.intent is not None and result.intent.intent == "faq"
|
||
generate.assert_not_awaited()
|
||
|
||
|
||
# --- ③ 中置信：命中多条 → 调模型且 Prompt 强约束"不得新增事实" ------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_multiple_hits_call_model_with_no_new_facts_constraint(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
second = "申购费率以基金合同和销售机构公示为准。"
|
||
agent, _, generate = build_agent(
|
||
monkeypatch,
|
||
output={"hits": [hit(FAQ_SNIPPET, score=0.91), hit(second, score=0.88, knowledge_id="102")],
|
||
"degraded": False},
|
||
endpoints=[endpoint()],
|
||
model_text="申购一般 T+1 确认份额；费率以基金合同为准。",
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认、费率怎么算"), CONTEXT)
|
||
|
||
generate.assert_awaited_once()
|
||
prompt = generate.await_args.args[1]
|
||
assert "不得添加任何未出现在原文中的新事实" in prompt
|
||
assert "不得承诺收益" in prompt
|
||
assert FAQ_SNIPPET in prompt and second in prompt
|
||
assert result.text.startswith("申购一般 T+1 确认份额")
|
||
assert "以上信息可能不完整" in result.text
|
||
assert result.intent is not None and result.intent.intent == "faq"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_multiple_hits_without_chat_endpoint_fall_back_to_originals(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""库里只有 embedding 端点（当前实测状态）时必须失败关闭，不拿它去发聊天请求。"""
|
||
second = "申购费率以基金合同和销售机构公示为准。"
|
||
agent, _, generate = build_agent(
|
||
monkeypatch,
|
||
output={"hits": [hit(FAQ_SNIPPET, score=0.91), hit(second, score=0.88, knowledge_id="102")],
|
||
"degraded": False},
|
||
endpoints=[],
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认、费率怎么算"), CONTEXT)
|
||
|
||
generate.assert_not_awaited()
|
||
assert FAQ_SNIPPET in result.text and second in result.text
|
||
assert "以上信息可能不完整" in result.text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_model_answer_adding_new_risk_words_is_discarded(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
second = "申购费率以基金合同和销售机构公示为准。"
|
||
agent, _, _ = build_agent(
|
||
monkeypatch,
|
||
output={"hits": [hit(FAQ_SNIPPET, score=0.91), hit(second, score=0.88, knowledge_id="102")],
|
||
"degraded": False},
|
||
endpoints=[endpoint()],
|
||
model_text="这只产品年化收益率5%，保本无风险。",
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认、费率怎么算"), CONTEXT)
|
||
|
||
assert "年化" not in result.text
|
||
assert "保本" not in result.text
|
||
assert FAQ_SNIPPET in result.text and second in result.text
|
||
|
||
|
||
# --- ④ 低置信：无命中 / 低于阈值 → 兜底话术 + 建议转人工 ------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_no_hits_returns_fallback_and_suggests_human(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, call_tool, generate = build_agent(
|
||
monkeypatch, output={"hits": [], "degraded": False}, endpoints=[endpoint()]
|
||
)
|
||
result = await agent.handle(make_request("奶龙基金的总部大楼有几个停车位"), CONTEXT)
|
||
|
||
assert result.text == P4_REPLY
|
||
assert result.intent is not None
|
||
assert result.intent.intent == "transfer_human"
|
||
assert result.intent.confidence == 0.0
|
||
assert result.intent.needs_clarification is True
|
||
assert result.transfer_required is True
|
||
assert result.transfer_reason == "P4_low_confidence"
|
||
call_tool.assert_awaited_once()
|
||
generate.assert_not_awaited()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_hit_below_score_threshold_is_treated_as_low_confidence(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, _, generate = build_agent(
|
||
monkeypatch,
|
||
output={"hits": [hit("可能相关的答案", score=cs.MIN_SCORE - 0.1)], "degraded": False},
|
||
endpoints=[endpoint()],
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认"), CONTEXT)
|
||
|
||
assert result.text == P4_REPLY
|
||
assert result.intent is not None
|
||
assert result.intent.intent == "transfer_human"
|
||
assert result.intent.confidence == pytest.approx(cs.MIN_SCORE - 0.1)
|
||
assert result.transfer_required is True
|
||
generate.assert_not_awaited()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_blank_snippet_hit_is_not_treated_as_an_answer(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
agent, _, generate = build_agent(
|
||
monkeypatch, output={"hits": [hit(" ", score=0.95)], "degraded": False}
|
||
)
|
||
result = await agent.handle(make_request("基金申购后多久确认"), CONTEXT)
|
||
|
||
assert result.text == P4_REPLY
|
||
generate.assert_not_awaited()
|
||
|
||
|
||
# --- ⑤ handle() 必须显式返回 intent（不得留空交给底座分类结果回填） ------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_handle_always_returns_explicit_intent(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
cases: list[tuple[str, object]] = [
|
||
("有人让我提供验证码", {"hits": [], "degraded": False}),
|
||
("基金申购后多久确认", {"hits": [hit(FAQ_SNIPPET)], "degraded": False}),
|
||
("你好", {"hits": [], "degraded": False}),
|
||
]
|
||
for message, output in cases:
|
||
agent, _, _ = build_agent(monkeypatch, output=output)
|
||
result = await agent.handle(make_request(message), CONTEXT)
|
||
assert result.intent is not None, message
|
||
assert result.intent.intent in cs.SUPPORTED_INTENTS, message
|
||
assert 0.0 <= result.intent.confidence <= 1.0, message
|
||
|
||
|
||
# --- ⑥ 定义声明自洽 + 已注册进工厂 ----------------------------------------------
|
||
|
||
|
||
def test_definition_matches_task_contract() -> None:
|
||
definition = CustomerServiceAgent.definition
|
||
assert definition.agent_type == "customer_service"
|
||
assert definition.allowed_roles == ("customer",)
|
||
assert definition.allowed_portals == ("api",)
|
||
# Task 9 契约原文只写了 `query_knowledge`；2026-09-11 追加了本人画像只读工具
|
||
# （`query_customer_profile`，用于回答"我的风险等级是多少"这类无法从知识库得到的问题）。
|
||
# 两个都是**只读**工具，且实际可用范围仍由发布配置的意图白名单收窄。
|
||
assert definition.allowed_tools == ("query_knowledge", "query_customer_profile")
|
||
assert definition.supported_intents == (
|
||
"faq", "product_inquiry", "policy_explain", "chitchat", "transfer_human",
|
||
)
|
||
|
||
|
||
def test_agent_does_not_override_governance_methods() -> None:
|
||
"""回归约束：覆盖治理方法会被 `BaseAgent.__init_subclass__` 抛 TypeError。"""
|
||
forbidden = {
|
||
"execute", "validate_input", "validate_access", "resolve_config", "recall_memory",
|
||
"check_compliance", "_execute_governed", "call_tool", "generate_with_model",
|
||
"bind_governance", "classify_intent",
|
||
}
|
||
assert forbidden.isdisjoint(CustomerServiceAgent.__dict__)
|
||
|
||
|
||
def test_bootstrap_registers_customer_service_agent() -> None:
|
||
factory = get_agent_factory()
|
||
definition = factory.definition("customer_service")
|
||
assert definition == CustomerServiceAgent.definition
|
||
agent = factory.create("customer_service", CONTEXT)
|
||
assert isinstance(agent, CustomerServiceAgent)
|
||
|
||
|
||
# --- ⑤ 画像问题：确定性识别 + 取本人权威字段 -------------------------------------
|
||
|
||
|
||
PROFILE_OUTPUT = {
|
||
"customer_id": "9001",
|
||
"version": "2",
|
||
"profile": {
|
||
"investor_type": "C3",
|
||
"investment_horizon": "medium_term",
|
||
"trading_frequency": "medium",
|
||
"preferred_asset_class": ["bond_fund"],
|
||
"customer_tier": "platinum",
|
||
"assessment_expired": False,
|
||
},
|
||
}
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("message", "expected"),
|
||
[
|
||
("我的风险等级是多少", True),
|
||
("我的风险测评什么时候到期", True),
|
||
("测评过期了吗", True),
|
||
("我是什么风险类型的投资者", True),
|
||
("我的投资偏好是什么", True),
|
||
("我的画像", True),
|
||
# 对照：问"规则"而不是"本人数据" → 不该走画像（应走知识检索）
|
||
("风险等级怎么划分", False),
|
||
("投资者分类标准是什么", False),
|
||
("基金申购后多久确认", False),
|
||
("有什么债券基金", False),
|
||
],
|
||
)
|
||
def test_profile_question_detection(message: str, expected: bool) -> None:
|
||
"""确定性识别：问"本人数据"才走画像；问"规则"走知识检索。"""
|
||
assert cs.is_profile_question(message) is expected
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_profile_question_answers_from_own_profile(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""问本人画像 → 调画像工具、用权威字段作答，且**只查自己**。"""
|
||
agent, call_tool, _ = build_agent(monkeypatch, output=PROFILE_OUTPUT)
|
||
|
||
result = await agent.handle(make_request("我的风险等级是多少"), CONTEXT)
|
||
|
||
assert result.intent is not None and result.intent.intent == "faq"
|
||
assert "C3" in result.text or "平衡型" in result.text
|
||
assert "测评" in result.text
|
||
# 只查本人：customer_id 取自 context.user_id
|
||
args = call_tool.await_args
|
||
assert args is not None
|
||
assert args.args[0] == "query_customer_profile"
|
||
assert args.args[1]["customer_id"] == "9001"
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_expired_assessment_is_stated_not_hidden(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""测评过期必须**明说**并引导重新测评（失败关闭口径）。"""
|
||
output = {
|
||
"customer_id": "9001",
|
||
"version": "1",
|
||
"profile": {"investor_type": "C1", "assessment_expired": True},
|
||
}
|
||
agent, _, _ = build_agent(monkeypatch, output=output)
|
||
|
||
result = await agent.handle(make_request("我的风险等级是多少"), CONTEXT)
|
||
|
||
assert "已过有效期" in result.text
|
||
assert "重新完成测评" in result.text
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_profile_lookup_failure_falls_back_to_transfer(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""查不到画像（工具抛错 / 画像为空）→ 兜底转人工，**不猜一个等级**。"""
|
||
agent, call_tool, _ = build_agent(monkeypatch, output=PROFILE_OUTPUT)
|
||
call_tool.side_effect = RuntimeError("工具不可用")
|
||
|
||
result = await agent.handle(make_request("我的风险等级是多少"), CONTEXT)
|
||
|
||
assert result.text == P4_REPLY
|
||
assert result.transfer_required is True
|
||
|
||
# 画像为空（如无当前版本）同样兜底
|
||
agent2, _, _ = build_agent(monkeypatch, output={"customer_id": "9001", "profile": {}})
|
||
result2 = await agent2.handle(make_request("我的风险等级是多少"), CONTEXT)
|
||
assert result2.text == P4_REPLY
|
||
assert result2.transfer_required is True
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_policy_question_still_goes_to_knowledge_not_profile(
|
||
monkeypatch: pytest.MonkeyPatch,
|
||
) -> None:
|
||
"""问"规则"类问题不得被画像分支截走（必须走知识检索）。"""
|
||
agent, call_tool, _ = build_agent(monkeypatch, output=[hit(FAQ_SNIPPET)])
|
||
|
||
await agent.handle(make_request("风险等级怎么划分"), CONTEXT)
|
||
|
||
args = call_tool.await_args
|
||
assert args is not None
|
||
assert args.args[0] == "query_knowledge"
|