181 lines
6.8 KiB
Python
181 lines
6.8 KiB
Python
"""运行期意图配置(`agent_intent_config`)生效证明(不连数据库)。
|
||||
|
|
|
|||
|
|
重点不是"读了配置",而是**改了配置 → 分类行为随之改变**:
|
|||
|
|
1. 分类提示词里出现配置的意图名/描述/示例/分类要求;
|
|||
|
|
2. 同一份模型输出下,配置里的 `confidence_threshold` 直接改变 `needs_clarification`;
|
|||
|
|
3. 配置不改变既有契约:严格 JSON、未声明意图失败关闭,配置里多出来的意图既不进提示
|
|||
|
|
也不放宽校验;
|
|||
|
|
4. 装载只读 `status='active'` 的行(用编译后的 SQL 断言生效判定条件)。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from datetime import UTC, datetime
|
|||
|
|
from decimal import Decimal
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
|
|||
|
|
from app.core.errors import RecoverableAgentError, ValidationAgentError
|
|||
|
|
from app.model.configuration import AgentIntentConfig
|
|||
|
|
from app.service.intent_classifier import IntentClassifier, IntentConfigEntry
|
|||
|
|
from app.service.model_gateway import ModelExecution
|
|||
|
|
from app.service.runtime_config_service import RuntimeConfigService
|
|||
|
|
|
|||
|
|
|
|||
|
|
class RecordingModel:
|
|||
|
|
"""记录每次分类的提示词,用于断言"配置改了,提示词就不同"。"""
|
|||
|
|
|
|||
|
|
def __init__(self, text: str) -> None:
|
|||
|
|
self.text = text
|
|||
|
|
self.prompts: list[str] = []
|
|||
|
|
|
|||
|
|
async def generate(self, _endpoints: list[object], prompt: str) -> ModelExecution:
|
|||
|
|
self.prompts.append(prompt)
|
|||
|
|
return ModelExecution("intent", self.text, 1)
|
|||
|
|
|
|||
|
|
|
|||
|
|
AGENT_TYPE = "fund_query_demo"
|
|||
|
|
SUPPORTED = ("fund_quote", "general")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def loader_returning(*entries: IntentConfigEntry) -> Any:
|
|||
|
|
async def load(_agent_type: str) -> tuple[IntentConfigEntry, ...]:
|
|||
|
|
return entries
|
|||
|
|
|
|||
|
|
return load
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def classify(classifier: IntentClassifier, message: str = "159382 的行情") -> Any:
|
|||
|
|
return await classifier.classify(
|
|||
|
|
message=message,
|
|||
|
|
supported_intents=SUPPORTED,
|
|||
|
|
endpoints=[object()],
|
|||
|
|
agent_type=AGENT_TYPE,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_active_intent_config_changes_classification_prompt() -> None:
|
|||
|
|
model = RecordingModel('{"intent":"fund_quote","confidence":0.9}')
|
|||
|
|
entry = IntentConfigEntry(
|
|||
|
|
intent_code="fund_quote",
|
|||
|
|
intent_name="场内基金行情查询",
|
|||
|
|
description="查询场内基金最新净值与涨跌信息",
|
|||
|
|
examples=("帮我看看 159382 的行情", "查一下这只基金净值"),
|
|||
|
|
classifier_instruction="只处理场内基金代码与行情类问题",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
await classify(IntentClassifier(model))
|
|||
|
|
await classify(IntentClassifier(model, config_loader=loader_returning(entry)))
|
|||
|
|
|
|||
|
|
without_config, with_config = model.prompts
|
|||
|
|
# 生效配置的意图名/描述/示例/分类要求进入提示词——这就是"配置生效"的可观测差异。
|
|||
|
|
assert "查询场内基金最新净值与涨跌信息" not in without_config
|
|||
|
|
assert "场内基金行情查询" in with_config
|
|||
|
|
assert "帮我看看 159382 的行情" in with_config
|
|||
|
|
assert "只处理场内基金代码与行情类问题" in with_config
|
|||
|
|
# 严格 JSON 输出契约不因配置而改变。
|
|||
|
|
for prompt in (without_config, with_config):
|
|||
|
|
assert "请仅输出 JSON" in prompt
|
|||
|
|
assert "intent 和 confidence" in prompt
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_configured_threshold_flips_needs_clarification() -> None:
|
|||
|
|
"""同一模型输出(confidence=0.70),只有配置阈值不同,判定结果必须不同。"""
|
|||
|
|
model = RecordingModel('{"intent":"fund_quote","confidence":0.70}')
|
|||
|
|
|
|||
|
|
default = await classify(IntentClassifier(model))
|
|||
|
|
strict = await classify(
|
|||
|
|
IntentClassifier(
|
|||
|
|
model,
|
|||
|
|
config_loader=loader_returning(
|
|||
|
|
IntentConfigEntry(intent_code="fund_quote", confidence_threshold=0.9)
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
assert (default.intent, default.needs_clarification) == ("fund_quote", False)
|
|||
|
|
assert (strict.intent, strict.needs_clarification) == ("fund_quote", True)
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_config_cannot_declare_intent_outside_definition() -> None:
|
|||
|
|
"""配置里的意图必须落在 `AgentDefinition.supported_intents` 内,否则既不进提示也不放宽校验。"""
|
|||
|
|
model = RecordingModel('{"intent":"trade","confidence":1}')
|
|||
|
|
classifier = IntentClassifier(
|
|||
|
|
model,
|
|||
|
|
config_loader=loader_returning(
|
|||
|
|
IntentConfigEntry(intent_code="trade", description="客户想直接下单交易")
|
|||
|
|
),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
with pytest.raises(ValidationAgentError):
|
|||
|
|
await classify(classifier, message="帮我下单")
|
|||
|
|
|
|||
|
|
assert "客户想直接下单交易" not in model.prompts[0]
|
|||
|
|
assert "trade" not in model.prompts[0]
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_fail_closed_survives_active_config() -> None:
|
|||
|
|
"""配置生效时,格式违约仍然失败关闭。"""
|
|||
|
|
classifier = IntentClassifier(
|
|||
|
|
RecordingModel("不是 JSON"),
|
|||
|
|
config_loader=loader_returning(IntentConfigEntry(intent_code="fund_quote")),
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
with pytest.raises(RecoverableAgentError):
|
|||
|
|
await classify(classifier)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class FakeSession:
|
|||
|
|
"""只实现 `scalars`:记录编译后的 SQL,并返回预置行。"""
|
|||
|
|
|
|||
|
|
def __init__(self, rows: list[Any]) -> None:
|
|||
|
|
self.rows = rows
|
|||
|
|
self.sql: list[str] = []
|
|||
|
|
|
|||
|
|
async def scalars(self, statement: Any) -> list[Any]:
|
|||
|
|
self.sql.append(str(statement.compile(compile_kwargs={"literal_binds": True})))
|
|||
|
|
return self.rows
|
|||
|
|
|
|||
|
|
|
|||
|
|
def active_row() -> AgentIntentConfig:
|
|||
|
|
now = datetime.now(UTC).replace(tzinfo=None)
|
|||
|
|
return AgentIntentConfig(
|
|||
|
|
id=7,
|
|||
|
|
agent_type=AGENT_TYPE,
|
|||
|
|
intent_code="fund_quote",
|
|||
|
|
intent_name="场内基金行情查询",
|
|||
|
|
description="查询场内基金最新净值与涨跌信息",
|
|||
|
|
examples=["帮我看看 159382 的行情"],
|
|||
|
|
classifier_instruction="只处理行情查询",
|
|||
|
|
confidence_threshold=Decimal("0.6000"),
|
|||
|
|
max_clarification_rounds=2,
|
|||
|
|
transfer_on_failure=True,
|
|||
|
|
priority=100,
|
|||
|
|
version=1,
|
|||
|
|
status="active",
|
|||
|
|
created_by=1,
|
|||
|
|
created_at=now,
|
|||
|
|
updated_at=now,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
async def test_runtime_loader_reads_only_active_rows_and_maps_fields() -> None:
|
|||
|
|
session = FakeSession([active_row()])
|
|||
|
|
entries = await RuntimeConfigService(session).active_intents(AGENT_TYPE) # type: ignore[arg-type]
|
|||
|
|
|
|||
|
|
sql = session.sql[0]
|
|||
|
|
assert "agent_intent_config.agent_type = 'fund_query_demo'" in sql
|
|||
|
|
# 生效判定口径:只认 status='active',并尊重有效期窗口。
|
|||
|
|
assert "agent_intent_config.status = 'active'" in sql
|
|||
|
|
assert "agent_intent_config.effective_at IS NULL OR" in sql
|
|||
|
|
assert "agent_intent_config.expire_at IS NULL OR" in sql
|
|||
|
|
assert entries == (
|
|||
|
|
IntentConfigEntry(
|
|||
|
|
intent_code="fund_quote",
|
|||
|
|
intent_name="场内基金行情查询",
|
|||
|
|
description="查询场内基金最新净值与涨跌信息",
|
|||
|
|
examples=("帮我看看 159382 的行情",),
|
|||
|
|
classifier_instruction="只处理行情查询",
|
|||
|
|
confidence_threshold=0.6,
|
|||
|
|
),
|
|||
|
|
)
|