197 lines
8.7 KiB
Python
197 lines
8.7 KiB
Python
|
"""`W29` · NL2SQL 接线守卫、只读边界回归锁与「写意图」判据。
|
|||
|
|
|
|||
|
|
## 这个文件为什么存在
|
|||
|
|
|
|||
|
|
`W29` 把 `agent_tools/financial_nl2sql:financial_query` 发布出去后，端到端实测
|
|||
|
|
（`_w29_e2e_check.py`）**量出**一个判据缺口：`RuleBasedFinancialPlanner` 对问句里的
|
|||
|
|
**写意图动词**（删除 / 修改 / 清空 / 撤销 / 更新 / 导出 / 改成）**毫无识别** ——
|
|||
|
|
「删除所有客户的持仓记录」被判成「查持仓」，返回 `status="ready"` 并生成一段 SELECT，
|
|||
|
|
**8 条写意图问句 8/8 复现**。
|
|||
|
|
|
|||
|
|
当时数据安全**并没有破**（SQL 仍是 SELECT，被 `_safe_sql_check` 兜住），所以缺口定性是
|
|||
|
|
「答复与诉求不符」而非「越权写库」；但一旦将来给该工具加写能力，这里就是起点。
|
|||
|
|
|
|||
|
|
## 缺口已按会签流程修复（2026-09-22）
|
|||
|
|
|
|||
|
|
`app/service/financial_nl2sql_service.py` **原本不在 `docs/48` 白名单任何一档**
|
|||
|
|
（等于「白名单之外一律不动」）⇒ 先补会签单（`docs/49` 组 5 · 会签 18）、登记为类 3，
|
|||
|
|
获批后才改动。修法 = `RuleBasedFinancialPlanner.plan()` 入口的**写意图预检**
|
|||
|
|
+ `_validate_plan()` 的**不可执行计划优先**判据 ⇒ 走现成的 `status="rejected"` 分支
|
|||
|
|
（未新增代码路径 —— 这是会签单「最小化边界」的要求）。
|
|||
|
|
|
|||
|
|
## 本文件守三件事
|
|||
|
|
|
|||
|
|
1. **只读护栏回归锁**：即使判据将来退化、写意图又被误判，产出的 SQL 也必须是
|
|||
|
|
SELECT-only、无写关键字、带 LIMIT。它与判据**互为独立防线**，必须一直绿。
|
|||
|
|
2. **写意图必须被拒**：修复后由 `xfail` 转为正式断言。
|
|||
|
|
3. **误杀边界**：中文的「修改 / 更新 / 变更 / 导出」在**查询语境**同样高频
|
|||
|
|
（「费率变更历史」），判据不得把它们一并拦下 —— 这是本次修复最容易出错的地方。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import asyncio
|
|||
|
|
import re
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
|
|||
|
|
from app.core.contracts import RequestContext
|
|||
|
|
from app.core.nl2sql_catalog import ALLOWED_TABLES
|
|||
|
|
from app.core.nl2sql_contracts import FinancialNL2SQLInput
|
|||
|
|
from app.service.financial_nl2sql_service import FinancialNL2SQLService
|
|||
|
|
|
|||
|
|
WRITE_WORDS = ("INSERT", "UPDATE", "DELETE", "DROP", "ALTER", "TRUNCATE", "GRANT", "REVOKE")
|
|||
|
|
|
|||
|
|
#: 写意图问句族。**当前 8/8 会被误判成查询**（实测），故 `xfail`。
|
|||
|
|
WRITE_INTENTS = (
|
|||
|
|
"删除所有客户的持仓记录",
|
|||
|
|
"修改这只基金的风险等级",
|
|||
|
|
"清空我的交易流水",
|
|||
|
|
"撤销上笔委托",
|
|||
|
|
"更新产品费率表",
|
|||
|
|
"导出所有客户手机号",
|
|||
|
|
"把客户A的余额改成100万",
|
|||
|
|
"删除客户资料",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
#: 只读问句族：这些必须一直能正常作答（防止守卫改过头）。
|
|||
|
|
READ_INTENTS = (
|
|||
|
|
"查询近30天净值",
|
|||
|
|
"最近30天成交金额统计",
|
|||
|
|
"查询我的持仓",
|
|||
|
|
"查询产品费率",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _context(*, roles=("operator",), permissions=("financial:nl2sql:read",), scope="all"):
|
|||
|
|
return RequestContext(
|
|||
|
|
user_id="1",
|
|||
|
|
trace_id="w29-test",
|
|||
|
|
roles=roles,
|
|||
|
|
customer_ids=("1",),
|
|||
|
|
data_scope=scope,
|
|||
|
|
portal="api",
|
|||
|
|
permissions=permissions,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _dry_run(question: str, context: RequestContext | None = None) -> dict:
|
|||
|
|
"""`dry_run=True`：只走计划 + 编译 + 只读校验，**不执行 SQL**（测试不依赖库）。"""
|
|||
|
|
return asyncio.run(
|
|||
|
|
FinancialNL2SQLService().query(
|
|||
|
|
FinancialNL2SQLInput(question=question, dry_run=True),
|
|||
|
|
context or _context(),
|
|||
|
|
)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _is_readonly(sql: str) -> bool:
|
|||
|
|
upper = sql.upper()
|
|||
|
|
return upper.startswith("SELECT") and not any(word in upper for word in WRITE_WORDS)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("question", WRITE_INTENTS)
|
|||
|
|
def test_sql_stays_readonly_even_when_intent_misjudged(question: str) -> None:
|
|||
|
|
"""**安全兜底回归锁**：写意图被误判时，产出的 SQL 也必须是只读的。
|
|||
|
|
|
|||
|
|
这条测的不是「规划器对」，而是「**即使规划器错了也不会写库**」——
|
|||
|
|
只读边界由 `_safe_sql_check`（SELECT 开头 / 禁 `BANNED_SQL` / 禁 `SELECT *` /
|
|||
|
|
表集合与计划一致）与工具本身的只读定义共同保证，不依赖规划器的语义理解。
|
|||
|
|
"""
|
|||
|
|
result = _dry_run(question)
|
|||
|
|
sql = result.get("sql")
|
|||
|
|
if sql is None:
|
|||
|
|
# 若将来改成直接拒绝，也是可接受的更优结果（见下方 xfail 用例）。
|
|||
|
|
assert result["status"] in {"rejected", "need_confirmation", "unknown"}
|
|||
|
|
return
|
|||
|
|
assert _is_readonly(sql), f"写意图问句产出了非只读 SQL：{sql}"
|
|||
|
|
assert "LIMIT" in sql.upper(), f"SQL 缺少 LIMIT：{sql}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("question", WRITE_INTENTS)
|
|||
|
|
def test_write_intents_are_rejected(question: str) -> None:
|
|||
|
|
"""**写意图必须被拒**（会签项 18 落地后由 `xfail` 转为正式断言）。
|
|||
|
|
|
|||
|
|
拒绝形态刻意断言为 `status="rejected"` 而**不是** `need_confirmation`：
|
|||
|
|
后者会让客户以为「补充信息就能办」，而这件事本工具**永远不办**。
|
|||
|
|
"""
|
|||
|
|
result = _dry_run(question)
|
|||
|
|
assert result["status"] == "rejected", (
|
|||
|
|
f"写意图问句未被拒绝：status={result['status']}｜message={result.get('message')}｜"
|
|||
|
|
f"生成的 SQL={(result.get('sql') or '(无)')[:120]}"
|
|||
|
|
)
|
|||
|
|
assert "只读" in str(result.get("message") or ""), "拒绝话术必须说明只读边界"
|
|||
|
|
assert result.get("sql") in (None, ""), "被拒的请求不得产出任何 SQL"
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("question", READ_INTENTS)
|
|||
|
|
def test_read_intents_still_answerable(question: str) -> None:
|
|||
|
|
"""只读问句必须仍能出计划 + 出 SQL（守卫不得改过头）。"""
|
|||
|
|
result = _dry_run(question)
|
|||
|
|
assert result["status"] in {"ready", "success", "need_confirmation"}, result["message"]
|
|||
|
|
if result["status"] in {"ready", "success"}:
|
|||
|
|
sql = result["sql"]
|
|||
|
|
assert sql and _is_readonly(sql)
|
|||
|
|
assert "LIMIT" in sql.upper()
|
|||
|
|
|
|||
|
|
|
|||
|
|
#: **误杀边界**：这些问句含「修改 / 更新 / 变更 / 调整 / 导出」这类词，
|
|||
|
|
#: 但意图是**看**不是**改**。中文里这组词在查询语境高频 ——
|
|||
|
|
#: 判据把它们一并拦下就是误杀，而且会误杀真实业务里很自然的**续问**
|
|||
|
|
#: （客户刚看到费率，接着问「之前变更过吗」）。
|
|||
|
|
QUERY_CONTEXT_INTENTS = (
|
|||
|
|
"查询费率变更历史",
|
|||
|
|
"看一下这支基金的费率调整记录",
|
|||
|
|
"最近一次更新的时间是什么时候",
|
|||
|
|
"查一下产品资料的更新时间",
|
|||
|
|
"统计近30天的交易明细",
|
|||
|
|
"看看我的持仓变动情况",
|
|||
|
|
"查询所有客户的持仓统计",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("question", QUERY_CONTEXT_INTENTS)
|
|||
|
|
def test_query_context_is_not_blocked(question: str) -> None:
|
|||
|
|
"""**误杀边界**：查询语境的歧义写动词不得被拦（本次修复最容易出错的地方）。"""
|
|||
|
|
result = _dry_run(question)
|
|||
|
|
assert result["status"] != "rejected", (
|
|||
|
|
f"查询语境被误判为写意图：{question}｜message={result.get('message')}"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_customer_role_cannot_use_nl2sql() -> None:
|
|||
|
|
"""**角色边界**：`customer` 不得使用 NL2SQL（档 B 口径：不开放给客户）。"""
|
|||
|
|
from app.core.errors import ForbiddenAgentError
|
|||
|
|
|
|||
|
|
context = _context(roles=("customer",), permissions=("financial:nl2sql:read",), scope="self")
|
|||
|
|
with pytest.raises(ForbiddenAgentError):
|
|||
|
|
_dry_run("查询我的持仓", context)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_missing_permission_is_rejected() -> None:
|
|||
|
|
"""**权限边界**：缺 `financial:nl2sql:read` 必须被拒。"""
|
|||
|
|
from app.core.errors import ForbiddenAgentError
|
|||
|
|
|
|||
|
|
with pytest.raises(ForbiddenAgentError):
|
|||
|
|
_dry_run("查询我的持仓", _context(permissions=()))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_customer_scope_is_injected_from_context_not_prompt() -> None:
|
|||
|
|
"""**主体过滤来自鉴权上下文**：同一句问话，只改 `data_scope`，SQL 就变。"""
|
|||
|
|
self_sql = _dry_run("查询我的持仓", _context(scope="self"))["sql"]
|
|||
|
|
all_sql = _dry_run("查询我的持仓", _context(scope="all"))["sql"]
|
|||
|
|
assert self_sql is not None and all_sql is not None
|
|||
|
|
assert "customer_id IN" in self_sql, f"data_scope=self 未注入主体过滤：{self_sql}"
|
|||
|
|
assert "customer_id IN" not in all_sql, f"data_scope=all 不应注入：{all_sql}"
|
|||
|
|
|
|||
|
|
|
|||
|
|
@pytest.mark.parametrize("question", ("查询所有用户的登录账号密码", "查询所有角色的权限配置"))
|
|||
|
|
def test_tables_outside_allowlist_never_appear(question: str) -> None:
|
|||
|
|
"""**越表白名单**：SQL 里只能出现 `nl2sql_catalog` 白名单内的表。"""
|
|||
|
|
result = _dry_run(question)
|
|||
|
|
sql = result.get("sql")
|
|||
|
|
if sql is None:
|
|||
|
|
return
|
|||
|
|
refs = set(re.findall(r"\b(?:FROM|JOIN)\s+([A-Za-z_][A-Za-z0-9_]*)", sql, re.I))
|
|||
|
|
assert refs <= ALLOWED_TABLES, f"出现白名单外表：{refs - ALLOWED_TABLES}"
|