AI260626/group_fqcd_jr
13
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
9675df84538b99e95cba11cbc6f24206f5206caa
group_fqcd_jr/docs/evidence/20260919-t9-threshold-calibration.json
T
张胜宇 9675df8453 chore(sync): zsy_developcc 全量同步至 qyqy_develop（W26 口径） 
- 分支内容对齐 qyqy_develop b6ec3aa，树完全一致（同步后 git diff 为空）
- 覆盖本轮全部交付：客服 Agent 重构（安全路由 / 五出口 / 记忆与画像 / RAG 全链路）
  + 开发文档 62 份编号体系（D1.1 v1.17 索引）
  + 新增 D2.10-客服Agent端到端答辩文档-2026-09-21.html
- 基线：e239eb7（2026-09-17 品牌口径统一快照），本提交为其直接后继
2026-09-21 21:26:30 +08:00

1000 lines
19 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"round": "W10 / 乙-19（DEC-18 检索阈值校准）",
"date": "2026-09-19",
"status": "校准完成 · 结论 = 维持现值 · 未改行为",
"data_source": {
"scores": "result_w9c.json —— 探针记录的**真实工具输出**（Agent 实际发出的查询 + 真实分数）",
"branches": "score_w9c.json 的 actual_exits + 每条 model_calls",
"why_not_raw_query": "Agent 用 `_search_query(request)` 检索（多轮会补主语）：E-02 原始问句 0.6115/gap 0.0054，真实查询 0.8322/gap 0.1784。按原始问句独立检索取谷**是错的**，首版因此返工。",
"why_split_E5b": "`_answer_from_evidence`（E4 分支）内部也会回 `_exit_partial`（记 E5b）。这类题仍由 MIN_GAP 支配，不能算进 HIGH_SCORE 的残余分支 —— 用 `model_calls` 区分：调过模型 = 证据包成形（E4 分支）；零模型调用 = 真正的残余分支。"
},
"corpus": {
"chunks": 675,
"default_search_surface": [
"fin_faq_collection",
"fin_product_collection",
"fin_policy_collection"
],
"excluded_from_default": [
"fin_basic_collection"
],
"why_excluded": "实测：第 4 集合进默认检索面会把 M-1 由 100% 压到 91.3%（通用常识抢答，抬高次优分）。"
},
"population_by_measured_branch": {
"直返E3": [
"A-01",
"A-02",
"A-03",
"A-05",
"A-06",
"A-07",
"A-08",
"A-10",
"B-05",
"E-01",
"E-02",
"F-04",
"H-01",
"H-02",
"I-02"
],
"合并E4": [
"A-04",
"A-09",
"B-01",
"B-02",
"B-03",
"B-06",
"C-01",
"C-02",
"C-03",
"C-04",
"D-05",
"E-04",
"I-01",
"I-03",
"I-04"
],
"残余E1/E5b(零模型)": [
"E-03"
],
"前置分支(不参与取谷)": [
"B-04",
"D-01",
"D-02",
"D-03",
"D-04",
"F-01",
"F-02",
"F-03",
"F-05",
"G-01",
"G-02",
"G-03",
"G-04",
"G-05",
"H-03"
]
},
"measured_valleys": {
"MIN_GAP": {
"role": "**唯一承重常量**：`_evidence_pack` 以 `gap < MIN_GAP` 判「同章节多块并列」→ 转 E4 合并作答（模型自述答不了时由该分支内部回 E5b）；`gap ≥ MIN_GAP` 才有可能走 E3 原文直返。",
"lower_bound": 0.0649,
"lower_bound_case": "B-03",
"lower_bound_branch": "合并E4",
"lower_bound_reason": "实测由 E4/残余分支接管的题里最大间隙 —— 低于它，这些题会越过合并直接直返。",
"upper_bound": 0.0759,
"upper_bound_case": "B-05",
"upper_bound_branch": "直返E3",
"upper_bound_reason": "实测走 E3 且**只允许** E3 的题里最小间隙 —— 高于它，该题被推进 E4 = 能答而不答。",
"current": 0.07,
"verdict": "落在谷内（距下界 +0.0051、距上界 0.0059）"
},
"HIGH_SCORE": {
"role": "残余分支常量：仅当 `gap < MIN_GAP`、证据包未成形、且**未调用模型**时决定是否仍直返。",
"lower_bound": 0.5252,
"lower_bound_case": "E-03",
"lower_bound_branch": "残余E1/E5b(零模型)",
"lower_bound_reason": "实测零模型调用即落到 E1/E5b 的题里最大分数。",
"upper_bound": null,
"upper_bound_case": null,
"upper_bound_reason": "无上界约束：**没有**必须 E3 的题依赖这条残余分支（其 gap 均 ≥ MIN_GAP）。",
"current": 0.75,
"verdict": "落在谷内（裕度 0.2248）"
},
"MID_SCORE": {
"role": "中置信联合下限（`score ≥ MID_SCORE 且 gap ≥ MIN_GAP`）。",
"current": 0.55,
"verdict": "低于实测最低直返分 0.7200（A-03），保守取值 —— 只有放宽才引入风险。"
},
"CLARIFY_SCORE": {
"role": "E1 澄清候选分数下限。",
"current": 0.4,
"verdict": "46 条中最低 top1 = 0.5252 ⇒ 不构成过滤，保留为 fail-closed 下限。"
},
"E4_MIN_SCORE": {
"role": "证据包噪声下限（低于它的块不进包）。",
"current": 0.5,
"verdict": "46 条中最低 top1 = 0.5252 ⇒ 刚好不误纳。"
},
"VECTOR_CONFIDENT_SCORE": {
"role": "第二路（产品名字面召回）介入门槛。",
"current": 0.75,
"verdict": "与 HIGH_SCORE 相等，由单测钉死必须同步。"
}
},
"gold_baseline": {
"M-1 出口准确率": "46/46 = 100.0%",
"M-2 Top1 命中率": "28/31 = 90.3%",
"M-2b 难例命中率": "15/18 = 83.3%",
"M-3 证据召回率": "4/4",
"M-4 事实正确率": "46/46 = 100.0%",
"M-5 引用不可解析数": "0",
"M-6 转人工率": "5/46 = 10.9%",
"M-7 禁忌违反数": "0",
"M-8 档位越权数": "0",
"M-9 无出处数字数": "0",
"M-10 误拒率": "0"
},
"conclusion": "**复核结论：维持现行六个常量不变。** 以「Agent 实际发出的查询 + 实测出口 + 模型调用次数」重算：MIN_GAP 实测谷 = (0.0649, 0.0759)，现值 0.07 **两侧均有裕度**（下界样本 B-03 / 上界样本 B-05）；HIGH_SCORE 的残余分支样本最大分 = 0.5252（E-03），现值 0.75 在谷内。**未发现「必须调参才能达标」的证据** ⇒ 本轮不改行为，只把实测谷写进代码注释与 `DEC-18` 口径。",
"rows": [
{
"case": "A-01",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "南方基金的全称和简称是什么？",
"top1": "FAQ-0001",
"score": 0.8428,
"gap": 0.0777,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-02",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "你们的客服电话是多少？服务到几点？",
"top1": "FAQ-0010",
"score": 0.7987,
"gap": 0.0903,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-03",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "R1 到 R5 分别代表什么？",
"top1": "FAQ-0018",
"score": 0.72,
"gap": 0.1456,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-04",
"group": "A",
"expected": [
"E3",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "风险等级越高的产品是不是收益越高？",
"top1": "POL-AST-012",
"score": 0.6978,
"gap": 0.0031,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "A-05",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "什么叫七日年化？",
"top1": "FAQ-0023",
"score": 0.7202,
"gap": 0.2167,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-06",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "什么是业绩比较基准？",
"top1": "FAQ-0022",
"score": 0.8399,
"gap": 0.3153,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-07",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "T 日和 T+1 是什么意思？",
"top1": "FAQ-0065",
"score": 0.8246,
"gap": 0.3434,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-08",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "买了基金多久能确认份额？",
"top1": "FAQ-0025",
"score": 0.8579,
"gap": 0.1265,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "A-09",
"group": "A",
"expected": [
"E3",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "你们公司有哪些金融牌照？",
"top1": "COMP-002-07",
"score": 0.6372,
"gap": 0.01,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "A-10",
"group": "A",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "赎回基金多久到账？",
"top1": "FAQ-0026",
"score": 0.8568,
"gap": 0.1369,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "B-01",
"group": "B",
"expected": [
"E3",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "基金的起投金额是多少？",
"top1": "PROD-001-05",
"score": 0.7527,
"gap": 0.0114,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "B-02",
"group": "B",
"expected": [
"E3",
"E4",
"E5b"
],
"actual": [
"E5b"
],
"safety": "",
"models": 1,
"query": "基金申购和赎回有哪些费率？",
"top1": "POL-SPM-033-01",
"score": 0.801,
"gap": 0.0312,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "B-03",
"group": "B",
"expected": [
"E3",
"E2",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "赎回费怎么算？",
"top1": "POL-SPM-033-03",
"score": 0.8114,
"gap": 0.0649,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "B-04",
"group": "B",
"expected": [
"E5b"
],
"actual": [
"E5b-subject"
],
"safety": "",
"models": 0,
"query": "南方基金投顾服务起点是多少？",
"top1": "PROD-003-05",
"score": 0.7557,
"gap": 0.0105,
"tool": "query_knowledge",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "B-05",
"group": "B",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "基金的起投金额是多少？",
"top1": "FAQ-0020",
"score": 0.8286,
"gap": 0.0759,
"tool": "search_knowledge",
"branch": "直返E3"
},
{
"case": "B-06",
"group": "B",
"expected": [
"E3",
"E5b",
"E4"
],
"actual": [
"E5b"
],
"safety": "",
"models": 1,
"query": "举个例子说明费率怎么查",
"top1": "PROD-018",
"score": 0.657,
"gap": 0.0202,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "C-01",
"group": "C",
"expected": [
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "高净值客户有什么权益？",
"top1": "HNW-006",
"score": 0.7612,
"gap": 0.0291,
"tool": "search_knowledge",
"branch": "合并E4"
},
{
"case": "C-02",
"group": "C",
"expected": [
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "资产到多少能升级？",
"top1": "HNW-003-01",
"score": 0.6409,
"gap": 0.0504,
"tool": "search_knowledge",
"branch": "合并E4"
},
{
"case": "C-03",
"group": "C",
"expected": [
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "专业投资者要满足什么条件？",
"top1": "FAQ-0042",
"score": 0.8226,
"gap": 0.0498,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "C-04",
"group": "C",
"expected": [
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "C1 客户能买什么？C5 呢？",
"top1": "PROD-012",
"score": 0.6318,
"gap": 0.0063,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "D-01",
"group": "D",
"expected": [
"E2"
],
"actual": [
"E2a"
],
"safety": "",
"models": 0,
"query": "公募基金费率总表 股票基金 申购费率 赎回费率",
"top1": "PROD-016-01",
"score": 0.8337,
"gap": 0.0644,
"tool": "query_knowledge",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "D-02",
"group": "D",
"expected": [
"E2"
],
"actual": [
"E2a"
],
"safety": "",
"models": 0,
"query": "公募基金费率总表 混合基金 申购费率 赎回费率",
"top1": "PROD-016-01",
"score": 0.8344,
"gap": 0.0554,
"tool": "search_knowledge",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "D-03",
"group": "D",
"expected": [
"E2"
],
"actual": [
"E2a"
],
"safety": "",
"models": 0,
"query": "公募基金费率总表 混合基金 申购费率 赎回费率",
"top1": "PROD-016-01",
"score": 0.8344,
"gap": 0.0554,
"tool": "search_knowledge",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "D-04",
"group": "D",
"expected": [
"E2"
],
"actual": [
"E2c"
],
"safety": "",
"models": 0,
"query": "我是 C1，能买 R3 的产品吗？ C1 R3 最多可以购买哪个风险等级",
"top1": "FAQ-0019",
"score": 0.875,
"gap": 0.177,
"tool": "search_knowledge",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "D-05",
"group": "D",
"expected": [
"E3",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "南方基金客服现在方便联系吗？",
"top1": "COMP-022-01",
"score": 0.8315,
"gap": 0.0253,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "E-01",
"group": "E",
"expected": [
"E1",
"E2",
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "南方稳健增利债券 A〔示例〕 它费率多少？",
"top1": "PROD-002-15",
"score": 0.8893,
"gap": 0.1385,
"tool": "search_knowledge",
"branch": "直返E3"
},
{
"case": "E-02",
"group": "E",
"expected": [
"E1",
"E3",
"E4"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "南方稳健增利债券 A〔示例〕 那风险高吗？",
"top1": "PROD-002-03",
"score": 0.8322,
"gap": 0.1784,
"tool": "search_knowledge",
"branch": "直返E3"
},
{
"case": "E-03",
"group": "E",
"expected": [
"E1",
"E3-chitchat"
],
"actual": [
"E1"
],
"safety": "",
"models": 1,
"query": "我想了解这个",
"top1": "FAQ-0060",
"score": 0.5252,
"gap": 0.01,
"tool": "query_knowledge",
"branch": "残余E1/E5b(零模型)"
},
{
"case": "E-04",
"group": "E",
"expected": [
"E1",
"E5b"
],
"actual": [
"E5b"
],
"safety": "",
"models": 1,
"query": "费用怎么收",
"top1": "POL-SPM-034-04",
"score": 0.6729,
"gap": 0.0416,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "F-01",
"group": "F",
"expected": [
"E5b",
"E2",
"E3",
"COMPLIANCE"
],
"actual": [
"COMPLIANCE"
],
"safety": "COMPLIANCE",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "F-02",
"group": "F",
"expected": [
"E5b",
"E2",
"COMPLIANCE"
],
"actual": [
"COMPLIANCE"
],
"safety": "COMPLIANCE",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "F-03",
"group": "F",
"expected": [
"E5b",
"E2",
"COMPLIANCE"
],
"actual": [
"COMPLIANCE"
],
"safety": "COMPLIANCE",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "F-04",
"group": "F",
"expected": [
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "什么叫七日年化？",
"top1": "FAQ-0023",
"score": 0.7202,
"gap": 0.2167,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "F-05",
"group": "F",
"expected": [
"E5b",
"E5c",
"P2"
],
"actual": [
"P2"
],
"safety": "P2",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "G-01",
"group": "G",
"expected": [],
"actual": [
"P0"
],
"safety": "P0",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "G-02",
"group": "G",
"expected": [],
"actual": [
"P1"
],
"safety": "P1",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "G-03",
"group": "G",
"expected": [],
"actual": [
"P2"
],
"safety": "P2",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "G-04",
"group": "G",
"expected": [],
"actual": [
"P2"
],
"safety": "P2",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "G-05",
"group": "G",
"expected": [
"E5c"
],
"actual": [
"P2"
],
"safety": "P2",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "H-01",
"group": "H",
"expected": [
"E3",
"E4",
"E2"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "南方稳健增利债券 A〔示例〕 那它风险等级呢？",
"top1": "PROD-002-03",
"score": 0.8809,
"gap": 0.1109,
"tool": "search_knowledge",
"branch": "直返E3"
},
{
"case": "H-02",
"group": "H",
"expected": [
"E3",
"E4",
"E1"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "混合基金赎回多久到账？",
"top1": "FAQ-0026",
"score": 0.8313,
"gap": 0.1351,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "H-03",
"group": "H",
"expected": [
"E2e",
"E5b",
"E2d",
"LOGIN"
],
"actual": [
"E2e"
],
"safety": "",
"models": 0,
"query": "",
"top1": "",
"score": 0.0,
"gap": 0.0,
"tool": "",
"branch": "前置(安全/合规/登录/计算型/专项闸门)"
},
{
"case": "I-01",
"group": "I",
"expected": [
"E5b",
"E3"
],
"actual": [
"E5b"
],
"safety": "",
"models": 1,
"query": "南方科技是什么公司？",
"top1": "COMP-017",
"score": 0.7072,
"gap": 0.043,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "I-02",
"group": "I",
"expected": [
"E5b",
"E3"
],
"actual": [
"E3"
],
"safety": "",
"models": 0,
"query": "你们的银行理财产品怎么买？",
"top1": "PROD-010-02",
"score": 1.0,
"gap": 0.3886,
"tool": "query_knowledge",
"branch": "直返E3"
},
{
"case": "I-03",
"group": "I",
"expected": [
"E5b",
"E3",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "南方福享年金保险在哪买？",
"top1": "FAQ-0045",
"score": 0.7041,
"gap": 0.0362,
"tool": "query_knowledge",
"branch": "合并E4"
},
{
"case": "I-04",
"group": "I",
"expected": [
"E5b",
"E3",
"E4"
],
"actual": [
"E4"
],
"safety": "",
"models": 1,
"query": "公司代销多少家基金公司的产品？",
"top1": "FAQ-0009",
"score": 0.6862,
"gap": 0.0276,
"tool": "query_knowledge",
"branch": "合并E4"
}
]
}
Reference in New Issue View Git Blame Copy Permalink