lzf_0626
7b3a72860c
客户实测反馈：「C1 客户能买什么」被引导到人工客服，而知识库里其实有答案。 实测分数：这条问句 top1 只有 0.5633、与次优差 0.0236（低于 0.07 门槛）→ 转人工； 而「C1 保守型客户可以买哪些风险等级的产品」是 0.7794，过了 0.75 硬门槛、能答。 根因是客户与知识库的用词鸿沟：客户说「C1 客户」，知识块标题写的是「C1 保守型」。 短问法少了"保守型"这个锚点就差 0.19 分——而客户不知道 C1 就等于保守型，这正是他要问的。 按 A 方案（数据问题用数据解决）为 C1-C5 各补一条 FAQ，答案全部取自 《个人投资者适当性管理指南》原文，不自行编写： - 第十二条投资者与产品匹配矩阵（各级别可购买的产品风险等级） - 第十四条硬匹配规则的跨级禁止要求 - 第十五条豁免规则（C3 买 R4、C4 买 R5 的签署揭示书与持仓上限） 知识块 631 → 636。同时修正 load 脚本自检里过时的期望：这句话现在命中 FAQ 而非 POL-AST（两者是同一份内容，只是 FAQ 的问句措辞更接近客户口语）。 验证：C1-C5 六个等级的「能买什么」问法全部直接回答、无一转人工； ruff / mypy / 468 unit+contract 全绿。
187 lines
8.2 KiB
Python
187 lines
8.2 KiB
Python
"""把 knowledge/_chunks.jsonl 灌入 Milvus 三个知识集合（临时脚本，跑完即删）。
|
||
|
||
设计要点：
|
||
|
||
1. **集合 schema** 按方案 §4.2 的统一字段（10 项），另加 5 个检索与合规必需的字段：
|
||
`chapter`/`section`（定位）、`source_file`（溯源）、`doc_no`（内部文件编号）、
|
||
`visibility`（反洗钱手册标 internal，客服侧按此过滤）。维度 1024 与
|
||
`qwen3.7-text-embedding-flash` 实测一致，索引按方案 §2.4.4：IVF_FLAT + COSINE + nlist=128。
|
||
2. **向量输入用「标题 + 正文」**而不是只喂正文：标题里带条款号与章节名（如
|
||
「第五章 … 第十三条 … 第一类：资金流转异常」），是比正文更干净的检索信号。
|
||
3. **幂等**：用 upsert，同一 doc_id 重复灌库不会产生重复行，可以反复重跑。
|
||
4. 灌完立刻做检索自检（拿几个真实客户问题去查），不看自检结果不算灌成功。
|
||
"""
|
||
|
||
import asyncio
|
||
import json
|
||
import os
|
||
from collections import defaultdict
|
||
from pathlib import Path
|
||
|
||
import httpx
|
||
from dotenv import load_dotenv
|
||
from pymilvus import DataType, MilvusClient
|
||
|
||
load_dotenv(override=False)
|
||
|
||
MILVUS_URI = os.environ.get("MILVUS_URI", "http://127.0.0.1:19530")
|
||
MILVUS_TOKEN = os.environ.get("MILVUS_TOKEN") or None
|
||
EMBED_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||
EMBED_MODEL = "qwen3.7-text-embedding-flash"
|
||
EMBED_KEY = os.environ.get("QWEN_EMBEDDING_API_KEY", "")
|
||
DIM = 1024
|
||
BATCH = 10
|
||
|
||
COLLECTIONS = ["fin_faq_collection", "fin_product_collection", "fin_policy_collection"]
|
||
|
||
# 检索自检用例：(自然语言问题, 期望命中的 doc_id 前缀)
|
||
CHECKS = [
|
||
("基金赎回到账需要多长时间", "FAQ-0016"),
|
||
# 分等级的「能买什么」现在由 FAQ 承接（与 POL-AST 的匹配矩阵是同一份内容，
|
||
# 但 FAQ 的问句措辞更接近客户口语，所以问这句话时 FAQ 会排在前面）。
|
||
("C1 保守型客户可以买哪些风险等级的产品", "FAQ"),
|
||
("C1 客户能买什么", "FAQ"),
|
||
("南方季季盈90天的起投金额是多少", "PROD"),
|
||
("开户需要准备哪些材料", "FAQ-0021"),
|
||
("业绩比较基准是什么意思", "FAQ-0019"),
|
||
("高净值客户能享受什么费率优惠", "HNW"),
|
||
]
|
||
|
||
|
||
def build_schema(client: MilvusClient) -> object:
|
||
schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False)
|
||
schema.add_field("doc_id", DataType.VARCHAR, max_length=64, is_primary=True)
|
||
schema.add_field("title", DataType.VARCHAR, max_length=1024)
|
||
schema.add_field("content", DataType.VARCHAR, max_length=16384)
|
||
schema.add_field("chapter", DataType.VARCHAR, max_length=512)
|
||
schema.add_field("section", DataType.VARCHAR, max_length=512)
|
||
schema.add_field("tags", DataType.VARCHAR, max_length=512)
|
||
schema.add_field("doc_no", DataType.VARCHAR, max_length=64)
|
||
schema.add_field("version", DataType.VARCHAR, max_length=32)
|
||
schema.add_field("effective_date", DataType.VARCHAR, max_length=32)
|
||
schema.add_field("expire_date", DataType.VARCHAR, max_length=32)
|
||
schema.add_field("source_url", DataType.VARCHAR, max_length=512)
|
||
schema.add_field("reviewer", DataType.VARCHAR, max_length=64)
|
||
schema.add_field("source_file", DataType.VARCHAR, max_length=128)
|
||
schema.add_field("visibility", DataType.VARCHAR, max_length=16)
|
||
schema.add_field("embedding", DataType.FLOAT_VECTOR, dim=DIM)
|
||
return schema
|
||
|
||
|
||
def ensure_collections(client: MilvusClient) -> None:
|
||
existing = set(client.list_collections())
|
||
for name in COLLECTIONS:
|
||
if name in existing:
|
||
print(f" 集合已存在，跳过创建：{name}")
|
||
continue
|
||
index_params = client.prepare_index_params()
|
||
index_params.add_index(
|
||
field_name="embedding", index_type="IVF_FLAT", metric_type="COSINE",
|
||
params={"nlist": 128},
|
||
)
|
||
client.create_collection(
|
||
collection_name=name, schema=build_schema(client), index_params=index_params
|
||
)
|
||
print(f" 已创建集合：{name}")
|
||
|
||
|
||
async def embed(texts: list[str]) -> list[list[float]]:
|
||
async with httpx.AsyncClient(timeout=90) as client:
|
||
response = await client.post(
|
||
f"{EMBED_BASE}/embeddings",
|
||
headers={"Authorization": f"Bearer {EMBED_KEY}"},
|
||
json={"model": EMBED_MODEL, "input": texts},
|
||
)
|
||
response.raise_for_status()
|
||
payload = response.json()
|
||
items = sorted(payload["data"], key=lambda item: item["index"])
|
||
return [item["embedding"] for item in items]
|
||
|
||
|
||
async def main() -> None:
|
||
if not EMBED_KEY:
|
||
print("缺少 QWEN_EMBEDDING_API_KEY")
|
||
return
|
||
|
||
records = [
|
||
json.loads(line)
|
||
for line in (Path("knowledge") / "_chunks.jsonl").read_text(encoding="utf-8").splitlines()
|
||
if line.strip()
|
||
]
|
||
print(f"待入库块数：{len(records)}")
|
||
|
||
client = MilvusClient(uri=MILVUS_URI, token=MILVUS_TOKEN)
|
||
print("\n== 建集合 ==")
|
||
ensure_collections(client)
|
||
|
||
grouped: dict[str, list[dict[str, object]]] = defaultdict(list)
|
||
for record in records:
|
||
grouped[str(record["collection"])].append(record)
|
||
|
||
print("\n== 生成向量并写入 ==")
|
||
for name, group in grouped.items():
|
||
rows: list[dict[str, object]] = []
|
||
for start in range(0, len(group), BATCH):
|
||
batch = group[start:start + BATCH]
|
||
vectors = await embed(
|
||
[f"{record['title']}\n{record['content']}" for record in batch]
|
||
)
|
||
for record, vector in zip(batch, vectors, strict=True):
|
||
rows.append({
|
||
"doc_id": record["doc_id"],
|
||
"title": str(record["title"])[:500],
|
||
"content": str(record["content"])[:8000],
|
||
"chapter": str(record["chapter"])[:250],
|
||
"section": str(record["section"])[:250],
|
||
"tags": str(record["tags"])[:250],
|
||
"doc_no": str(record["doc_no"])[:60],
|
||
"version": str(record["version"])[:30],
|
||
"effective_date": str(record["effective_date"])[:30],
|
||
"expire_date": str(record["expire_date"])[:30],
|
||
"source_url": str(record["source_url"])[:500],
|
||
"reviewer": str(record["reviewer"])[:60],
|
||
"source_file": str(record["source_file"])[:120],
|
||
"visibility": str(record["visibility"])[:16],
|
||
"embedding": vector,
|
||
})
|
||
print(f" {name}: 已向量化 {min(start + BATCH, len(group))}/{len(group)}")
|
||
client.upsert(collection_name=name, data=rows)
|
||
client.flush(collection_name=name)
|
||
|
||
print("\n== 各集合条目数 ==")
|
||
for name in COLLECTIONS:
|
||
stats = client.get_collection_stats(collection_name=name)
|
||
print(f" {name}: {stats.get('row_count')} 条")
|
||
|
||
print("\n== 检索自检 ==")
|
||
queries = [question for question, _ in CHECKS]
|
||
query_vectors = await embed(queries)
|
||
hits = 0
|
||
for (question, expected), vector in zip(CHECKS, query_vectors, strict=True):
|
||
results = client.search(
|
||
collection_name="fin_policy_collection", data=[vector], limit=3,
|
||
output_fields=["doc_id", "title", "visibility"],
|
||
)
|
||
top = client.search(
|
||
collection_name="fin_faq_collection", data=[vector], limit=3,
|
||
output_fields=["doc_id", "title"],
|
||
)
|
||
product = client.search(
|
||
collection_name="fin_product_collection", data=[vector], limit=3,
|
||
output_fields=["doc_id", "title"],
|
||
)
|
||
merged = [item for group in (top, product, results) for item in group[0]]
|
||
merged.sort(key=lambda item: item["distance"], reverse=True)
|
||
best = merged[0] if merged else None
|
||
# 注意：COSINE 下 pymilvus 返回的 distance 越大越相似
|
||
found = ""
|
||
if best is not None:
|
||
found = f"{best['entity']['doc_id']} (score={best['distance']:.3f}) {best['entity']['title'][:44]}"
|
||
mark = "OK " if best is not None and str(best["entity"]["doc_id"]).startswith(expected) else "检查"
|
||
hits += 1 if mark == "OK " else 0
|
||
print(f" [{mark}] {question}\n → {found}")
|
||
print(f"\n自检命中 {hits}/{len(CHECKS)}")
|
||
|
||
|
||
asyncio.run(main())
|