docs: 品牌全量口径统一为「南方基金」+ 作废文档清理
1) 客服 Agent 四份交付文档 + 构建脚手架：品牌由包装占位 XX科技 / 旧名 南方财富 统一为南方基金（热线 400-889-8899 / 官网 nffund.com），系统名改为「智能服务系统」； 同步追加 §0.4 修订记录行，工程记录行保留原占位字面以支撑硬编码扫描验收。 2) 开发文档：清理 28 份已作废/残留文档（14 份移出归档 + 14 份仓库副本）， 新增《文档规整方案与开发前待决事项-2026-09-17》。 3) 客服agent 四份交付文档首次纳入本分支。
This commit is contained in:
张胜宇
@@ -0,0 +1,186 @@
|
||||
"""把 knowledge/_chunks.jsonl 灌入 Milvus 三个知识集合（临时脚本，跑完即删）。
|
||||
|
||||
设计要点：
|
||||
|
||||
1. **集合 schema** 按方案 §4.2 的统一字段（10 项），另加 5 个检索与合规必需的字段：
|
||||
`chapter`/`section`（定位）、`source_file`（溯源）、`doc_no`（内部文件编号）、
|
||||
`visibility`（反洗钱手册标 internal，客服侧按此过滤）。维度 1024 与
|
||||
`qwen3.7-text-embedding-flash` 实测一致，索引按方案 §2.4.4：IVF_FLAT + COSINE + nlist=128。
|
||||
2. **向量输入用「标题 + 正文」**而不是只喂正文：标题里带条款号与章节名（如
|
||||
「第五章 … 第十三条 … 第一类：资金流转异常」），是比正文更干净的检索信号。
|
||||
3. **幂等**：用 upsert，同一 doc_id 重复灌库不会产生重复行，可以反复重跑。
|
||||
4. 灌完立刻做检索自检（拿几个真实客户问题去查），不看自检结果不算灌成功。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from dotenv import load_dotenv
|
||||
from pymilvus import DataType, MilvusClient
|
||||
|
||||
load_dotenv(override=False)
|
||||
|
||||
MILVUS_URI = os.environ.get("MILVUS_URI", "http://127.0.0.1:19530")
|
||||
MILVUS_TOKEN = os.environ.get("MILVUS_TOKEN") or None
|
||||
EMBED_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
EMBED_MODEL = "qwen3.7-text-embedding-flash"
|
||||
EMBED_KEY = os.environ.get("QWEN_EMBEDDING_API_KEY", "")
|
||||
DIM = 1024
|
||||
BATCH = 10
|
||||
|
||||
COLLECTIONS = ["fin_faq_collection", "fin_product_collection", "fin_policy_collection"]
|
||||
|
||||
# 检索自检用例：(自然语言问题, 期望命中的 doc_id 前缀)
|
||||
CHECKS = [
|
||||
("基金赎回到账需要多长时间", "FAQ-0016"),
|
||||
# 分等级的「能买什么」现在由 FAQ 承接（与 POL-AST 的匹配矩阵是同一份内容，
|
||||
# 但 FAQ 的问句措辞更接近客户口语，所以问这句话时 FAQ 会排在前面）。
|
||||
("C1 保守型客户可以买哪些风险等级的产品", "FAQ"),
|
||||
("C1 客户能买什么", "FAQ"),
|
||||
("南方季季盈90天的起投金额是多少", "PROD"),
|
||||
("开户需要准备哪些材料", "FAQ-0021"),
|
||||
("业绩比较基准是什么意思", "FAQ-0019"),
|
||||
("高净值客户能享受什么费率优惠", "HNW"),
|
||||
]
|
||||
|
||||
|
||||
def build_schema(client: MilvusClient) -> object:
|
||||
schema = MilvusClient.create_schema(auto_id=False, enable_dynamic_field=False)
|
||||
schema.add_field("doc_id", DataType.VARCHAR, max_length=64, is_primary=True)
|
||||
schema.add_field("title", DataType.VARCHAR, max_length=1024)
|
||||
schema.add_field("content", DataType.VARCHAR, max_length=16384)
|
||||
schema.add_field("chapter", DataType.VARCHAR, max_length=512)
|
||||
schema.add_field("section", DataType.VARCHAR, max_length=512)
|
||||
schema.add_field("tags", DataType.VARCHAR, max_length=512)
|
||||
schema.add_field("doc_no", DataType.VARCHAR, max_length=64)
|
||||
schema.add_field("version", DataType.VARCHAR, max_length=32)
|
||||
schema.add_field("effective_date", DataType.VARCHAR, max_length=32)
|
||||
schema.add_field("expire_date", DataType.VARCHAR, max_length=32)
|
||||
schema.add_field("source_url", DataType.VARCHAR, max_length=512)
|
||||
schema.add_field("reviewer", DataType.VARCHAR, max_length=64)
|
||||
schema.add_field("source_file", DataType.VARCHAR, max_length=128)
|
||||
schema.add_field("visibility", DataType.VARCHAR, max_length=16)
|
||||
schema.add_field("embedding", DataType.FLOAT_VECTOR, dim=DIM)
|
||||
return schema
|
||||
|
||||
|
||||
def ensure_collections(client: MilvusClient) -> None:
|
||||
existing = set(client.list_collections())
|
||||
for name in COLLECTIONS:
|
||||
if name in existing:
|
||||
print(f" 集合已存在，跳过创建：{name}")
|
||||
continue
|
||||
index_params = client.prepare_index_params()
|
||||
index_params.add_index(
|
||||
field_name="embedding", index_type="IVF_FLAT", metric_type="COSINE",
|
||||
params={"nlist": 128},
|
||||
)
|
||||
client.create_collection(
|
||||
collection_name=name, schema=build_schema(client), index_params=index_params
|
||||
)
|
||||
print(f" 已创建集合：{name}")
|
||||
|
||||
|
||||
async def embed(texts: list[str]) -> list[list[float]]:
|
||||
async with httpx.AsyncClient(timeout=90) as client:
|
||||
response = await client.post(
|
||||
f"{EMBED_BASE}/embeddings",
|
||||
headers={"Authorization": f"Bearer {EMBED_KEY}"},
|
||||
json={"model": EMBED_MODEL, "input": texts},
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
items = sorted(payload["data"], key=lambda item: item["index"])
|
||||
return [item["embedding"] for item in items]
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
if not EMBED_KEY:
|
||||
print("缺少 QWEN_EMBEDDING_API_KEY")
|
||||
return
|
||||
|
||||
records = [
|
||||
json.loads(line)
|
||||
for line in (Path("knowledge") / "_chunks.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
print(f"待入库块数：{len(records)}")
|
||||
|
||||
client = MilvusClient(uri=MILVUS_URI, token=MILVUS_TOKEN)
|
||||
print("\n== 建集合 ==")
|
||||
ensure_collections(client)
|
||||
|
||||
grouped: dict[str, list[dict[str, object]]] = defaultdict(list)
|
||||
for record in records:
|
||||
grouped[str(record["collection"])].append(record)
|
||||
|
||||
print("\n== 生成向量并写入 ==")
|
||||
for name, group in grouped.items():
|
||||
rows: list[dict[str, object]] = []
|
||||
for start in range(0, len(group), BATCH):
|
||||
batch = group[start:start + BATCH]
|
||||
vectors = await embed(
|
||||
[f"{record['title']}\n{record['content']}" for record in batch]
|
||||
)
|
||||
for record, vector in zip(batch, vectors, strict=True):
|
||||
rows.append({
|
||||
"doc_id": record["doc_id"],
|
||||
"title": str(record["title"])[:500],
|
||||
"content": str(record["content"])[:8000],
|
||||
"chapter": str(record["chapter"])[:250],
|
||||
"section": str(record["section"])[:250],
|
||||
"tags": str(record["tags"])[:250],
|
||||
"doc_no": str(record["doc_no"])[:60],
|
||||
"version": str(record["version"])[:30],
|
||||
"effective_date": str(record["effective_date"])[:30],
|
||||
"expire_date": str(record["expire_date"])[:30],
|
||||
"source_url": str(record["source_url"])[:500],
|
||||
"reviewer": str(record["reviewer"])[:60],
|
||||
"source_file": str(record["source_file"])[:120],
|
||||
"visibility": str(record["visibility"])[:16],
|
||||
"embedding": vector,
|
||||
})
|
||||
print(f" {name}: 已向量化 {min(start + BATCH, len(group))}/{len(group)}")
|
||||
client.upsert(collection_name=name, data=rows)
|
||||
client.flush(collection_name=name)
|
||||
|
||||
print("\n== 各集合条目数 ==")
|
||||
for name in COLLECTIONS:
|
||||
stats = client.get_collection_stats(collection_name=name)
|
||||
print(f" {name}: {stats.get('row_count')} 条")
|
||||
|
||||
print("\n== 检索自检 ==")
|
||||
queries = [question for question, _ in CHECKS]
|
||||
query_vectors = await embed(queries)
|
||||
hits = 0
|
||||
for (question, expected), vector in zip(CHECKS, query_vectors, strict=True):
|
||||
results = client.search(
|
||||
collection_name="fin_policy_collection", data=[vector], limit=3,
|
||||
output_fields=["doc_id", "title", "visibility"],
|
||||
)
|
||||
top = client.search(
|
||||
collection_name="fin_faq_collection", data=[vector], limit=3,
|
||||
output_fields=["doc_id", "title"],
|
||||
)
|
||||
product = client.search(
|
||||
collection_name="fin_product_collection", data=[vector], limit=3,
|
||||
output_fields=["doc_id", "title"],
|
||||
)
|
||||
merged = [item for group in (top, product, results) for item in group[0]]
|
||||
merged.sort(key=lambda item: item["distance"], reverse=True)
|
||||
best = merged[0] if merged else None
|
||||
# 注意：COSINE 下 pymilvus 返回的 distance 越大越相似
|
||||
found = ""
|
||||
if best is not None:
|
||||
found = f"{best['entity']['doc_id']} (score={best['distance']:.3f}) {best['entity']['title'][:44]}"
|
||||
mark = "OK " if best is not None and str(best["entity"]["doc_id"]).startswith(expected) else "检查"
|
||||
hits += 1 if mark == "OK " else 0
|
||||
print(f" [{mark}] {question}\n → {found}")
|
||||
print(f"\n自检命中 {hits}/{len(CHECKS)}")
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user