zhanghongyu_0626
2945108f66
- Added new configuration for knowledge base root directory in `.env.example` and `settings.py`. - Implemented `find_products` method in `CoreReadOnlyRepository` for fuzzy product search based on user queries. - Introduced `search_cs_knowledge` function in `rag_service.py` to facilitate semantic search across new `fin_*` collections. - Updated document parsing to support Markdown and YAML front-matter for knowledge base entries. - Created multiple new FAQ and policy documents in the `data/kb_collections` directory to enrich the knowledge base. This update significantly improves the knowledge retrieval capabilities for customer service interactions, ensuring more relevant and accurate responses.
131 lines
4.9 KiB
Python
131 lines
4.9 KiB
Python
"""RAG 检索编排（T21-4 · FLOW §2「milvus_tool：产品规则 RAG + source_refs」）。
|
||
|
||
职责：query 文本 → embedding（Ollama bge-m3）→ Milvus kb_product_rules
|
||
向量检索 → 返回 chunks + **source_refs 溯源清单**（source_doc_id +
|
||
source_version 必带，03-milvus-collections.md §2.3 合规口径：回答必须可溯源）。
|
||
|
||
kb_business_ops（代理人内部制度）一期不接——按 05-底座清单「代理人组开发
|
||
时再建」；本模块仅 kb_product_rules（客户 + 代理人共用，用户拍板 2026-09-07：
|
||
Tool 仅 customer/advisor 开放）。
|
||
|
||
失败口径：EmbeddingError / Milvus 异常**原样上抛**（不吞不降级）——检索
|
||
失败由调用方决定呈现（对话 Tool 层 run_tool 统一转 TOOL_ERROR 留痕），
|
||
此处假装「检索到空结果」会让 LLM 编造回答，比报错更危险。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
from app.service import embedding, milvus_service
|
||
from app.tool.milvus_tool import get_milvus_client
|
||
|
||
# 对外默认 TopK（对话 Tool 引用；脚本/服务可显式覆盖）
|
||
DEFAULT_TOP_K = 3
|
||
|
||
|
||
def _embed(query: str) -> list[float]:
|
||
"""query 向量化（测试注入点）。"""
|
||
return embedding.embed_text(query)
|
||
|
||
|
||
def _client() -> "milvus_service.MilvusClient":
|
||
"""Milvus 连接（测试注入点；与 milvus_service.milvus_client 同源）。"""
|
||
return milvus_service.milvus_client()
|
||
|
||
|
||
def _source_refs(results: list[dict[str, Any]]) -> list[dict[str, str]]:
|
||
"""结果 → 去重溯源清单（来源文档 × 版本 × 产品维度）。"""
|
||
refs: dict[tuple[str, str, str], dict[str, str]] = {}
|
||
for r in results:
|
||
key = (r.get("source_doc_id", ""), r.get("source_version", ""), r.get("product_id", ""))
|
||
refs.setdefault(
|
||
key,
|
||
{
|
||
"source_doc_id": r.get("source_doc_id", ""),
|
||
"source_version": r.get("source_version", ""),
|
||
"product_id": r.get("product_id", ""),
|
||
"product_name": r.get("product_name", ""),
|
||
},
|
||
)
|
||
return list(refs.values())
|
||
|
||
|
||
def search_knowledge(
|
||
query: str,
|
||
*,
|
||
product_id: str | None = None,
|
||
doc_type: str | None = None,
|
||
top_k: int = DEFAULT_TOP_K,
|
||
) -> dict[str, Any]:
|
||
"""知识检索主入口：query → chunks（含溯源字段）+ source_refs。
|
||
|
||
product_id / doc_type 为可选标量过滤；effective_date 合规过滤
|
||
（只返回已生效文档）内建在 milvus_service.search_kb。
|
||
"""
|
||
if not query or not query.strip():
|
||
return {"query": query.strip(), "results": [], "source_refs": []}
|
||
vector = _embed(query.strip())
|
||
client = _client()
|
||
try:
|
||
# ensure_collection 幂等防御：空库/首访时明确空结果而非报错
|
||
milvus_service.ensure_collection(client)
|
||
results = milvus_service.search_kb(
|
||
client, vector, top_k=top_k, product_id=product_id, doc_type=doc_type
|
||
)
|
||
finally:
|
||
client.close()
|
||
return {"query": query.strip(), "results": results, "source_refs": _source_refs(results)}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 客服线 · fin_* 三库 RAG（build_collections.py · 与 kb_product_rules 并存）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
_CS_INTENT_COLLECTION: dict[str, str] = {
|
||
"product_consult": "fin_product",
|
||
"policy_interpret": "fin_policy",
|
||
"faq": "fin_faq",
|
||
}
|
||
|
||
|
||
def search_cs_knowledge(intent: str, query: str, top_k: int = 5) -> tuple[str, list[dict]]:
|
||
"""客服/游客 RAG：按意图选 fin_* collection 语义检索。"""
|
||
collection = _CS_INTENT_COLLECTION.get(intent)
|
||
if not collection or not query or not query.strip():
|
||
return "", []
|
||
try:
|
||
vector = _embed(query.strip())
|
||
client = get_milvus_client()
|
||
try:
|
||
hits = client.search(collection, vector, top_k=top_k)
|
||
finally:
|
||
client.close()
|
||
except Exception:
|
||
return "", []
|
||
if not hits:
|
||
return "", []
|
||
context_parts: list[str] = []
|
||
sources: list[dict] = []
|
||
for i, hit in enumerate(hits, 1):
|
||
chunk_text = hit.get("chunk_text") or ""
|
||
source_doc = hit.get("source_doc") or hit.get("id") or ""
|
||
context_parts.append(f"[来源: {source_doc} | 片段 {i}]\n{chunk_text}")
|
||
sources.append(
|
||
{
|
||
"source_doc": source_doc,
|
||
"chunk_no": hit.get("chunk_no", i),
|
||
"score": round(float(hit.get("score", 0.0)), 4),
|
||
}
|
||
)
|
||
return "\n\n".join(context_parts), sources
|
||
|
||
|
||
class VisitorRagService:
|
||
"""游客/客户 RAG：intent → fin_* Milvus 检索 + 溯源。"""
|
||
|
||
def retrieve(self, intent: str, query: str, top_k: int = 5) -> tuple[str, list[dict]]:
|
||
if intent not in _CS_INTENT_COLLECTION:
|
||
return "", []
|
||
return search_cs_knowledge(intent, query, top_k=top_k)
|