zhanghongyu_0626
b841f68295
- Added a new visitor chat API endpoint (`/api/chat/visitor`) to allow unauthenticated users to engage in conversations without requiring customer data. - Introduced a visitor context dependency to manage visitor interactions seamlessly. - Enhanced the chat API to support explicit session termination and improved response handling for customer service interactions. - Updated the database configuration to include Redis client support for caching visitor data. - Added a new customer note repository to persist user notes independently of the L1 profile slots. This update significantly improves the customer service experience by enabling visitor interactions and ensuring efficient data handling for both registered and unregistered users.
114 lines
3.7 KiB
Python
114 lines
3.7 KiB
Python
"""知识库入库脚本：解析桌面知识库文件 → 向量化 → 存入 Milvus。
|
|
|
|
用法：
|
|
python scripts/kb/build_collections.py # 仅创建不存在的 collection
|
|
python scripts/kb/build_collections.py --rebuild # 先删除再重建
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# 把项目根目录加入 sys.path
|
|
_ROOT = Path(__file__).resolve().parent.parent.parent
|
|
sys.path.insert(0, str(_ROOT))
|
|
|
|
from app.config.settings import settings
|
|
from app.tool.document_parser import parse_collection_dir
|
|
from app.tool.embedding_tool import get_embedder
|
|
from app.tool.milvus_tool import get_milvus_client
|
|
|
|
|
|
# Collection → 子目录映射
|
|
KB_LAYOUT: dict[str, str] = {
|
|
"fin_faq": "fin_faq_collection",
|
|
"fin_product": "fin_product_collection",
|
|
"fin_policy": "fin_policy_collection",
|
|
}
|
|
|
|
# 每个 collection 的标量字段（从 chunk.metadata 中取值）
|
|
COLLECTION_FIELDS: dict[str, list[str]] = {
|
|
"fin_faq": ["question", "answer", "category", "source_doc", "chunk_no", "chunk_text"],
|
|
"fin_product": ["product_name", "risk_level", "doc_type", "source_doc", "chunk_no", "chunk_text"],
|
|
"fin_policy": ["policy_name", "chapter", "source_doc", "chunk_no", "chunk_text"],
|
|
}
|
|
|
|
|
|
def chunk_to_record(chunk, collection: str, vector: list[float]) -> dict:
|
|
"""把 Chunk + 向量转为 Milvus 插入记录。"""
|
|
record = {
|
|
"id": chunk.chunk_id,
|
|
"embedding": vector,
|
|
}
|
|
for field in COLLECTION_FIELDS[collection]:
|
|
record[field] = chunk.metadata.get(field, "")
|
|
# 类型修正：chunk_no 必须是 int
|
|
record["chunk_no"] = int(chunk.metadata.get("chunk_no", 0))
|
|
return record
|
|
|
|
|
|
def main(rebuild: bool = False) -> None:
|
|
embedder = get_embedder()
|
|
milvus = get_milvus_client()
|
|
kb_root = Path(settings.kb_root_dir)
|
|
|
|
if not kb_root.exists():
|
|
print(f"[ERROR] 知识库目录不存在: {kb_root}")
|
|
sys.exit(1)
|
|
|
|
total_inserted = 0
|
|
|
|
for coll_name, sub_dir in KB_LAYOUT.items():
|
|
dir_path = kb_root / sub_dir
|
|
if not dir_path.exists():
|
|
print(f"[WARN] 跳过 {coll_name}: 目录不存在 {dir_path}")
|
|
continue
|
|
|
|
# 1. 解析文档
|
|
print(f"\n{'=' * 60}")
|
|
print(f"[{coll_name}] 开始解析: {dir_path}")
|
|
chunks, _ = parse_collection_dir(dir_path)
|
|
print(f"[{coll_name}] 解析完成: {len(chunks)} 块")
|
|
|
|
if not chunks:
|
|
print(f"[WARN] {coll_name} 无有效内容，跳过")
|
|
continue
|
|
|
|
# 2. 重建或确保 collection
|
|
if rebuild:
|
|
milvus.drop_collection(coll_name)
|
|
milvus.ensure_collection(coll_name)
|
|
|
|
# 3. 向量化
|
|
print(f"[{coll_name}] 开始向量化 ({len(chunks)} 条)...")
|
|
texts = [c.text for c in chunks]
|
|
vectors = embedder.embed_batch(texts)
|
|
|
|
# 4. 组装记录并插入
|
|
records = []
|
|
skipped = 0
|
|
for chunk, vec in zip(chunks, vectors):
|
|
if not vec:
|
|
skipped += 1
|
|
continue
|
|
records.append(chunk_to_record(chunk, coll_name, vec))
|
|
|
|
if skipped:
|
|
print(f"[{coll_name}] 跳过 {skipped} 条（向量化失败）")
|
|
|
|
insert_count = milvus.insert(coll_name, records)
|
|
print(f"[{coll_name}] 插入 {insert_count} 条记录")
|
|
total_inserted += insert_count
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print(f"入库完成：共插入 {total_inserted} 条记录到 {len(KB_LAYOUT)} 个 collection")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description="知识库入库")
|
|
parser.add_argument("--rebuild", action="store_true", help="先删除再重建 collection")
|
|
args = parser.parse_args()
|
|
main(rebuild=args.rebuild)
|