chore: update gitignore; feat: 新增customer_agent业务模块与api路由

rag/ingestion.py

@@ -0,0 +1,82 @@
+"""Validated, all-or-nothing ingestion of a document into Milvus."""
+from __future__ import annotations
+
+import logging
+from uuid import NAMESPACE_URL, uuid5
+
+from rag.chunk_config import ChunkConfig, resolve_chunk_config
+from rag.chunking import chunk_document
+from rag.cleaning import clean_document_text
+from rag.embedding import EMBEDDING_DIMENSION, embed_texts
+from rag.milvus_delete import _escape_filter_value
+
+
+logger = logging.getLogger("rag.ingestion")
+
+
+async def ingest_document_atomic(
+    document_text: str,
+    doc_id: str,
+    title: str,
+    collection_name: str,
+    strategy: str,
+    *,
+    milvus_client,
+    embedder=embed_texts,
+    config: ChunkConfig | None = None,
+) -> dict:
+    """Validate, embed, and insert one document without leaving partial rows."""
+    if not doc_id or not doc_id.strip():
+        raise ValueError("doc_id must not be empty")
+    if not title or not title.strip():
+        raise ValueError("title must not be empty")
+    if not collection_name or not collection_name.strip():
+        raise ValueError("collection_name must not be empty")
+
+    resolved_config = config or resolve_chunk_config()
+    cleaned = clean_document_text(document_text)
+    chunking = chunk_document(cleaned.text, strategy, config=resolved_config)
+    if not chunking.chunks:
+        raise ValueError("document produced no chunks")
+
+    try:
+        vectors = await embedder([chunk.text for chunk in chunking.chunks])
+        if len(vectors) != len(chunking.chunks) or any(
+            len(vector) != EMBEDDING_DIMENSION for vector in vectors
+        ):
+            raise ValueError(f"Embedding dimension must be {EMBEDDING_DIMENSION}")
+        rows = [
+            {
+                "chunk_id": str(uuid5(NAMESPACE_URL, f"{doc_id}:{index}")),
+                "doc_id": doc_id,
+                "title": title,
+                "section_title": chunk.section_title or "",
+                "text": chunk.text,
+                "strategy": strategy,
+                "vector": vector,
+            }
+            for index, (chunk, vector) in enumerate(zip(chunking.chunks, vectors))
+        ]
+        await milvus_client.insert(collection_name=collection_name, data=rows)
+    except Exception:
+        logger.exception("atomic Milvus ingestion failed for doc_id=%s", doc_id)
+        try:
+            await milvus_client.delete(
+                collection_name=collection_name,
+                filter=f'doc_id == "{_escape_filter_value(doc_id)}"',
+            )
+        except Exception:
+            logger.exception("failed to clean up document rows for doc_id=%s", doc_id)
+        raise
+
+    return {
+        "doc_id": doc_id,
+        "collection_name": collection_name,
+        "strategy": strategy,
+        "actual_strategy": chunking.actual_strategy,
+        "degraded": chunking.degraded,
+        "warning": chunking.warning,
+        "cleaning_changed": cleaned.changed,
+        "cleaning_warnings": cleaned.warnings,
+        "chunk_count": len(rows),
+    }