39 lines
1.4 KiB
Python
39 lines
1.4 KiB
Python
"""Preview service for manually selected knowledge-base chunking."""
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from rag.chunk_config import resolve_chunk_config
|
|
from rag.chunking import SUPPORTED_STRATEGIES, chunk_document
|
|
from rag.cleaning import clean_document_text
|
|
from rag.document_parser import parse_document
|
|
|
|
|
|
def preview_document(
|
|
path: str | Path,
|
|
*,
|
|
strategy: str,
|
|
chunk_size: int | None = None,
|
|
chunk_overlap: int | None = None,
|
|
) -> dict:
|
|
"""Parse and preview a document without writing anything to Milvus."""
|
|
if strategy not in SUPPORTED_STRATEGIES:
|
|
raise ValueError(f"Unsupported chunk strategy: {strategy}")
|
|
config = resolve_chunk_config(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
|
cleaned = clean_document_text(parse_document(path))
|
|
result = chunk_document(cleaned.text, strategy, config=config)
|
|
return {
|
|
"strategy": result.requested_strategy,
|
|
"actual_strategy": result.actual_strategy,
|
|
"degraded": result.degraded,
|
|
"warning": result.warning,
|
|
"cleaning_changed": cleaned.changed,
|
|
"cleaning_warnings": cleaned.warnings,
|
|
"chunk_size": config.size,
|
|
"chunk_overlap": config.overlap,
|
|
"chunks": [
|
|
{"text": chunk.text, "section_title": chunk.section_title}
|
|
for chunk in result.chunks
|
|
],
|
|
}
|