"""文档解析：Markdown + YAML front-matter → Chunk（客服 fin_* 知识库目录）。""" from __future__ import annotations import re from dataclasses import dataclass, field from pathlib import Path from typing import Any FRONT_MATTER_RE = re.compile(r"^---\s*\n(?P.*?)\n---\s*\n", re.DOTALL) @dataclass class Chunk: chunk_id: str text: str metadata: dict[str, Any] = field(default_factory=dict) def _parse_front_matter(text: str) -> tuple[dict[str, str], str]: match = FRONT_MATTER_RE.match(text) if not match: return {}, text.strip() meta: dict[str, str] = {} for line in match.group("body").splitlines(): line = line.strip() if not line or line.startswith("#") or ":" not in line: continue key, _, value = line.partition(":") meta[key.strip()] = value.strip() return meta, text[match.end() :].strip() def _slug(value: str, fallback: str) -> str: cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", (value or fallback).strip())[:48] return cleaned or fallback def _parse_markdown_file(path: Path, chunk_no: int) -> Chunk: raw = path.read_text(encoding="utf-8") meta, body = _parse_front_matter(raw) source_doc = meta.get("source_doc") or path.stem chunk_text = body or meta.get("answer") or meta.get("question") or path.stem text = chunk_text if meta.get("question") and meta.get("answer"): text = f"{meta['question']}\n{meta['answer']}" elif meta.get("question"): text = f"{meta['question']}\n{chunk_text}" record_meta = dict(meta) record_meta.setdefault("source_doc", source_doc) record_meta["chunk_no"] = str(chunk_no) record_meta["chunk_text"] = chunk_text chunk_id = f"{_slug(source_doc, path.stem)}_{chunk_no}" return Chunk(chunk_id=chunk_id, text=text, metadata=record_meta) def parse_collection_dir(dir_path: Path) -> tuple[list[Chunk], dict[str, Any]]: """解析目录下全部 ``*.md``，每文件 1 块（chunk_no 从 1 递增）。""" if not dir_path.is_dir(): return [], {"file_count": 0} files = sorted(dir_path.glob("*.md")) chunks = [_parse_markdown_file(path, i + 1) for i, path in enumerate(files)] return chunks, {"file_count": len(files)}