2026-09-09 20:00:06 +08:00
|
|
|
"""文档解析：Markdown + YAML front-matter → Chunk（客服 fin_* 知识库目录）。"""
|
|
|
|
|
|
|
|
|
|
from __future__ import annotations
|
|
|
|
|
|
|
|
|
|
import re
|
|
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Any
|
|
|
|
|
|
|
|
|
|
FRONT_MATTER_RE = re.compile(r"^---\s*\n(?P<body>.*?)\n---\s*\n", re.DOTALL)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class Chunk:
|
|
|
|
|
chunk_id: str
|
|
|
|
|
text: str
|
|
|
|
|
metadata: dict[str, Any] = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _parse_front_matter(text: str) -> tuple[dict[str, str], str]:
|
|
|
|
|
match = FRONT_MATTER_RE.match(text)
|
|
|
|
|
if not match:
|
|
|
|
|
return {}, text.strip()
|
|
|
|
|
meta: dict[str, str] = {}
|
|
|
|
|
for line in match.group("body").splitlines():
|
|
|
|
|
line = line.strip()
|
|
|
|
|
if not line or line.startswith("#") or ":" not in line:
|
|
|
|
|
continue
|
|
|
|
|
key, _, value = line.partition(":")
|
|
|
|
|
meta[key.strip()] = value.strip()
|
|
|
|
|
return meta, text[match.end() :].strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _slug(value: str, fallback: str) -> str:
|
|
|
|
|
cleaned = re.sub(r"[^A-Za-z0-9._-]+", "-", (value or fallback).strip())[:48]
|
|
|
|
|
return cleaned or fallback
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def _parse_markdown_file(path: Path, chunk_no: int) -> Chunk:
|
|
|
|
|
raw = path.read_text(encoding="utf-8")
|
|
|
|
|
meta, body = _parse_front_matter(raw)
|
|
|
|
|
source_doc = meta.get("source_doc") or path.stem
|
|
|
|
|
chunk_text = body or meta.get("answer") or meta.get("question") or path.stem
|
|
|
|
|
text = chunk_text
|
|
|
|
|
if meta.get("question") and meta.get("answer"):
|
|
|
|
|
text = f"{meta['question']}\n{meta['answer']}"
|
|
|
|
|
elif meta.get("question"):
|
|
|
|
|
text = f"{meta['question']}\n{chunk_text}"
|
|
|
|
|
|
|
|
|
|
record_meta = dict(meta)
|
|
|
|
|
record_meta.setdefault("source_doc", source_doc)
|
|
|
|
|
record_meta["chunk_no"] = str(chunk_no)
|
|
|
|
|
record_meta["chunk_text"] = chunk_text
|
|
|
|
|
|
|
|
|
|
chunk_id = f"{_slug(source_doc, path.stem)}_{chunk_no}"
|
|
|
|
|
return Chunk(chunk_id=chunk_id, text=text, metadata=record_meta)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def parse_collection_dir(dir_path: Path) -> tuple[list[Chunk], dict[str, Any]]:
|
|
|
|
|
"""解析目录下全部 ``*.md``，每文件 1 块（chunk_no 从 1 递增）。"""
|
|
|
|
|
if not dir_path.is_dir():
|
|
|
|
|
return [], {"file_count": 0}
|
|
|
|
|
files = sorted(dir_path.glob("*.md"))
|
|
|
|
|
chunks = [_parse_markdown_file(path, i + 1) for i, path in enumerate(files)]
|
|
|
|
|
return chunks, {"file_count": len(files)}
|
