31 lines
841 B
Python
31 lines
841 B
Python
|
"""Project-level chunking defaults and validation."""
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from dataclasses import dataclass
|
||
|
|
|
||
|
|
|
||
|
|
DEFAULT_CHUNK_SIZE = 512
|
||
|
|
DEFAULT_CHUNK_OVERLAP = 64
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True)
|
||
|
|
class ChunkConfig:
|
||
|
|
size: int
|
||
|
|
overlap: int
|
||
|
|
|
||
|
|
|
||
|
|
def resolve_chunk_config(
|
||
|
|
*,
|
||
|
|
chunk_size: int | None = None,
|
||
|
|
chunk_overlap: int | None = None,
|
||
|
|
) -> ChunkConfig:
|
||
|
|
size = DEFAULT_CHUNK_SIZE if chunk_size is None else chunk_size
|
||
|
|
overlap = DEFAULT_CHUNK_OVERLAP if chunk_overlap is None else chunk_overlap
|
||
|
|
if size <= 0:
|
||
|
|
raise ValueError("chunk_size must be greater than zero")
|
||
|
|
if overlap < 0:
|
||
|
|
raise ValueError("chunk_overlap cannot be negative")
|
||
|
|
if overlap >= size:
|
||
|
|
raise ValueError("chunk_overlap must be smaller than chunk_size")
|
||
|
|
return ChunkConfig(size=size, overlap=overlap)
|