class MarkdownLLMSplitter()
Splits Markdown files into sections based on GPT token limits. Only supports OpenAI models via the tiktoken library.
Attributes: gptoker: GPT tokenizer instance. gptok_limit: Maximum tokens per section. md_meta: Metadata from the Markdown file. md_str: Source Markdown content. md_doc: Parsed Markdown document. md_dict: Structured representation of the document. md_path: Path to the source file. md_sections: List of sections with content and token counts.
Args: gptok_model: Tokenizer model, defaults to “gpt-3.5-turbo”. gptok_limit: Token limit per section, defaults to model’s maximum.
def load_md_path(md_path: Union[Path, str]) -> None
Load Markdown from file path.
Args: md_path: Path to Markdown file.
def load_md_file(md_file: TextIOWrapper) -> None
Load Markdown from file object.
Args: md_file: File-like object with Markdown content.
def load_md_str(md_str: str) -> None
Load Markdown from string.
Args: md_str: Markdown content string.
def load_md(md: Union[str, Path, TextIOWrapper]) -> None
Load Markdown from string, path, or file object.
Args: md: Markdown source (string, path, or file object).
@lru_cache(maxsize=None)
def gpttok_size(text: str) -> int
Count GPT tokens in text.
Args: text: Input text.
Returns: Number of tokens.
def build_md_dict() -> None
Parse Markdown into structured dictionary.
def calculate_sizes(md_dict: Dict) -> int
Calculate total tokens in Markdown structure.
Args: md_dict: Structured Markdown dictionary.
Returns: Total token count.
def process_item(
item: Dict, current_section: List[str], current_size: int,
md_sections: List[Dict[str, Union[str,
int]]]) -> Tuple[List[str], int]
Add item to section within token limits.
Args: item: Markdown element. current_section: Current section content. current_size: Current section token count. md_sections: List of completed sections.
Returns: Updated section and token count.
def prep_section(section_text: str,
size: int = None) -> Dict[str, Union[str, int]]
Clean section text and calculate token size.
Args: section_text: Markdown content. size: Token count (optional).
Returns: Section dictionary with content and size.
def process_md(
item: Dict, current_section: List[str], current_size: int,
md_sections: List[Dict[str, Union[str,
int]]]) -> Tuple[List[str], int]
Process Markdown item into sections.
Args: item: Markdown element. current_section: Current section content. current_size: Current section token count. md_sections: List of completed sections.
Returns: Updated section and token count.
def get_sections_from_md_dict_by_limit(
md_dict: Dict) -> List[Dict[str, Union[str, int]]]
Split Markdown structure into token-limited sections.
Args: md_dict: Structured Markdown content.
Returns: List of section dictionaries.
def build() -> None
Generate sections from loaded Markdown.
def list_section_dicts() -> List[Dict[str, Union[str, int]]]
Return list of section dictionaries.
Returns: List with ‘md’ and ‘size’ keys.
def gen_section_dicts() -> Generator[Dict[str, Union[str, int]], None, None]
Yield section dictionaries.
def list_section_texts() -> List[str]
Return list of section texts.
Returns: Markdown content strings.
def gen_section_texts() -> Generator[str, None, None]
Yield section texts.
def split(md: Union[str, Path, TextIOWrapper]) -> List[str]
Split Markdown into token-limited sections.
Args: md: Source content (string, path, or file object).
Returns: List of section strings.
def split(md: Union[str, Path, TextIOWrapper],
model: str = "gpt-3.5-turbo",
limit: int = None) -> List[str]
Split Markdown document into sections.
Args: md: Source content. model: Tokenizer model, defaults to “gpt-3.5-turbo”. limit: Token limit per section.
Returns: List of section strings.
def split_md_file(md_path: Union[str, Path],
model: str = "gpt-3.5-turbo",
limit: int = None,
separator: str = "=== SPLIT ===") -> str
Split Markdown file into sections.
Args: md_path: Source file path. model: Tokenizer model, defaults to “gpt-3.5-turbo”. limit: Token limit per section. separator: Section separator string.
Returns: Concatenated sections separated by delimiter.