first commit

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/rag_cut/splitters/delimiter.py

@@ -0,0 +1,63 @@
+"""Delimiter-based splitting for non-tabular documents."""
+
+from __future__ import annotations
+
+from rag_cut.models import Block, BlockType, SplitConfig
+from rag_cut.splitters.default_splitter import _split_by_size
+
+ATOMIC_TYPES = {BlockType.TABLE, BlockType.IMAGE}
+
+
+def partition_blocks_by_delimiter(blocks: list[Block], delimiter: str) -> list[list[Block]]:
+    """Split block stream on delimiter; delimiter text is discarded from chunks."""
+    if not delimiter:
+        return [blocks] if blocks else []
+
+    groups: list[list[Block]] = []
+    current: list[Block] = []
+
+    def flush() -> None:
+        nonlocal current
+        if current:
+            groups.append(current)
+            current = []
+
+    for block in blocks:
+        if block.type in ATOMIC_TYPES:
+            current.append(block)
+            continue
+
+        text = block.text or block.markdown
+        if delimiter not in text:
+            current.append(block)
+            continue
+
+        parts = text.split(delimiter)
+        for i, part in enumerate(parts):
+            part = part.strip()
+            if part:
+                piece = Block(type=block.type, text=part, level=block.level, meta=dict(block.meta))
+                current.append(piece)
+            if i < len(parts) - 1:
+                flush()
+
+    flush()
+    return groups
+
+
+def split_by_delimiter(blocks: list[Block], config: SplitConfig) -> list[list[Block]]:
+    if not config.delimiter:
+        raise ValueError("delimiter is required for delimiter split mode")
+
+    groups = partition_blocks_by_delimiter(blocks, config.delimiter)
+    if not groups:
+        return _split_by_size(blocks, config)
+
+    sized: list[list[Block]] = []
+    for group in groups:
+        rendered_len = sum(len(b.render()) + 2 for b in group)
+        if rendered_len <= config.max_chunk_size:
+            sized.append(group)
+        else:
+            sized.extend(_split_by_size(group, config))
+    return sized