From 10b354680fd2d383765384c58e1af182b44ea227 Mon Sep 17 00:00:00 2001 From: minisha23 Date: Wed, 7 Oct 2026 16:27:08 +0530 Subject: [PATCH] fix: handle whitespace-only documents --- src/rag/chunkers.py | 2 +- tests/test_chunkers.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/rag/chunkers.py b/src/rag/chunkers.py index d5c9d61..f1fe008 100644 --- a/src/rag/chunkers.py +++ b/src/rag/chunkers.py @@ -25,7 +25,7 @@ def chunk_document(document: Document, chunk_size: int, overlap: int) -> list[Ch if overlap >= chunk_size: raise ValueError("overlap must be smaller than chunk_size") - if not document.text: + if not document.text.strip(): return [] if len(document.text) <= chunk_size: diff --git a/tests/test_chunkers.py b/tests/test_chunkers.py index d7fefe9..49051f1 100644 --- a/tests/test_chunkers.py +++ b/tests/test_chunkers.py @@ -20,6 +20,25 @@ def test_chunk_document_empty_text() -> None: assert chunk_document(document, chunk_size=10, overlap=2) == [] +def test_chunk_document_whitespace_only_text() -> None: + document = Document( + document_id="doc-whitespace", + text=" ", + metadata={"source": "fixture", "filename": "whitespace.txt"}, + ) + + assert chunk_document(document, chunk_size=10, overlap=2) == [] + + +def test_chunk_document_newline_only_text() -> None: + document = Document( + document_id="doc-newlines", + text="\n\n", + metadata={"source": "fixture", "filename": "newlines.txt"}, + ) + + assert chunk_document(document, chunk_size=10, overlap=2) == [] + def test_chunk_document_shorter_than_chunk_size(sample_document: Document) -> None: chunks = chunk_document(sample_document, chunk_size=20, overlap=2)