From 8fcb0640d9cdde0cbfb0081b8b8bdb0f30e4a7d0 Mon Sep 17 00:00:00 2001
From: durgaprasadmothikari-ux <durgaprasadmothikari@gmail.com>
Date: Wed, 7 Oct 2026 16:23:33 +0530
Subject: [PATCH] fix: skip unreadable document files

---
 src/rag/loaders.py    |  5 ++++-
 tests/test_loaders.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/src/rag/loaders.py b/src/rag/loaders.py
index 9dd2468..b873be0 100644
--- a/src/rag/loaders.py
+++ b/src/rag/loaders.py
@@ -36,7 +36,10 @@ def load_documents(directory: str | Path) -> list[Document]:
         if suffix not in SUPPORTED_TEXT_EXTENSIONS:
             continue
 
-        content = file_path.read_text(encoding="utf-8")
+        try:
+            content = file_path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            continue
         relative_source = file_path.relative_to(path).as_posix()
         documents.append(
             Document(
diff --git a/tests/test_loaders.py b/tests/test_loaders.py
index 94c80c3..de7e93b 100644
--- a/tests/test_loaders.py
+++ b/tests/test_loaders.py
@@ -62,3 +62,31 @@ def test_workshop_corpus_is_a_markdown_collection() -> None:
     assert len(documents) == 6
     assert all(doc.filename.endswith(".md") for doc in documents)
     assert all(doc.metadata["source"].endswith(".md") for doc in documents)
+
+
+def test_load_documents_skips_invalid_utf8_files(tmp_path: Path) -> None:
+    doc_dir = tmp_path / "invalid_utf8"
+    doc_dir.mkdir()
+    (doc_dir / "valid.txt").write_text("valid utf-8 content\n", encoding="utf-8")
+    (doc_dir / "invalid.txt").write_bytes(b"\xff\xfe invalid utf-8 \x80")
+
+    documents = load_documents(doc_dir)
+
+    filenames = [doc.filename for doc in documents]
+    assert "invalid.txt" not in filenames
+    assert filenames == ["valid.txt"]
+    assert documents[0].text == "valid utf-8 content\n"
+
+
+def test_load_documents_continues_after_invalid_utf8_file(tmp_path: Path) -> None:
+    doc_dir = tmp_path / "continues_after_invalid"
+    doc_dir.mkdir()
+    (doc_dir / "01_first.txt").write_text("first valid document\n", encoding="utf-8")
+    (doc_dir / "02_corrupt.txt").write_bytes(b"\x80\xff corrupt bytes")
+    (doc_dir / "03_second.txt").write_text("second valid document\n", encoding="utf-8")
+
+    documents = load_documents(doc_dir)
+
+    assert [doc.filename for doc in documents] == ["01_first.txt", "03_second.txt"]
+    assert [doc.text for doc in documents] == ["first valid document\n", "second valid document\n"]
+