fix: retry Google embedding calls on 429 RESOURCE_EXHAUSTED

backend/pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
     "sphinxcontrib-mermaid==1.0.0",
     "sqlalchemy>=2.0.43",
     "starlette>=1.0.1",
+    "tenacity>=9.0.0",
     "unstructured==0.18.18",
 ]
 

backend/src/vectorstores/faiss.py

@@ -9,6 +9,7 @@
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
 from langchain_google_vertexai import VertexAIEmbeddings
 from langchain_core.documents import Document
+from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception
 
 from ..tools.process_md import process_md
 from ..tools.process_pdf import process_pdf_docs
@@ -71,6 +72,14 @@ def __init__(
     def faiss_db(self) -> Optional[FAISS]:
         return self._faiss_db
 
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_exponential(multiplier=2, min=10, max=120),
+        retry=retry_if_exception(
+            lambda e: "RESOURCE_EXHAUSTED" in str(e) or "429" in str(e)
+        ),
+        reraise=True,
+    )
     def _add_to_db(self, documents: list[Document]) -> None:
         if self._faiss_db is None:
             self._faiss_db = FAISS.from_documents(
@@ -218,6 +227,14 @@ def load_db(self, name: str) -> None:
     def get_documents(self) -> list[Document]:
         return self._faiss_db.docstore._dict.values()  # type: ignore
 
+    @retry(
+        stop=stop_after_attempt(5),
+        wait=wait_exponential(multiplier=2, min=10, max=120),
+        retry=retry_if_exception(
+            lambda e: "RESOURCE_EXHAUSTED" in str(e) or "429" in str(e)
+        ),
+        reraise=True,
+    )
     def process_json(self, folder_paths: list[str]) -> FAISS:
         logging.info("Processing json files...")
         if not isinstance(folder_paths, list):