From 4537a50f1893eb2e70c92ead04ee1643ee9b9edb Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Mon, 25 Aug 2025 14:32:29 -0400 Subject: [PATCH 1/4] 0.1.6 - patched to handle google drive URLs and URLs better --- deepdoc_client_action/CHANGELOG.md | 5 +- deepdoc_client_action/add_documents.jac | 25 ++- .../deepdoc_client_action.jac | 145 ++++++++++++++++-- deepdoc_client_action/info.yaml | 2 +- deepdoc_client_action/job_entry.jac | 4 +- 5 files changed, 147 insertions(+), 34 deletions(-) diff --git a/deepdoc_client_action/CHANGELOG.md b/deepdoc_client_action/CHANGELOG.md index 0347963..c446159 100644 --- a/deepdoc_client_action/CHANGELOG.md +++ b/deepdoc_client_action/CHANGELOG.md @@ -54,4 +54,7 @@ - Updated DeepDoc pager filter and added collection_id as fk ref to localize its listing to specific collection. Useful for multi-agent setups. # 0.1.5 -- Optimized DeepDoc get_job_id with node_get utility \ No newline at end of file +- Optimized DeepDoc get_job_id with node_get utility + +# 0.1.6 +- Patched to process URLs \ No newline at end of file diff --git a/deepdoc_client_action/add_documents.jac b/deepdoc_client_action/add_documents.jac index 48799dc..7bd5f6c 100644 --- a/deepdoc_client_action/add_documents.jac +++ b/deepdoc_client_action/add_documents.jac @@ -59,9 +59,8 @@ walker add_documents(agent_graph_walker) { } # iterate through all files and read their content + file_list = []; if self.files { - - file_list = []; loop = asyncio.new_event_loop(); asyncio.set_event_loop(loop); try { @@ -80,19 +79,19 @@ walker add_documents(agent_graph_walker) { } finally { loop.close(); } - - self.response = here.queue_job( - urls=self.urls, - files=file_list, - metadatas=self.metadatas, - from_page=self.from_page, - to_page=self.to_page, - lang=self.lang, - with_embeddings=self.with_embeddings, - callback_url=callback_url - ); } + self.response = here.queue_job( + urls=self.urls, + files=file_list, + metadatas=self.metadatas, + from_page=self.from_page, + to_page=self.to_page, + lang=self.lang, + with_embeddings=self.with_embeddings, + callback_url=callback_url + ); + if self.reporting { report self.response; } diff --git a/deepdoc_client_action/deepdoc_client_action.jac b/deepdoc_client_action/deepdoc_client_action.jac index 223172d..6f35b56 100644 --- a/deepdoc_client_action/deepdoc_client_action.jac +++ b/deepdoc_client_action/deepdoc_client_action.jac @@ -1,4 +1,5 @@ import os; +import re; import requests; import logging; import traceback; @@ -18,6 +19,7 @@ import from jac_cloud.core.archetype {BaseCollection, NodeAnchor} import from jivas.agent.modules.data.node_pager { NodePager } import from jivas.agent.modules.data.node_get { node_get } import from jivas.agent.modules.system.common { node_obj } +import from urllib.parse { urlparse, unquote, parse_qs } node DeepDocClientAction(Action) { # Integrates with DeepDoc OCR and document parsing services to ingest documents into a vector store @@ -194,7 +196,7 @@ node DeepDocClientAction(Action) { metadata = metadatas[index] if metadatas and index < len(metadatas) else {}; # ensure the output filename is without whitespaces and slashes - output_filename = f"{job_id}_{self.format_filename(name)}"; + output_filename = f"{job_id}_{self.sanitize_filename(name)}"; # save document to the file system self.save_file(output_filename, file_content); # retrieve short file url @@ -217,10 +219,12 @@ node DeepDocClientAction(Action) { for (index, url) in enumerate(urls) { metadata = metadatas[index] if metadatas and index < len(metadatas) else {}; # update metadata + filename = self.extract_filename_from_url(url); metadata["source"] = url; - metadata["filename"] = url.split("/")[-1]; + metadata["filename"] = filename; metadata["job_id"] = job_id; job_entry.add_doc_url_entry( + name = filename, url = url, metadata = metadata ); @@ -483,6 +487,9 @@ node DeepDocClientAction(Action) { self.logger.error(traceback.format_exc()); success = False; } finally { + if not doc_entry { + continue; + } doc_entry.set_status(ItemStatus.COMPLETED if doc_success else ItemStatus.FAILED); } @@ -499,6 +506,124 @@ node DeepDocClientAction(Action) { } } + def extract_filename_from_url(url: str) -> str { + #* + Extract filename from URL with proper handling of query strings, Google Drive links, etc. + + Args: + url: The URL to extract filename from + + Returns: + Extracted filename with proper extension + *# + + parsed = urlparse(url); + # Handle URLs with query parameters + path = parsed.path; + query = parsed.query; + + # Handle Google Drive URLs specifically + if 'drive.google.com' in url or 'docs.google.com' in url { + return self.extract_google_drive_filename(url); + } + + # Extract filename from path + filename = unquote(os.path.basename(path)) if path else f"download_{hash(url) % 10000}"; + + # If filename is empty or generic, try to get from query parameters + if not filename or filename in ['', 'download', 'file', 'document'] { + filename = self.extract_filename_from_query(query, url); + } + + # Ensure we have a valid filename + if not filename or '.' not in filename { + filename = self.generate_safe_filename(url); + } + + return self.sanitize_filename(filename); + } + + def extract_google_drive_filename(url: str) -> str { + # Extract filename from Google Drive URL + filename = ''; + file_id_match = re.search(r'/d/([a-zA-Z0-9_-]+)', url) or re.search(r'id=([a-zA-Z0-9_-]+)', url); + + if not file_id_match { + filename = self.generate_safe_filename(url); + } else { + filename = file_id_match.group(1); + } + + return self.sanitize_filename(filename); + } + + def extract_filename_from_query(query: str, url: str) -> str { + # Try to extract filename from query parameters + query_params = parse_qs(query); + + # Common query parameter names that might contain filenames + filename_params = ['filename', 'file', 'name', 'download', 'doc', 'document']; + + for param in filename_params { + if param in query_params { + filename = query_params[param][0]; + if filename and '.' in filename { # Likely has extension + return filename; + } + } + } + + # Check for common file extension patterns in the entire URL + extension_pattern = r'\.(pdf|docx?|xlsx?|pptx?|txt|jpg|jpeg|png|gif|bmp|zip|rar|7z)'; + extension_match = re.search(extension_pattern, url, re.IGNORECASE); + if extension_match { + ext = extension_match.group(1); + return f"document_{hash(url) % 10000}.{ext.lower()}"; + } + + return None; + } + + def generate_safe_filename(url: str) -> str { + # Generate a safe filename based on URL and response headers + parsed = urlparse(url); + + # Try to guess from URL path + if parsed.path { + # Look for common file extensions in the path + extension_pattern = r'\.([a-zA-Z0-9]{2,5})$'; + extension_match = re.search(extension_pattern, parsed.path); + if extension_match { + ext = extension_match.group(1); + return f"document_{hash(url) % 10000}.{ext.lower()}"; + } + } + + # Final fallback + return f"download_{hash(url) % 10000}.bin"; + } + + def sanitize_filename(filename: str) -> str { + # Sanitize filename to remove invalid characters + + # Remove invalid characters for filenames + invalid_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*', "'", '"']; + for char in invalid_chars { + filename = filename.replace(char, '_'); + } + + # Remove multiple underscores and trim + filename = re.sub(r'_+', '_', filename).strip('_'); + + # Ensure filename is not too long + if len(filename) > 200 { + (name, ext) = os.path.splitext(filename); + filename = name[:200-len(ext)] + ext; + } + + return filename; + } + def format_page_range(page_list:list[int]) -> str { # takes a list of page numbers for chunks and returns a string representing the range @@ -654,7 +779,7 @@ node DeepDocClientAction(Action) { if not isinstance(doc_entry, DocURLEntry) { # Attempt to delete the file from the filesystem if not url try { - self.delete_file(f"{job_id}_{self.format_filename(doc_entry.name)}"); + self.delete_file(f"{job_id}_{self.sanitize_filename(doc_entry.name)}"); } except Exception as e { self.logger.error(f"Failed to delete file from filesystem: {str(e)}"); } @@ -744,20 +869,6 @@ node DeepDocClientAction(Action) { return success; } - def format_filename(filename:str) -> str { - # Formats the filename by removing spaces and slashes. - # Returns the formatted filename. - - if not filename { - self.logger.error("Filename is empty or None."); - return filename; - } - - # Remove spaces and slashes from the filename - formatted_filename = filename.replace(" ", "_").replace("/", "_").replace("\\", "_"); - - return formatted_filename; - } } diff --git a/deepdoc_client_action/info.yaml b/deepdoc_client_action/info.yaml index dd9ba36..ce7abbf 100644 --- a/deepdoc_client_action/info.yaml +++ b/deepdoc_client_action/info.yaml @@ -2,7 +2,7 @@ package: name: jivas/deepdoc_client_action author: V75 Inc. archetype: DeepDocClientAction - version: 0.1.5 + version: 0.1.6 meta: title: DeepDoc Client Action description: Integrates with DeepDoc OCR and document parsing services to ingest documents into a vector store diff --git a/deepdoc_client_action/job_entry.jac b/deepdoc_client_action/job_entry.jac index 148ef54..3e527eb 100644 --- a/deepdoc_client_action/job_entry.jac +++ b/deepdoc_client_action/job_entry.jac @@ -41,7 +41,7 @@ node JobEntry(GraphNode) { self.job_id = job_id; } - def add_doc_url_entry(url:str, metadata:dict={}) -> DocEntry { + def add_doc_url_entry(name:str, url:str, metadata:dict={}) -> DocEntry { # adds a doc url entry to this job entry if not url { @@ -54,7 +54,7 @@ node JobEntry(GraphNode) { collection_id = self.collection_id, job_id = self.job_id, status = ItemStatus.PENDING if not self.job_id else ItemStatus.PROCESSING, - name = url.split("/")[-1], + name = name, source = url, metadata = metadata ); From f5949dd2d09d4ec1009dd60275cc424e43196734 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Mon, 25 Aug 2025 16:24:54 -0400 Subject: [PATCH 2/4] updated to add bounding box metadata, if available --- deepdoc_client_action/deepdoc_client_action.jac | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deepdoc_client_action/deepdoc_client_action.jac b/deepdoc_client_action/deepdoc_client_action.jac index 6f35b56..0bbc090 100644 --- a/deepdoc_client_action/deepdoc_client_action.jac +++ b/deepdoc_client_action/deepdoc_client_action.jac @@ -431,11 +431,12 @@ node DeepDocClientAction(Action) { # Prepare chunk metadata chunk_metadata = doc_metadata.copy(); chunk_metadata["page"] = self.format_page_range(result_page_nums); - + # Process bbox, if present + chunk_metadata["bbox"] = result.get("metadata", {}).get("bbox", []); # Add to batch texts.append(text); metadatas.append(chunk_metadata); - ids.append(result.get("id") or f"doc_{os.urandom(8).hex()}"); + ids.append(result.get("id") or f"chunk_{os.urandom(8).hex()}"); # add embeddings to batch if available if "embeddings" in result { From 29cfc388de0f6c6c04c78aa8cd29391325a90fa6 Mon Sep 17 00:00:00 2001 From: Tharick Jairam Date: Tue, 26 Aug 2025 15:06:20 -0400 Subject: [PATCH 3/4] Added import and export collections --- deepdoc_client_action/CHANGELOG.md | 3 +- .../deepdoc_client_action.jac | 135 ++++++++++++++++++ 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/deepdoc_client_action/CHANGELOG.md b/deepdoc_client_action/CHANGELOG.md index c446159..0ca5248 100644 --- a/deepdoc_client_action/CHANGELOG.md +++ b/deepdoc_client_action/CHANGELOG.md @@ -57,4 +57,5 @@ - Optimized DeepDoc get_job_id with node_get utility # 0.1.6 -- Patched to process URLs \ No newline at end of file +- Patched to process URLs +- Added import and export collections to DeepDocClientAction \ No newline at end of file diff --git a/deepdoc_client_action/deepdoc_client_action.jac b/deepdoc_client_action/deepdoc_client_action.jac index 0bbc090..4d82b28 100644 --- a/deepdoc_client_action/deepdoc_client_action.jac +++ b/deepdoc_client_action/deepdoc_client_action.jac @@ -870,6 +870,25 @@ node DeepDocClientAction(Action) { return success; } + def export_collection() -> dict { + collection = self.get_collection(); + export_collection = collection spawn _export_collection(); + return {"documents": export_collection.documents}; + } + + def import_collection(collection_info:dict, purge_collection:bool=True) -> bool { + if purge_collection { + self.get_agent().get_memory().purge_collection_memory(self.label); + } + collection = self.get_collection(); + + if collection_info { + documents = collection_info.get("documents"); + import_collection = collection spawn _import_collection(documents=documents); + return True; + } + return False; + } } @@ -891,4 +910,120 @@ walker _get_job_entry { self.job_entry = here; } +} + + +walker _export_collection { + has documents:dict = {}; + + obj __specs__ { + static has private:bool = True; + } + + can on_collection with Collection entry { + visit [-->](`?JobEntry); + } + + can on_job_entry with JobEntry entry { + visit [-->]; + } + + can on_doc_file_entry with DocFileEntry entry { + job_entry_id = [<--](`?JobEntry)[0].job_id; + if job_entry_id not in self.documents { + self.documents[job_entry_id] = [here.export()]; + } else { + self.documents[job_entry_id].append(here.export()); + } + } + + can on_doc_url_entry with DocURLEntry entry { + job_entry_id = [<--](`?JobEntry)[0].job_id; + if job_entry_id not in self.documents { + self.documents[job_entry_id] = [here.export()]; + } else { + self.documents[job_entry_id].append(here.export()); + } + } + +} + +walker _import_collection { + has documents:dict = {}; + + obj __specs__ { + static has private:bool = True; + } + + can on_collection with Collection entry { + for job_id in self.documents { + visit [-->](`?JobEntry)(?job_id == job_id) else { + job_entry = JobEntry(collection_id=here.id, job_id=job_id); + here ++> job_entry; + + for document_entry in self.documents[job_id] { + if document_entry.get("mimetype") == "url"{ + + doc_url_entry = DocURLEntry( + collection_id = here.id, + job_id = job_id, + status = ItemStatus.PENDING if not job_id else ItemStatus.PROCESSING, + name = document_entry.get("name"), + source = document_entry.get("source"), + metadata = document_entry.get("metadata") + ); + + # now we attach it to the job + job_entry ++> doc_url_entry; + + } else { + doc_file_entry = DocFileEntry( + collection_id = here.id, + job_id = job_id, + status = ItemStatus.PENDING if not job_id else ItemStatus.PROCESSING, + name = document_entry.get("name"), + source = document_entry.get("source"), + mimetype = document_entry.get("mimetype"), + metadata = document_entry.get("metadata") + ); + # now we attach it to the job + job_entry ++> doc_file_entry; + } + } + + } + } + } + + can on_job_entry with JobEntry entry { + for document_entry in self.documents[here.job_id] { + if document_entry.get("mimetype") == "url" and not [-->](`?DocURLEntry)(?name == document_entry.get("name")){ + + doc_url_entry = DocURLEntry( + collection_id = here.collection_id, + job_id = here.job_id, + status = ItemStatus.PENDING if not here.job_id else ItemStatus.PROCESSING, + name = document_entry.get("name"), + source = document_entry.get("source"), + metadata = document_entry.get("metadata") + ); + + # now we attach it to the job + here ++> doc_url_entry; + + } elif not [-->](`?DocFileEntry)(?name == document_entry.get("name")) { + doc_file_entry = DocFileEntry( + collection_id = here.collection_id, + job_id = here.job_id, + status = ItemStatus.PENDING if not here.job_id else ItemStatus.PROCESSING, + name = document_entry.get("name"), + source = document_entry.get("source"), + mimetype = document_entry.get("mimetype"), + metadata = document_entry.get("metadata") + ); + # now we attach it to the job + here ++> doc_file_entry; + } + } + } } \ No newline at end of file From 244b3888b90df08c5b194194f696f715b9bdf9f8 Mon Sep 17 00:00:00 2001 From: Tharick Jairam Date: Tue, 26 Aug 2025 15:13:29 -0400 Subject: [PATCH 4/4] fix pre commit --- deepdoc_client_action/deepdoc_client_action.jac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepdoc_client_action/deepdoc_client_action.jac b/deepdoc_client_action/deepdoc_client_action.jac index 4d82b28..2b123fb 100644 --- a/deepdoc_client_action/deepdoc_client_action.jac +++ b/deepdoc_client_action/deepdoc_client_action.jac @@ -875,7 +875,7 @@ node DeepDocClientAction(Action) { export_collection = collection spawn _export_collection(); return {"documents": export_collection.documents}; } - + def import_collection(collection_info:dict, purge_collection:bool=True) -> bool { if purge_collection { self.get_agent().get_memory().purge_collection_memory(self.label);