-
Notifications
You must be signed in to change notification settings - Fork 0
feat: added a list files in dir method #1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| # UFAClient: list_files_in_dir | ||
|
|
||
| ```python | ||
| from ufa.client import UFAClient | ||
|
|
||
| client = UFAClient( | ||
| base_url="https://api.edge.deeporigin.io/files/", | ||
| token="...", | ||
| org_key="deeporigin", | ||
| ) | ||
|
|
||
| listing = await client.list_files_in_dir(directory_path="tests/ufa") | ||
| print(listing.get("continuationToken", "")) | ||
| for entry in listing.get("data", []): | ||
| print(entry["Key"], entry["Size"]) # example fields | ||
| ``` | ||
|
|
||
| This maps to the HTTP GET endpoint: | ||
|
|
||
| ``` | ||
| GET https://api.edge.deeporigin.io/files/{org_key}/directory/{percent_encoded_prefix} | ||
| ``` | ||
|
|
||
| Where the `directory_path` is percent-encoded as a single path segment (slashes are encoded). | ||
|
|
||
| Optional parameters: | ||
|
|
||
| - `continuation_token`: Pass to continue a previous listing | ||
| - `max_keys`: Limit the number of items returned (if supported) | ||
|
|
||
| The returned JSON has fields similar to: | ||
|
|
||
| ```json | ||
| { | ||
| "data": [ | ||
| {"Key": "tests/ufa/ligand.sdf", "Size": 123, "LastModified": "..."} | ||
| ], | ||
| "continuationToken": "" | ||
| } | ||
| ``` | ||
|
|
||
|
|
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -220,6 +220,68 @@ async def upload_file( | |
| f"Failed to upload file to UFA: {local_file_path} -> {url}" | ||
| ) from last_exception | ||
|
|
||
| @beartype | ||
| async def list_files_in_dir( | ||
| self, | ||
| *, | ||
| directory_path: str, | ||
| continuation_token: str | None = None, | ||
| max_keys: int | None = None, | ||
| ) -> dict: | ||
| """ | ||
| List files within a remote directory/prefix. | ||
|
|
||
| This calls the UFA directory listing endpoint and returns the parsed JSON | ||
| response, which includes a list of objects for each file and an optional | ||
| continuation token for pagination. | ||
|
|
||
| Parameters | ||
| ---------- | ||
| directory_path : str | ||
| The remote directory or prefix to list (e.g. ``"tests/ufa"``). | ||
| continuation_token : str | None | ||
| Optional pagination token returned by a previous call. | ||
| max_keys : int | None | ||
| Optional maximum number of keys to return in this request if the | ||
| backend supports it. | ||
|
|
||
| Returns | ||
| ------- | ||
| dict | ||
| Parsed JSON payload, typically with keys ``"data"`` (list of | ||
| entries) and ``"continuationToken"`` (str). | ||
| """ | ||
|
|
||
| # The directory endpoint expects the entire prefix as a single path | ||
| # segment, so we percent-encode slashes as well (safe=""). | ||
| clean_path = directory_path.strip("/") | ||
| encoded_path = quote(clean_path, safe="") | ||
|
|
||
| base = f"{self.base_url}/{self.org_key}/directory/{encoded_path}" | ||
|
|
||
| query_parts: list[str] = [] | ||
| if continuation_token: | ||
| query_parts.append( | ||
| f"continuationToken={quote(continuation_token, safe='')}" | ||
| ) | ||
| if isinstance(max_keys, int) and max_keys > 0: | ||
| query_parts.append(f"maxKeys={max_keys}") | ||
| url = base if not query_parts else f"{base}?{'&'.join(query_parts)}" | ||
|
|
||
| async with httpx.AsyncClient() as client: | ||
| response = await self._make_request(client, url) | ||
| resp_ct = response.headers.get("content-type", "") | ||
| if "application/json" in resp_ct: | ||
| return response.json() | ||
| # Fallback: attempt to decode JSON even if content-type is missing | ||
| try: | ||
| return response.json() | ||
| except ValueError: | ||
|
||
| logger.error("Unexpected non-JSON response for directory listing") | ||
| raise UFAError( | ||
|
Comment on lines
+271
to
+281
|
||
| "Expected JSON response from directory listing endpoint" | ||
| ) from None | ||
|
|
||
|
|
||
| class UFAError(Exception): | ||
| """Base exception for UFA client errors.""" | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -114,3 +114,28 @@ def test_download_file(tmp_path: Path, local_path: Path, ufa_client: UFAClient) | |||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| assert downloaded_path.parent.resolve() == tmp_path.resolve() | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| assert downloaded_path.name == Path(remote_path).name | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| assert downloaded_path.read_bytes() == original_bytes | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| def test_list_files_in_dir(ufa_client: UFAClient) -> None: | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| """Upload fixtures and verify they appear in directory listing.""" | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| local_files = _iter_fixture_files() | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| if not local_files: | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| pytest.skip("no fixtures to upload") | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| # Ensure all fixtures are present remotely under REMOTE_PREFIX | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| for p in local_files: | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| asyncio.run( | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ufa_client.upload_file( | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| local_file_path=str(p), | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| remote_file_path=_remote_path_for(p), | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| # List the directory and assert each filename is present | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| listing = asyncio.run( | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ufa_client.list_files_in_dir(directory_path=REMOTE_PREFIX.rstrip("/")) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| ) | ||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+119
to
+137
|
||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||
| def test_list_files_in_dir(ufa_client: UFAClient) -> None: | |
| """Upload fixtures and verify they appear in directory listing.""" | |
| local_files = _iter_fixture_files() | |
| if not local_files: | |
| pytest.skip("no fixtures to upload") | |
| # Ensure all fixtures are present remotely under REMOTE_PREFIX | |
| for p in local_files: | |
| asyncio.run( | |
| ufa_client.upload_file( | |
| local_file_path=str(p), | |
| remote_file_path=_remote_path_for(p), | |
| ) | |
| ) | |
| # List the directory and assert each filename is present | |
| listing = asyncio.run( | |
| ufa_client.list_files_in_dir(directory_path=REMOTE_PREFIX.rstrip("/")) | |
| ) | |
| @pytest.mark.asyncio | |
| async def test_list_files_in_dir(ufa_client: UFAClient) -> None: | |
| """Upload fixtures and verify they appear in directory listing.""" | |
| local_files = _iter_fixture_files() | |
| if not local_files: | |
| pytest.skip("no fixtures to upload") | |
| # Ensure all fixtures are present remotely under REMOTE_PREFIX | |
| await asyncio.gather(*[ | |
| ufa_client.upload_file( | |
| local_file_path=str(p), | |
| remote_file_path=_remote_path_for(p), | |
| ) | |
| for p in local_files | |
| ]) | |
| # List the directory and assert each filename is present | |
| listing = await ufa_client.list_files_in_dir(directory_path=REMOTE_PREFIX.rstrip("/")) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
list_files_in_dirmethod creates a newhttpx.AsyncClientfor each call without specifying a timeout, unlikeupload_filewhich sets explicit timeouts. This could lead to indefinite hangs if the server is unresponsive. Consider adding a timeout configuration similar to the upload method.