From 5209f84a99895ef4625ec4d7c3c8c026912ddcb5 Mon Sep 17 00:00:00 2001 From: lshaw8317 Date: Tue, 18 Nov 2025 17:53:19 +0100 Subject: [PATCH] Add uploading from memory --- caterva2/client.py | 48 +++++++++++++++++++++++++------------- caterva2/tests/test_api.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 16 deletions(-) diff --git a/caterva2/client.py b/caterva2/client.py index 6343342b..eb5dd319 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -199,14 +199,14 @@ def __len__(self): def __str__(self): return self.name - def upload(self, localpath, remotepath=None): + def upload(self, local_dset, remotepath=None): """ Uploads a local file to this root. Parameters ---------- - localpath : Path - Path of the local file to upload. + local_dset : Path | in-memory object + Path to the local dataset or an in-memory object (convertible to blosc2.SChunk). remotepath : Path, optional Remote path where the file will be uploaded. If not provided, the file will be uploaded to the top level of this root. @@ -231,12 +231,15 @@ def upload(self, localpath, remotepath=None): """ if remotepath is None: # localpath cannot be absolute in this case (too much prone to errors) - if pathlib.PurePosixPath(localpath).is_absolute(): - raise ValueError("When `dataset` is not specified, `localpath` must be a relative path") - remotepath = pathlib.PurePosixPath(self.name) / localpath + if ( + not isinstance(local_dset, (str, pathlib.Path)) + or pathlib.PurePosixPath(local_dset).is_absolute() + ): + raise ValueError("When `remotepath` is not specified, `localpath` must be a relative path") + remotepath = pathlib.PurePosixPath(self.name) / local_dset else: remotepath = pathlib.PurePosixPath(self.name) / pathlib.PurePosixPath(remotepath) - uploadpath = self.client.upload(localpath, remotepath) + uploadpath = self.client.upload(local_dset, remotepath) # Remove the first component of the upload path (the root name) and return a new File/Dataset return self[str(uploadpath.relative_to(self.name))] @@ -1214,17 +1217,30 @@ def download(self, dataset, localpath=None): path = localpath return self._download_url(url, str(path), auth_cookie=self.cookie) - def _upload_file(self, localpath, remotepath, urlbase, auth_cookie=None): + def _upload_file(self, local_dset, remotepath, urlbase, auth_cookie=None): client = self.httpx_client url = f"{urlbase}/api/upload/{remotepath}" headers = {"Cookie": auth_cookie} if auth_cookie else None - with open(localpath, "rb") as f: + if isinstance(local_dset, (str, pathlib.Path)): + with open(local_dset, "rb") as f: + response = client.post(url, files={"file": f}, headers=headers) + response.raise_for_status() + else: + if isinstance(local_dset, blosc2.LazyExpr): + return self.upload_lazyexpr(remotepath, local_dset).path + ndarray = ( + blosc2.asarray(local_dset) + if hasattr(local_dset, "shape") + else blosc2.SChunk(data=local_dset) + ) + cframe = ndarray.to_cframe() + f = io.BytesIO(cframe) response = client.post(url, files={"file": f}, headers=headers) response.raise_for_status() return pathlib.PurePosixPath(response.json()) - def upload(self, localpath, dataset): + def upload(self, local_dset, remotepath): """ Uploads a local dataset to a remote repository. @@ -1234,9 +1250,9 @@ def upload(self, localpath, dataset): Parameters ---------- - localpath : Path - Path to the local dataset. - dataset : Path + local_dset : Path | in-memory object + Path to the local dataset or an in-memory object (convertible to blosc2.SChunk). + remotepath : Path Remote path to upload the dataset to. Returns @@ -1255,10 +1271,10 @@ def upload(self, localpath, dataset): >>> str(uploaded_path) == newpath True """ - urlbase, dataset = _format_paths(self.urlbase, dataset) + urlbase, remotepath = _format_paths(self.urlbase, remotepath) return self._upload_file( - localpath, - dataset, + local_dset, + remotepath, urlbase, auth_cookie=self.cookie, ) diff --git a/caterva2/tests/test_api.py b/caterva2/tests/test_api.py index 0fad4ab7..cf152531 100644 --- a/caterva2/tests/test_api.py +++ b/caterva2/tests/test_api.py @@ -622,6 +622,48 @@ def test_upload(fnames, remove, root, examples_dir, tmp_path, auth_client): assert "Not Found" in str(e_info.value) +@pytest.mark.parametrize( + "fnames", + [ + ( + blosc2.ones( + 10, + ), + "blosc1d.b2nd", + ), + ( + np.ones( + 10, + ), + "np1d.b2nd", + ), + (blosc2.lazyexpr("linspace(0, 8, 10)"), "dir2/ds-1d.b2nd"), + ], +) +@pytest.mark.parametrize("root", ["@personal", "@shared", "@public"]) +@pytest.mark.parametrize("remove", [False, True]) +def test_upload_frommem(fnames, remove, root, examples_dir, tmp_path, auth_client): + if not auth_client: + pytest.skip("authentication support needed") + + ds, remotepath = fnames + remote_root = auth_client.get(root) + myroot = auth_client.get(TEST_CATERVA2_ROOT) + with contextlib.chdir(tmp_path): + # Now, upload the file to the remote root + remote_ds = remote_root.upload(ds, remotepath) + # Check whether the file has been uploaded with the correct name + assert remote_ds.name == remotepath + # Check removing the file + if remove: + remote_removed = pathlib.Path(remote_ds.remove()) + assert remote_removed == remote_ds.path + # Check that the file has been removed + with pytest.raises(Exception) as e_info: + _ = remote_root[remote_removed] + assert "Not Found" in str(e_info.value) + + def test_loadfromurl(examples_dir, tmp_path, auth_client): if not auth_client: pytest.skip("authentication support needed")