diff --git a/caterva2/api_utils.py b/caterva2/api_utils.py index d25a13b7..b049b0db 100644 --- a/caterva2/api_utils.py +++ b/caterva2/api_utils.py @@ -163,6 +163,16 @@ def upload_file(localpath, remotepath, urlbase, auth_cookie=None): return pathlib.PurePosixPath(response.json()) +def load_from_url(path_to_url, remotepath, urlbase, auth_cookie=None): + client = get_client() + url = f"{urlbase}/api/load_from_url/{remotepath}" + + headers = {"Cookie": auth_cookie} if auth_cookie else None + response = client.post(url, data={"remote_url": path_to_url}, headers=headers) + response.raise_for_status() + return pathlib.PurePosixPath(response.json()) + + def unfold_file(remotepath, urlbase, auth_cookie=None): client = get_client() url = f"{urlbase}/api/unfold/{remotepath}" diff --git a/caterva2/client.py b/caterva2/client.py index 18739322..deefd08c 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -239,6 +239,32 @@ def upload(self, localpath, remotepath=None): # Remove the first component of the upload path (the root name) and return a new File/Dataset return self[str(uploadpath.relative_to(self.name))] + def load_from_url(self, path_to_url, remotepath=None): + """ + Loads a third party file via url to this root. + + Parameters + ---------- + path_to_url : str + Url path of the file to get. + remotepath : Path, optional + Remote path where the file will be placed. If not provided, the + file will be placed in the top level of this root. + + Returns + ------- + File + A instance of :class:`File` or :class:`Dataset`. + + """ + if remotepath is None: + remotepath = pathlib.PurePosixPath(self.name) / path_to_url + else: + remotepath = pathlib.PurePosixPath(self.name) / pathlib.PurePosixPath(remotepath) + uploadpath = self.client.load_from_url(path_to_url, remotepath) + # Remove the first component of the upload path (the root name) and return a new File/Dataset + return self[str(uploadpath.relative_to(self.name))] + class File: def __init__(self, root, path): @@ -1118,6 +1144,30 @@ def upload(self, localpath, dataset): auth_cookie=self.cookie, ) + def load_from_url(self, path_to_url, dataset): + """ + Loads a remote dataset to a remote repository. + + Parameters + ---------- + path_to_url : Path + Url to the remote third party dataset. + dataset : Path + Remote path to place the dataset into. + + Returns + ------- + Path + Path of the uploaded file on the server. + """ + urlbase, _ = _format_paths(self.urlbase) + return api_utils.load_from_url( + path_to_url, + dataset, + urlbase, + auth_cookie=self.cookie, + ) + def append(self, remotepath, data): """ Appends data to the remote location. diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 7493554a..126f4c3e 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -31,6 +31,7 @@ import dotenv import fastapi import furl +import httpx import markdown import nbconvert import nbformat @@ -967,7 +968,7 @@ async def upload_file( path : pathlib.Path The path to store the uploaded file. file : UploadFile - The file to upload. + The file to upload (from local source). Returns ------- @@ -1018,6 +1019,75 @@ async def upload_file( return str(path) +@app.post("/api/load_from_url/{path:path}") +async def load_from_url( + path: pathlib.Path, + remote_url: str = fastapi.Form(...), + user: db.User = Depends(current_active_user), +): + """ + Load a file from a url to a root. + + Parameters + ---------- + path : pathlib.Path + The path to store the file. + remote_url : str + The url from which to get the file (from remote source). + + Returns + ------- + str + The path of the uploaded file. + """ + if not user: + raise srv_utils.raise_unauthorized("Uploading requires authentication") + + # Get the absolute path for this user + abspath = get_writable_path(path, user) + # We may upload a new file, or replace an existing file + if abspath.is_dir(): + abspath /= remote_url.filename + path /= remote_url.filename + + # Check quota + # TODO To be fair we should check quota later (after compression, zip unpacking etc.) + async with httpx.AsyncClient(follow_redirects=True, timeout=None) as client: + response = await client.get(remote_url) + response.raise_for_status() + data = response.content + + if abspath.suffix not in {".b2", ".b2frame", ".b2nd"}: + schunk = blosc2.SChunk(data=data) + newsize = schunk.nbytes + else: + newsize = len(data) + + if settings.quota: + try: + oldsize = abspath.stat().st_size + except FileNotFoundError: + oldsize = 0 + + total_size = get_disk_usage() - oldsize + newsize + if total_size > settings.quota: + detail = "Upload failed because quota limit has been exceeded." + raise fastapi.HTTPException(detail=detail, status_code=400) + + # If regular file, compress it + abspath.parent.mkdir(exist_ok=True, parents=True) + if abspath.suffix not in {".b2", ".b2frame", ".b2nd", ".h5", ".hdf5"}: + data = schunk.to_cframe() + abspath = abspath.with_suffix(abspath.suffix + ".b2") + + # Write the file + with open(abspath, "wb") as f: + f.write(data) + + # Return the urlpath + return str(path) + + @app.post("/api/append/{path:path}") async def append_file( path: pathlib.Path, diff --git a/caterva2/tests/test_api.py b/caterva2/tests/test_api.py index ea183fd8..4e0146d4 100644 --- a/caterva2/tests/test_api.py +++ b/caterva2/tests/test_api.py @@ -629,6 +629,40 @@ def test_upload(fnames, remove, root, examples_dir, tmp_path, auth_client): assert "Not Found" in str(e_info.value) +def test_loadfromurl(examples_dir, tmp_path, auth_client): + if not auth_client: + pytest.skip("authentication support needed") + + root = "@public" + path, remotepath = ( + "https://github.com/ironArray/data-cat2-demo/raw/refs/heads/main/root-example/ds-1d.b2nd", + "myfile.b2nd", + ) + + remote_root = auth_client.get(root) + myroot = auth_client.get(TEST_CATERVA2_ROOT) + arr_ = myroot["ds-1d.b2nd"] + with contextlib.chdir(tmp_path): + # Now, download the file to the remote root + remote_ds = remote_root.load_from_url(path, remotepath) + # Check whether the file has been downloaded with the correct name + if remotepath: + if remotepath.endswith("/"): + assert remote_ds.name == remotepath + path.name + else: + assert remote_ds.name == remotepath + else: + assert remote_ds.name == str(path) + np.testing.assert_array_equal(remote_ds[:], arr_[:]) + # Check removing the file + remote_removed = pathlib.Path(remote_ds.remove()) + assert remote_removed == remote_ds.path + # Check that the file has been removed + with pytest.raises(Exception) as e_info: + _ = remote_root[remote_removed] + assert "Not Found" in str(e_info.value) + + def test_upload_public_unauthorized(client, auth_client, examples_dir, tmp_path): if auth_client: pytest.skip("not authentication needed")