Upload fromurl

caterva2/api_utils.py

@@ -163,6 +163,16 @@ def upload_file(localpath, remotepath, urlbase, auth_cookie=None):
     return pathlib.PurePosixPath(response.json())
 
 
+def load_from_url(path_to_url, remotepath, urlbase, auth_cookie=None):
+    client = get_client()
+    url = f"{urlbase}/api/load_from_url/{remotepath}"
+
+    headers = {"Cookie": auth_cookie} if auth_cookie else None
+    response = client.post(url, data={"remote_url": path_to_url}, headers=headers)
+    response.raise_for_status()
+    return pathlib.PurePosixPath(response.json())
+
+
 def unfold_file(remotepath, urlbase, auth_cookie=None):
     client = get_client()
     url = f"{urlbase}/api/unfold/{remotepath}"

caterva2/client.py

@@ -239,6 +239,32 @@ def upload(self, localpath, remotepath=None):
         # Remove the first component of the upload path (the root name) and return a new File/Dataset
         return self[str(uploadpath.relative_to(self.name))]
 
+    def load_from_url(self, path_to_url, remotepath=None):
+        """
+        Loads a third party file via url to this root.
+
+        Parameters
+        ----------
+        path_to_url : str
+            Url path of the file to get.
+        remotepath : Path, optional
+            Remote path where the file will be placed.  If not provided, the
+            file will be placed in the top level of this root.
+
+        Returns
+        -------
+        File
+            A instance of :class:`File` or :class:`Dataset`.
+
+        """
+        if remotepath is None:
+            remotepath = pathlib.PurePosixPath(self.name) / path_to_url
+        else:
+            remotepath = pathlib.PurePosixPath(self.name) / pathlib.PurePosixPath(remotepath)
+        uploadpath = self.client.load_from_url(path_to_url, remotepath)
+        # Remove the first component of the upload path (the root name) and return a new File/Dataset
+        return self[str(uploadpath.relative_to(self.name))]
+
 
 class File:
     def __init__(self, root, path):
@@ -1118,6 +1144,30 @@ def upload(self, localpath, dataset):
             auth_cookie=self.cookie,
         )
 
+    def load_from_url(self, path_to_url, dataset):
+        """
+        Loads a remote dataset to a remote repository.
+
+        Parameters
+        ----------
+        path_to_url : Path
+            Url to the remote third party dataset.
+        dataset : Path
+            Remote path to place the dataset into.
+
+        Returns
+        -------
+        Path
+            Path of the uploaded file on the server.
+        """
+        urlbase, _ = _format_paths(self.urlbase)
+        return api_utils.load_from_url(
+            path_to_url,
+            dataset,
+            urlbase,
+            auth_cookie=self.cookie,
+        )
+
     def append(self, remotepath, data):
         """
         Appends data to the remote location.

caterva2/services/server.py

@@ -31,6 +31,7 @@
 import dotenv
 import fastapi
 import furl
+import httpx
 import markdown
 import nbconvert
 import nbformat
@@ -967,7 +968,7 @@ async def upload_file(
     path : pathlib.Path
         The path to store the uploaded file.
     file : UploadFile
-        The file to upload.
+        The file to upload (from local source).
 
     Returns
     -------
@@ -1018,6 +1019,75 @@ async def upload_file(
     return str(path)
 
 
+@app.post("/api/load_from_url/{path:path}")
+async def load_from_url(
+    path: pathlib.Path,
+    remote_url: str = fastapi.Form(...),
+    user: db.User = Depends(current_active_user),
+):
+    """
+    Load a file from a url to a root.
+
+    Parameters
+    ----------
+    path : pathlib.Path
+        The path to store the file.
+    remote_url : str
+        The url from which to get the file (from remote source).
+
+    Returns
+    -------
+    str
+        The path of the uploaded file.
+    """
+    if not user:
+        raise srv_utils.raise_unauthorized("Uploading requires authentication")
+
+    # Get the absolute path for this user
+    abspath = get_writable_path(path, user)
+    # We may upload a new file, or replace an existing file
+    if abspath.is_dir():
+        abspath /= remote_url.filename
+        path /= remote_url.filename
+
+    # Check quota
+    # TODO To be fair we should check quota later (after compression, zip unpacking etc.)
+    async with httpx.AsyncClient(follow_redirects=True, timeout=None) as client:
+        response = await client.get(remote_url)
+        response.raise_for_status()
+    data = response.content
+
+    if abspath.suffix not in {".b2", ".b2frame", ".b2nd"}:
+        schunk = blosc2.SChunk(data=data)
+        newsize = schunk.nbytes
+    else:
+        newsize = len(data)
+
+    if settings.quota:
+        try:
+            oldsize = abspath.stat().st_size
+        except FileNotFoundError:
+            oldsize = 0
+
+        total_size = get_disk_usage() - oldsize + newsize
+        if total_size > settings.quota:
+            detail = "Upload failed because quota limit has been exceeded."
+            raise fastapi.HTTPException(detail=detail, status_code=400)
+
+    # If regular file, compress it
+    abspath.parent.mkdir(exist_ok=True, parents=True)
+    if abspath.suffix not in {".b2", ".b2frame", ".b2nd", ".h5", ".hdf5"}:
+        data = schunk.to_cframe()
+        abspath = abspath.with_suffix(abspath.suffix + ".b2")
+
+    # Write the file
+    with open(abspath, "wb") as f:
+        f.write(data)
+
+    # Return the urlpath
+    return str(path)
+
+
 @app.post("/api/append/{path:path}")
 async def append_file(
     path: pathlib.Path,

caterva2/tests/test_api.py

@@ -629,6 +629,40 @@ def test_upload(fnames, remove, root, examples_dir, tmp_path, auth_client):
             assert "Not Found" in str(e_info.value)
 
 
+def test_loadfromurl(examples_dir, tmp_path, auth_client):
+    if not auth_client:
+        pytest.skip("authentication support needed")
+
+    root = "@public"
+    path, remotepath = (
+        "https://github.com/ironArray/data-cat2-demo/raw/refs/heads/main/root-example/ds-1d.b2nd",
+        "myfile.b2nd",
+    )
+
+    remote_root = auth_client.get(root)
+    myroot = auth_client.get(TEST_CATERVA2_ROOT)
+    arr_ = myroot["ds-1d.b2nd"]
+    with contextlib.chdir(tmp_path):
+        # Now, download the file to the remote root
+        remote_ds = remote_root.load_from_url(path, remotepath)
+        # Check whether the file has been downloaded with the correct name
+        if remotepath:
+            if remotepath.endswith("/"):
+                assert remote_ds.name == remotepath + path.name
+            else:
+                assert remote_ds.name == remotepath
+        else:
+            assert remote_ds.name == str(path)
+        np.testing.assert_array_equal(remote_ds[:], arr_[:])
+        # Check removing the file
+        remote_removed = pathlib.Path(remote_ds.remove())
+        assert remote_removed == remote_ds.path
+        # Check that the file has been removed
+        with pytest.raises(Exception) as e_info:
+            _ = remote_root[remote_removed]
+        assert "Not Found" in str(e_info.value)
+
+
 def test_upload_public_unauthorized(client, auth_client, examples_dir, tmp_path):
     if auth_client:
         pytest.skip("not authentication needed")