diff --git a/caterva2/client.py b/caterva2/client.py index 18739322..a923caeb 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -533,78 +533,6 @@ def copy(self, dst): """ return self.client.copy(self.path, dst) - def concat(self, srcs, dst, axis): - """ - Concatenate the file with srcs along axis to a new location dst. - - Parameters - ---------- - srcs: list of Paths - Source files to be concatenated with current file - dst : Path - The destination path for the file. - axis: int - Axis along which to concatenate. - - Returns - ------- - Path - The new path of the concatenated file. - - Examples - -------- - >>> import caterva2 as cat2 - >>> import numpy as np - >>> # For concatenating a file you need to be a registered user - >>> client = cat2.Client("https://cat2.cloud/demo", ("joedoe@example.com", "foobar")) - >>> root = client.get('@personal') - >>> root.upload('root-example/dir2/ds-4d.b2nd', "a.b2nd") - - >>> root.upload('root-example/dir2/ds-4d.b2nd', "b.b2nd") - - >>> file = root['a.b2nd'] - >>> file.concat('@personal/b.b2nd', '@personal/c.b2nd', axis=0) - PurePosixPath('@personal/c.b2nd') - """ - srcs = [srcs] if not isinstance(srcs, list) else srcs # assure that srcs is list - return self.client.concat([self.path] + srcs, dst, axis) - - def stack(self, srcs, dst, axis): - """ - Stack the file with srcs along new axis to a new location dst. - - Parameters - ---------- - srcs: list of Paths - Source files to be stacked with current file - dst : Path - The destination path for the file. - axis: int - Axis along which to stack. - - Returns - ------- - Path - The new path of the stacked file. - - Examples - -------- - >>> import caterva2 as cat2 - >>> import numpy as np - >>> # For stacking a file you need to be a registered user - >>> client = cat2.Client("https://cat2.cloud/demo", ("joedoe@example.com", "foobar")) - >>> root = client.get('@personal') - >>> root.upload('root-example/dir2/ds-4d.b2nd', "a.b2nd") - - >>> root.upload('root-example/dir2/ds-4d.b2nd', "b.b2nd") - - >>> file = root['a.b2nd'] - >>> file.stack('@personal/b.b2nd', '@personal/c.b2nd', axis=0) - PurePosixPath('@personal/c.b2nd') - """ - srcs = [srcs] if not isinstance(srcs, list) else srcs # assure that srcs is list - return self.client.stack([self.path] + srcs, dst, axis) - def remove(self): """ Removes the file from the remote repository. @@ -1314,88 +1242,6 @@ def copy(self, src, dst): ) return pathlib.PurePosixPath(result) - def concat(self, srcs, dst, axis): - """ - Concatenate the srcs along axis to a new location dst. - - Parameters - ---------- - srcs: list of Paths - Source files to be concatenated - dst : Path - The destination path for the file. - axis: int - Axis along which to concatenate. - - Returns - ------- - Path - The new path of the concatenated file. - - Examples - -------- - >>> import caterva2 as cat2 - >>> import numpy as np - >>> # For concatenating a file you need to be a registered user - >>> client = cat2.Client("https://cat2.cloud/demo", ("joedoe@example.com", "foobar")) - >>> root = client.get('@personal') - >>> root.upload('root-example/dir2/ds-4d.b2nd', "a.b2nd") - - >>> root.upload('root-example/dir2/ds-4d.b2nd', "b.b2nd") - - >>> client.concat(['@personal/a.b2nd', '@personal/b.b2nd'], '@personal/c.b2nd', axis=0) - PurePosixPath('@personal/c.b2nd') - """ - urlbase, _ = _format_paths(self.urlbase) - result = api_utils.post( - f"{self.urlbase}/api/concat/", - {"srcs": [str(src) for src in srcs], "dst": str(dst), "axis": int(axis)}, - auth_cookie=self.cookie, - timeout=self.timeout, - ) - return pathlib.PurePosixPath(result) - - def stack(self, srcs, dst, axis): - """ - Stack the files in srcs along new axis to a new location dst. - - Parameters - ---------- - srcs: list of Paths - Source files accessible by client to be stacked - dst : Path - The destination path for the file. - axis: int - Axis along which to stack. - - Returns - ------- - Path - The new path of the stacked file. - - Examples - -------- - >>> import caterva2 as cat2 - >>> import numpy as np - >>> # For stacking a file you need to be a registered user - >>> client = cat2.Client("https://cat2.cloud/demo", ("joedoe@example.com", "foobar")) - >>> root = client.get('@personal') - >>> root.upload('root-example/dir2/ds-4d.b2nd', "a.b2nd") - - >>> root.upload('root-example/dir2/ds-4d.b2nd', "b.b2nd") - - >>> client.stack(['@personal/a.b2nd', '@personal/b.b2nd'], '@personal/c.b2nd', axis=0) - PurePosixPath('@personal/c.b2nd') - """ - urlbase, _ = _format_paths(self.urlbase) - result = api_utils.post( - f"{self.urlbase}/api/stack/", - {"srcs": [str(src) for src in srcs], "dst": str(dst), "axis": int(axis)}, - auth_cookie=self.cookie, - timeout=self.timeout, - ) - return pathlib.PurePosixPath(result) - def lazyexpr(self, name, expression, operands=None, compute=False): """ Creates a lazy expression dataset in personal space. diff --git a/caterva2/models.py b/caterva2/models.py index f25273dc..2f3677ff 100644 --- a/caterva2/models.py +++ b/caterva2/models.py @@ -86,12 +86,6 @@ class MoveCopyPayload(pydantic.BaseModel): dst: str -class ConcatStackPayload(pydantic.BaseModel): - axis: int - srcs: list[str] - dst: str - - class AddUserPayload(pydantic.BaseModel): username: str password: str | None diff --git a/caterva2/services/server.py b/caterva2/services/server.py index 93fde113..bf8c76d2 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -18,7 +18,6 @@ import mimetypes import os import pathlib -import re import shutil import string import tarfile @@ -856,80 +855,6 @@ async def copy( return str(destpath) -def concatstackhelper(payload: models.ConcatStackPayload, user: db.User = Depends(current_active_user)): - if not user: - raise srv_utils.raise_unauthorized("Stacking or concatenating files requires authentication") - - srcs, dst = payload.srcs, payload.dst - # src should start with a special root or known root - for src in srcs: - if not src.startswith(("@personal", "@shared", "@public")): - raise fastapi.HTTPException( - status_code=400, - detail="Only stacking/concatenating from @personal or @shared or @public roots is allowed", - ) - # dst should start with a special root and if not try and massage it - if not dst.startswith(("@personal", "@shared", "@public")): - raise fastapi.HTTPException( - status_code=400, - detail="Only stacking/concatenating to @personal or @shared or @public roots is allowed", - ) - - destpath = pathlib.Path(dst) - dest_abspath = get_abspath(destpath, user, may_not_exist=True) - - abspaths = [get_abspath(pathlib.Path(src), user) for src in srcs] - - # dst should be a .b2nd array and if not try and massage it - if not dest_abspath.suffix: - dest_abspath = dest_abspath.with_suffix(".b2nd") - destpath = destpath.with_suffix(".b2nd") - else: - if not (dest_abspath.suffix == ".b2nd"): - raise fastapi.HTTPException( - status_code=400, detail="Stack/concat destination must be a .b2nd file" - ) - return abspaths, dest_abspath, destpath - - -@app.post("/api/concat/") -async def concat( - payload: models.ConcatStackPayload, - user: db.User = Depends(current_active_user), -): - """ - Concatenate datasets - - Returns - ------- - str - The path of the concatenated dataset. - """ - abspaths, dest_abspath, destpath = concatstackhelper(payload, user) - list_of_arrays = [blosc2.open(path) for path in abspaths] - blosc2.concat(list_of_arrays, payload.axis, urlpath=str(dest_abspath), mode="w") - return str(destpath) - - -@app.post("/api/stack/") -async def stack( - payload: models.ConcatStackPayload, - user: db.User = Depends(current_active_user), -): - """ - Stack datasets - - Returns - ------- - str - The path of the stacked dataset. - """ - abspaths, dest_abspath, destpath = concatstackhelper(payload, user) - list_of_arrays = [blosc2.open(path) for path in abspaths] - blosc2.stack(list_of_arrays, payload.axis, urlpath=str(dest_abspath), mode="w") - return str(destpath) - - def get_writable_path(path: pathlib.Path, user: db.User) -> pathlib.Path: """ Convert a path with special root to an absolute path that can be written to. @@ -1999,60 +1924,6 @@ async def call(cls, request, user, argv, operands, hx_current_url): return htmx_redirect(hx_current_url, url) -class ConcatCmd: - """Concatenate arrays.""" - - names = ("concat",) - expected = "dst = concat([, ... ], axis) or dst = concat([, ... ])" - nargs = 5 # can be more if more than 2 sources - - @classmethod - async def call(cls, request, user, argv, operands, hx_current_url): - dst = argv[1] # expect to receive [concat, dst, src1, src2, ..., srcN, axis] - list_of_arrays = [] - i = 2 - while True: - src = operands.get(argv[i], argv[i]) # get the path - i += 1 - if isinstance(src, int): - break - list_of_arrays.append(src) - axis = src - payload = models.ConcatStackPayload(srcs=list_of_arrays, dst=dst, axis=axis) - result_path = await concat(payload, user) - # Redirect to display new dataset - result_path = await display_first(result_path, user) - url = make_url(request, "html_home", path=result_path) - return htmx_redirect(hx_current_url, url) - - -class StackCmd: - """Stack arrays.""" - - names = ("stack",) - expected = "dst = stack([, ... ], axis) or dst = stack([, ... ])" - nargs = 5 # can be more if more than 2 sources - - @classmethod - async def call(cls, request, user, argv, operands, hx_current_url): - dst = argv[1] - list_of_arrays = [] - i = 2 - while True: - src = operands.get(argv[i], argv[i]) # get the path - i += 1 - if isinstance(src, int): - break - list_of_arrays.append(src) - axis = src - payload = models.ConcatStackPayload(srcs=list_of_arrays, dst=dst, axis=axis) - result_path = await stack(payload, user) - # Redirect to display new dataset - result_path = await display_first(result_path, user) - url = make_url(request, "html_home", path=result_path) - return htmx_redirect(hx_current_url, url) - - commands_list = [ AddUserCmd, DelUserCmd, @@ -2062,8 +1933,6 @@ async def call(cls, request, user, argv, operands, hx_current_url): RemoveCmd, AddNotebookCmd, UnfoldCmd, - ConcatCmd, - StackCmd, ] commands = {} @@ -2103,76 +1972,33 @@ async def htmx_command( elif nargs > 1 and argv[1] in {"=", ":="}: operator = argv[1] compute = operator == ":=" - if (argv[2][:6] != "concat") and (argv[2][:5] != "stack"): - try: - result_name, expr = command.split(operator, maxsplit=1) - if "#" in expr: # get alternative operands - expr, alt_ops = expr.split("#", maxsplit=1) - alt_ops = ast.literal_eval(alt_ops.strip()) # convert str to dict - for k, v in alt_ops.items(): - operands[k] = v # overwrite or add operands if necessary - result_path = make_expr(result_name, expr, operands, user, compute=compute) - url = make_url(request, "html_home", path=result_path) - return htmx_redirect(hx_current_url, url) - except SyntaxError: - return htmx_error(request, "Invalid syntax: expected = ") - except ValueError as exc: - return htmx_error(request, f"Invalid expression: {exc}") - except TypeError as exc: - return htmx_error(request, f"Invalid expression: {exc}") - except KeyError as exc: - error = f"Expression error: {exc.args[0]} is not in the list of available datasets" - return htmx_error(request, error) - except RuntimeError as exc: - return htmx_error(request, f"Runtime error: {exc}") - else: # used dst = concat([src1, ..., srcN], 1) - dst, expr = command.split(operator, maxsplit=1) - args = re.split(r"[()]", expr) - args = [a.strip() for a in args] - cmd = commands.get(args[0]) - err_msg = cmd.expected - if cmd not in {ConcatCmd, StackCmd}: - return htmx_error(request, "Invalid syntax: Expected concat or stack. " + err_msg) - if args[-1] != "": - return htmx_error(request, "Invalid syntax: " + err_msg) - argv = [args[0], dst.strip()] - *sources, ax = args[1].split(",") - ax_ = 0 - try: - ax_ = int(ax.split("=")[-1]) - except Exception: - # assume no axis provided, will use default 0 - sources = args[1].split(",") - - num_sources = len(sources) - if num_sources < 2: - return htmx_error(request, "Require at least two sources. " + err_msg) - for i, s in enumerate(sources): - if i == 0: - # get opening parentheses - bracket = next((i for i, item in enumerate(("[", "(", "{")) if s[0] == item), -1) - if bracket != -1: - sources[0] = s[1:] - else: - return htmx_error(request, "Unable to get iterable of sources. " + err_msg) - if i == num_sources - 1: - if s[-1] == ["]", ")", "}"][bracket]: # parentheses must match - sources[-1] = s[:-1] - else: - return htmx_error(request, "Unable to get iterable of sources. " + err_msg) - argv += sources - argv += [ax_] # argv = [concat/stack, dst, src1, src2, ..., srcN, axis] + try: + result_name, expr = command.split(operator, maxsplit=1) + if "#" in expr: # get alternative operands + expr, alt_ops = expr.split("#", maxsplit=1) + alt_ops = ast.literal_eval(alt_ops.strip()) # convert str to dict + for k, v in alt_ops.items(): + operands[k] = v # overwrite or add operands if necessary + result_path = make_expr(result_name, expr, operands, user, compute=compute) + url = make_url(request, "html_home", path=result_path) + return htmx_redirect(hx_current_url, url) + except SyntaxError: + return htmx_error(request, "Invalid syntax: expected = ") + except ValueError as exc: + return htmx_error(request, f"Invalid expression: {exc}") + except TypeError as exc: + return htmx_error(request, f"Invalid expression: {exc}") + except KeyError as exc: + error = f"Expression error: {exc.args[0]} is not in the list of available datasets" + return htmx_error(request, error) + except RuntimeError as exc: + return htmx_error(request, f"Runtime error: {exc}") # Commands cmd = commands.get(argv[0]) if cmd is not None: - if (cmd in (ConcatCmd, StackCmd)) and len(argv) < 5: - return htmx_error( - request, f"Invalid syntax: expected {cmd.expected} (at least 4 args for concat)." - ) - if cmd not in (ConcatCmd, StackCmd) and len(argv) != cmd.nargs: + if len(argv) != cmd.nargs: return htmx_error(request, f"Invalid syntax: expected {cmd.expected}") - try: return await cmd.call(request, user, argv, operands, hx_current_url) except Exception as exc: diff --git a/caterva2/tests/test_api.py b/caterva2/tests/test_api.py index ea183fd8..e3642f49 100644 --- a/caterva2/tests/test_api.py +++ b/caterva2/tests/test_api.py @@ -207,6 +207,7 @@ def test_concat(auth_client, fill_auth, examples_dir): _, mypublic = fill_auth myshared = auth_client.get("@shared") + mypersonal = auth_client.get("@personal") # Copy a 1d dataset to the shared area file = mypublic["ds-1d.b2nd"] copyname = "a.b2nd" @@ -221,26 +222,22 @@ def test_concat(auth_client, fill_auth, examples_dir): # Test for File class file = myshared[copyname] - resultname = "result.b2nd" - finalpath = file.concat([newpath2, newpath3], f"{myshared.name}/{resultname}", axis=0) - - assert myshared[resultname].shape[0] == 3 * myshared[copyname].shape[0] + resultname = "result" + finalpath = auth_client.lazyexpr( + resultname, + expression="concat([a, b, c], axis=0)", + operands={"a": newpath, "b": newpath2, "c": newpath3}, + ) + result_ds = mypersonal[resultname + ".b2nd"] + assert result_ds.shape[0] == 3 * myshared[copyname].shape[0] + # check eager evaluation + assert "expression" not in auth_client.get_info(result_ds) # Check the data fname = examples_dir / "ds-1d.b2nd" a = blosc2.open(fname) locres = np.concat([a[:], a[:], a[:]], axis=0) - sfile = myshared[resultname] - np.testing.assert_array_equal(sfile[:], locres) - - # Test for Client class - resultname = "result2.b2nd" - finalpath = auth_client.concat([newpath, newpath2, newpath3], f"{myshared.name}/{resultname}", axis=0) - - assert myshared[resultname].shape[0] == 3 * myshared[copyname].shape[0] - - # Check the data - sfile = myshared[resultname] + sfile = auth_client.get(finalpath) return np.testing.assert_array_equal(sfile[:], locres) @@ -250,6 +247,7 @@ def test_stack(auth_client, fill_auth, examples_dir): _, mypublic = fill_auth myshared = auth_client.get("@shared") + mypersonal = auth_client.get("@personal") fstr = "dir1/ds-2d.b2nd" # Copy a 1d dataset to the shared area @@ -268,27 +266,22 @@ def test_stack(auth_client, fill_auth, examples_dir): # Test for File class file = myshared[copyname] - resultname = "result.b2nd" - finalpath = file.stack([newpath2, newpath3], f"{myshared.name}/{resultname}", axis=1) - assert myshared[resultname].shape[1] == 3 - assert myshared[resultname].shape == news + resultname = "result" + finalpath = auth_client.lazyexpr( + resultname, + expression="stack([a, b, c], axis=1)", + operands={"a": newpath, "b": newpath2, "c": newpath3}, + ) + result_ds = mypersonal[resultname + ".b2nd"] + assert result_ds.shape == news + # check eager evaluation + assert "expression" not in auth_client.get_info(result_ds) # Check the data fname = examples_dir / fstr a = blosc2.open(fname) locres = np.stack([a[:], a[:], a[:]], axis=1) - sfile = myshared[resultname] - np.testing.assert_array_equal(sfile[:], locres) - - # Test for Client class - resultname = "result2.b2nd" - finalpath = auth_client.stack([newpath, newpath2, newpath3], f"{myshared.name}/{resultname}", axis=1) - - assert myshared[resultname].shape[1] == 3 - assert myshared[resultname].shape == news - - # Check the data - sfile = myshared[resultname] + sfile = auth_client.get(finalpath) return np.testing.assert_array_equal(sfile[:], locres)