diff --git a/caterva2/client.py b/caterva2/client.py index a923caeb..45835101 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -561,7 +561,7 @@ def remove(self): return self.client.remove(self.path) -class Dataset(File): +class Dataset(File, blosc2.Operand): def __init__(self, root, path): """ Represents a dataset within a Blosc2 container. @@ -1292,6 +1292,53 @@ def lazyexpr(self, name, expression, operands=None, compute=False): ) return pathlib.PurePosixPath(dataset) + def upload_lazyexpr(self, remotepath, expression, compute=False): + """ + Creates a lazy expression dataset. + + A dataset at the specified path will be created or overwritten if already + exists. + + Parameters + ---------- + remotepath : str + Path to save the lazy expression to. + expression : blosc2.LazyExpr + Expression to be evaluated. + operands : dict + Mapping of variables in the expression to their corresponding dataset paths. + compute : bool, optional + If false, generate lazyexpr and do not compute anything. + If true, compute lazy expression on creation and save (full) result. + Default false. + + Returns + ------- + Path + Path of the created dataset. + """ + urlbase, remotepath = _format_paths(self.urlbase, remotepath) + if not isinstance(expression, blosc2.LazyExpr): + raise ValueError("argument ``expression`` must be blosc2.LazyExpr instance.") + operands = expression.operands + if operands is not None: + operands = {k: str(v) for k, v in operands.items()} + else: + operands = {} + expr = { + "name": None, + "expression": expression.expression, + "operands": operands, + "compute": compute, + } + dataset = api_utils.post( + f"{self.urlbase}/api/upload_lazyexpr/{remotepath}", + expr, + auth_cookie=self.cookie, + timeout=self.timeout, + ) + return pathlib.PurePosixPath(dataset) + def adduser(self, newuser, password=None, superuser=False): """ Adds a user to the server. diff --git a/caterva2/models.py b/caterva2/models.py index 2f3677ff..62087179 100644 --- a/caterva2/models.py +++ b/caterva2/models.py @@ -74,8 +74,8 @@ class LazyArray(pydantic.BaseModel): mtime: datetime.datetime | None -class NewLazyExpr(pydantic.BaseModel): - name: str +class Cat2LazyExpr(pydantic.BaseModel): + name: str | None expression: str operands: dict[str, str] compute: bool diff --git a/caterva2/services/server.py b/caterva2/services/server.py index ebfeca0e..c0701841 100644 --- a/caterva2/services/server.py +++ b/caterva2/services/server.py @@ -654,7 +654,14 @@ async def get_chunk( return responses.StreamingResponse(downloader) -def make_expr(name: str, expr: str, operands: dict[str, str], user: db.User, compute: bool = False) -> str: +def make_expr( + name: str | None, + expr: str, + operands: dict[str, str], + user: db.User, + compute: bool = False, + remotepath: pathlib.Path | None = None, +) -> str: """ Create a lazy expression dataset in personal space. @@ -665,12 +672,14 @@ def make_expr(name: str, expr: str, operands: dict[str, str], user: db.User, com Parameters ---------- name : str - The name of the dataset to be created (without extension). + The name of the dataset to be created without extension. expr : str The expression to be evaluated. It must result in a lazy expression. operands : dictionary of strings mapping to strings The variables used in the expression and which dataset paths they refer to. + remotepath: pathlib.Path + Where to save the lazy expression. Only valid if name is None. Returns ------- @@ -682,10 +691,9 @@ def make_expr(name: str, expr: str, operands: dict[str, str], user: db.User, com raise srv_utils.raise_unauthorized("Creating lazy expressions requires authentication") # Parse expression - name = name.strip() expr = expr.strip() - if not name or not expr: - raise ValueError("Name or expression should not be empty") + if not expr or (not remotepath and not name): + raise ValueError("Name/remotepath and expression should not be empty") vars = blosc2.get_expr_operands(expr) # Open expression datasets @@ -699,14 +707,23 @@ def make_expr(name: str, expr: str, operands: dict[str, str], user: db.User, com # Create the lazy expression dataset arr = blosc2.lazyexpr(expr, var_dict) - if not isinstance(arr, blosc2.LazyExpr): - cname = type(arr).__name__ - raise TypeError(f"Evaluates to {cname} instead of lazy expression") - # Save to filesystem - path = settings.personal / str(user.id) - path.mkdir(exist_ok=True, parents=True) - urlpath = f"{path / name}.b2nd" + # Handle name or path + if name is None: # provided a path + # Get the absolute path for this user + urlpath = get_writable_path(remotepath, user) + abspath = urlpath.parent + if urlpath.suffix != ".b2nd": + raise ValueError('If path extension provided must be ".b2nd".') + path = str(remotepath) + else: # just provided a name + name = name.strip() + abspath = settings.personal / str(user.id) + urlpath = f"{abspath / name}.b2nd" + path = f"@personal/{name}.b2nd" + + abspath.mkdir(exist_ok=True, parents=True) + if any(method in expr for method in linalg_funcs): compute = True if compute: @@ -714,12 +731,46 @@ def make_expr(name: str, expr: str, operands: dict[str, str], user: db.User, com else: arr.save(urlpath=urlpath, mode="w") - return f"@personal/{name}.b2nd" + return path + + +@app.post("/api/upload_lazyexpr/{path:path}") +async def upload_lazyexpr( + path: pathlib.Path, + expr: models.Cat2LazyExpr, + user: db.User = Depends(current_active_user), +) -> str: + """ + Upload a lazy expression dataset (to any root). + + The JSON request body must contain a "name"=None for the dataset to be created, + an "expression" to be evaluated, which must result in + a lazy expression, and an "operands" object which maps variable names used + in the expression to the dataset paths that they refer to. + + Returns + ------- + str + The path of the newly created (or overwritten) dataset. + """ + if expr.name is not None: + raise ValueError("Cannot provide name and path.") + try: + result_path = make_expr(expr.name, expr.expression, expr.operands, user, expr.compute, path) + except (SyntaxError, ValueError, TypeError) as exc: + raise srv_utils.raise_bad_request(f"Invalid name or expression: {exc}") from exc + except KeyError as ke: + detail = f"Expression error: {ke.args[0]} is not in the list of available datasets" + raise srv_utils.raise_bad_request(detail) from ke + except RuntimeError as exc: + raise srv_utils.raise_bad_request(f"Runtime error: {exc}") from exc + + return result_path @app.post("/api/lazyexpr/") async def lazyexpr( - expr: models.NewLazyExpr, + expr: models.Cat2LazyExpr, user: db.User = Depends(current_active_user), ) -> str: """ diff --git a/caterva2/tests/test_api.py b/caterva2/tests/test_api.py index e3642f49..f376708d 100644 --- a/caterva2/tests/test_api.py +++ b/caterva2/tests/test_api.py @@ -680,6 +680,16 @@ def test_lazyexpr(auth_client): b = auth_client.fetch(lxpath) np.testing.assert_array_equal(a[:], b[:]) + # test streamlined API + a = auth_client.get(oppt) + ls = blosc2.lazyexpr(f"linspace(0, 1, {a.shape[0]})") + mylazyexpr = a + 0 + mylazyexpr += 2 * ls + res = a[:] + 2 * ls[:] + lxpath = auth_client.upload_lazyexpr("@shared/newexpr.b2nd", mylazyexpr) + b = auth_client.fetch(lxpath) + np.testing.assert_array_equal(res, b[:]) + # More exercises for the expression evaluation with Blosc2 arrays @pytest.mark.parametrize(