diff --git a/Makefile b/Makefile index 1411d4f6..8e58b263 100644 --- a/Makefile +++ b/Makefile @@ -40,4 +40,4 @@ lite-test: # To run the server, for convenience run: - BLOSC_TRACE=1 ${BIN}/python3 -m caterva2.services.sub + ${BIN}/python3 -m caterva2.services.sub diff --git a/SPECS.md b/SPECS.md index 150e728b..a119b14e 100644 --- a/SPECS.md +++ b/SPECS.md @@ -43,20 +43,8 @@ The client must implement the following commands: There should be a configuration file (by default $CWD/caterva2.toml) where the configuration for each service is specified. For example: ``` -[broker] +[subscriber] http = "localhost:8000" -statedir = "_caterva2/bro" -loglevel = "warning" - -[publisher.1] -http = "localhost:8001" -statedir = "_caterva2/pub" -loglevel = "warning" -name = "foo" -root = "root-examples" - -[subscriber.1] -http = "localhost:8002" urlbase = "https://cat2.example.com" # e.g. served by reverse proxy statedir = "_caterva2/sub" loglevel = "warning" diff --git a/caterva2.sample.toml b/caterva2.sample.toml index 84567997..3e7ae0c2 100644 --- a/caterva2.sample.toml +++ b/caterva2.sample.toml @@ -6,8 +6,8 @@ # The subscriber section must define: # # - statedir: the directory where the subcriber's data will be stored (default: _caterva2/sub) -# - http: where the subscriber listens to (a unix socket or a host/port) (default: localhost:8002) -# - urlbase: the base url users will use to reach the subscriber (default: http://localhost:8002) +# - http: where the subscriber listens to (a unix socket or a host/port) (default: localhost:8000) +# - urlbase: the base url users will use to reach the subscriber (default: http://localhost:8000) # - quota: if defined, it will limit the disk usage (default: 0, no limit) # - maxusers: if defined, it will limit the number of users (default: 0, no limit) # - login: if true, users will need to authenticate (default: true) @@ -16,8 +16,8 @@ [subscriber] statedir = "_caterva2/sub" #http = "_caterva2/sub/uvicorn.socket" -http = "localhost:8002" -urlbase = "http://localhost:8002" +http = "localhost:8000" +urlbase = "http://localhost:8000" quota = "10G" maxusers = 5 register = true # allow users to register diff --git a/caterva2/client.py b/caterva2/client.py index ffd7c5e0..3b292d45 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -13,7 +13,7 @@ from . import api_utils, utils -sub_urlbase_default = "http://localhost:8002" +sub_urlbase_default = "http://localhost:8000" """The default base of URLs provided by the subscriber.""" diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 79c21cfe..c0ca8a62 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -169,6 +169,7 @@ def cmd_listusers(client, args): def main(): + # Build the parser conf = utils.get_conf() parser = utils.get_parser() parser.add_argument( diff --git a/caterva2/clients/tbrowser.py b/caterva2/clients/tbrowser.py index db9d5f9d..7a4ec0da 100644 --- a/caterva2/clients/tbrowser.py +++ b/caterva2/clients/tbrowser.py @@ -19,7 +19,6 @@ class TreeApp(App): - def __init__(self, args): super().__init__() self.root = args.root @@ -28,8 +27,7 @@ def __init__(self, args): user_auth = {"username": args.username, "password": args.password} auth_cookie = api_utils.get_auth_cookie(args.urlbase, user_auth) api.subscribe(args.root, args.urlbase, auth_cookie=auth_cookie) - self.data = api.get_list(args.root, args.urlbase, - auth_cookie=auth_cookie) + self.data = api.get_list(args.root, args.urlbase, auth_cookie=auth_cookie) def compose(self) -> ComposeResult: path = self.root / pathlib.Path(self.data[0]) @@ -49,18 +47,21 @@ def compose(self) -> ComposeResult: def main(): + # Load configuration (args) conf = utils.get_conf() parser = utils.get_parser() - parser.add_argument('--subscriber', - dest='urlbase', type=utils.urlbase_type, - default=conf.get('subscriber.url', - api.sub_urlbase_default)) - parser.add_argument('--username', default=conf.get('client.username')) - parser.add_argument('--password', default=conf.get('client.password')) - parser.add_argument('--root', default='foo') - - # Go + parser.add_argument( + "--subscriber", + dest="urlbase", + type=utils.urlbase_type, + default=conf.get("subscriber.url", api.sub_urlbase_default), + ) + parser.add_argument("--username", default=conf.get("client.username")) + parser.add_argument("--password", default=conf.get("client.password")) + parser.add_argument("--root", default="foo") args = utils.run_parser(parser) + + # Start client app = TreeApp(args) app.run() diff --git a/caterva2/services/plugins/tomography/__init__.py b/caterva2/services/plugins/tomography/__init__.py index 98f46ad9..d7339399 100644 --- a/caterva2/services/plugins/tomography/__init__.py +++ b/caterva2/services/plugins/tomography/__init__.py @@ -31,14 +31,11 @@ contenttype = "tomography" -abspath_and_dataprep = None urlbase = None -def init(absp_n_datap, urlbase_): - global abspath_and_dataprep +def init(urlbase_): global urlbase - abspath_and_dataprep = absp_n_datap urlbase = urlbase_ @@ -101,7 +98,6 @@ async def display( async def __get_image(path, user, ndim, i): - # Alternatively, call abspath_and_dataprep with the corresponding slice to download data async array = await get_container(path, user) index = [slice(None) for x in array.shape] index[ndim] = slice(i, i + 1, 1) diff --git a/caterva2/services/settings.py b/caterva2/services/settings.py index e1590f02..1792e77d 100644 --- a/caterva2/services/settings.py +++ b/caterva2/services/settings.py @@ -27,9 +27,9 @@ def parse_size(size): return int(number * units[unit]) -conf = utils.get_conf("subscriber", allow_id=True) +conf = utils.get_conf("subscriber") -urlbase = conf.get(".urlbase", "http://localhost:8002") +urlbase = conf.get(".urlbase", "http://localhost:8000") login = conf.get(".login", True) register = conf.get(".register", False) demo = conf.get(".demo", False) @@ -41,7 +41,6 @@ def parse_size(size): # Not strictly necessary but useful for documentation statedir = None database = None # instance -cache = None personal = None shared = None public = None diff --git a/caterva2/services/srv_utils.py b/caterva2/services/srv_utils.py index b8be6d16..449556e5 100644 --- a/caterva2/services/srv_utils.py +++ b/caterva2/services/srv_utils.py @@ -21,7 +21,6 @@ import blosc2 import fastapi import safer -import uvicorn from fastapi_users.exceptions import UserNotExists from sqlalchemy.future import select @@ -47,24 +46,6 @@ def compress_file(path): path.unlink() -def cache_lookup(cachedir, path, may_not_exist=False): - if cachedir == path: - # Special case for the cache root - return path - path = pathlib.Path(path) - if (cachedir / path).is_dir(): - # Special case for directories - return cachedir / path - - # HDF5 files cannot be compressed, as they are supported natively - if path.suffix not in {".b2frame", ".b2nd", ".h5"} and not may_not_exist: - if path.is_file(): - compress_file(path) - path = f"{path}.b2" - - return get_abspath(cachedir, path, may_not_exist) - - def get_model_from_obj(obj, model_class, **kwargs): if isinstance(obj, dict): @@ -103,7 +84,7 @@ def getter(o, k): return model_class(**data) -def read_metadata(obj, cache=None, personal=None, shared=None, public=None): +def read_metadata(obj): # Open dataset if isinstance(obj, pathlib.Path): path = obj @@ -154,10 +135,6 @@ def read_metadata(obj, cache=None, personal=None, shared=None, public=None): # overwrite operands and expression with _tosave versions for metadata display operands = operands_as_paths( obj.operands_tosave if hasattr(obj, "operands_tosave") else obj.operands, - cache, - personal, - shared, - public, ) return get_model_from_obj( obj, @@ -184,47 +161,33 @@ def reformat_cparams(cparams): return cparams -def get_relpath(path, cache=None, personal=None, shared=None, public=None): - if cache is None: - cache = settings.cache - if personal is None: - personal = settings.personal - if shared is None: - shared = settings.shared - if public is None: - public = settings.public - +def get_relpath(path): if not isinstance(path, pathlib.Path): path = pathlib.Path(path.schunk.urlpath) + # Public: /...// to (i.e. no change) + public = settings.public + if public is not None and path.is_relative_to(public): + path = path.relative_to(public) + parts = ["@public"] + list(path.parts) + return pathlib.Path(*parts) + + # Shared: /...// to (i.e. no change) + shared = settings.shared if shared is not None and path.is_relative_to(shared): - # Shared: /...// to (i.e. no change) path = path.relative_to(shared) parts = ["@shared"] + list(path.parts) return pathlib.Path(*parts) - elif public is not None and path.is_relative_to(public): - # Shared: /...// to (i.e. no change) - path = path.relative_to(public) - parts = ["@public"] + list(path.parts) - return pathlib.Path(*parts) - try: - # Cache: /...// to / - path = path.relative_to(cache) - except ValueError: - # personal: /...// to @personal/ - path = path.relative_to(personal) - parts = list(path.parts) - parts[0] = "@personal" - path = pathlib.Path(*parts) - - return path - - -def operands_as_paths(operands, cache, personal, shared, public): - return { - nm: None if op is None else str(get_relpath(op, cache, personal, shared, public)) - for (nm, op) in operands.items() - } + + # Personal: /...// to @personal/ + path = path.relative_to(settings.personal) + parts = list(path.parts) + parts[0] = "@personal" + return pathlib.Path(*parts) + + +def operands_as_paths(operands): + return {nm: None if op is None else str(get_relpath(op)) for (nm, op) in operands.items()} # @@ -246,28 +209,6 @@ def raise_not_found(detail="Not Found"): raise fastapi.HTTPException(status_code=404, detail=detail) -def get_abspath(root, path, may_not_exist=False): - abspath = root / path - - # Security check - if root not in abspath.parents: - raise_bad_request(f"Invalid path {path}") - - # Existence check - if not abspath.is_file() and not may_not_exist: - raise_not_found() - - return abspath - - -def uvicorn_run(app, args, root_path=""): - http = args.http - if http.uds: - uvicorn.run(app, uds=http.uds, root_path=root_path) - else: - uvicorn.run(app, host=http.host, port=http.port, root_path=root_path) - - # # Blosc2 related helpers # diff --git a/caterva2/services/sub.py b/caterva2/services/sub.py index d0198d2b..68b792b1 100644 --- a/caterva2/services/sub.py +++ b/caterva2/services/sub.py @@ -37,6 +37,7 @@ import nbconvert import nbformat import PIL.Image +import uvicorn # FastAPI from fastapi import Depends, FastAPI, Form, Request, UploadFile, responses @@ -324,23 +325,8 @@ async def get_list( list The list of datasets, as name strings relative to path. """ - # Get the root - root = path.parts[0] - if root == "@public": - rootdir = settings.public - elif root == "@personal": - if not user: - srv_utils.raise_not_found("@personal needs authentication") - rootdir = settings.personal / str(user.id) - elif root == "@shared": - if not user: - srv_utils.raise_not_found("@shared needs authentication") - rootdir = settings.shared - else: - raise ValueError(f"Unexpected root={root}") - # List the datasets in root or directory - directory = rootdir / pathlib.Path(*path.parts[1:]) + directory = get_writable_path(path, user) if directory.is_file(): name = pathlib.Path(directory.name) return [str(name.with_suffix("") if name.suffix == ".b2" else name)] @@ -370,10 +356,8 @@ async def get_info( dict The metadata of the dataset. """ - abspath, _ = abspath_and_dataprep(path, user=user) - return srv_utils.read_metadata( - abspath, settings.cache, settings.personal, settings.shared, settings.public - ) + abspath = get_abspath(path, user) + return srv_utils.read_metadata(abspath) async def partial_download(abspath, path, slice_=None): @@ -400,43 +384,49 @@ async def partial_download(abspath, path, slice_=None): await proxy.afetch(slice_) -def abspath_and_dataprep( - path: pathlib.Path, slice_: (tuple | None) = None, user: (db.User | None) = None, may_not_exist=False +def get_abspath( + path: pathlib.Path, user: (db.User | None), may_not_exist=False ) -> tuple[ pathlib.Path, collections.abc.Callable[[], collections.abc.Awaitable], ]: """ - Get absolute path in local storage and data preparation operation. - - After awaiting the preparation operation to complete, data in the - dataset should be ready for reading, either that covered by the slice if - given, or the whole data otherwise. + Get absolute path in local storage. """ - parts = path.parts - root = parts[0] - if root not in {"@personal", "@shared", "@public"}: - raise ValueError(f"Unexpected root={root}") - - if root in {"@personal", "@shared"} and not user: - raise fastapi.HTTPException(status_code=401) # Unauthorized + filepath = get_writable_path(path, user) + root = path.parts[0] if root == "@personal": - filepath = settings.personal / str(user.id) / pathlib.Path(*parts[1:]) - abspath = srv_utils.cache_lookup(settings.personal, filepath, may_not_exist) - + cachedir = settings.personal elif root == "@shared": - filepath = settings.shared / pathlib.Path(*parts[1:]) - abspath = srv_utils.cache_lookup(settings.shared, filepath, may_not_exist) - + cachedir = settings.shared elif root == "@public": - filepath = settings.public / pathlib.Path(*parts[1:]) - abspath = srv_utils.cache_lookup(settings.public, filepath, may_not_exist) + cachedir = settings.public - async def dataprep(): - pass + # Special case for the cache root + if cachedir == filepath: + return filepath + + # Special case for directories + elif (cachedir / filepath).is_dir(): + return cachedir / filepath + + # HDF5 files cannot be compressed, as they are supported natively + if filepath.suffix not in {".b2frame", ".b2nd", ".h5"} and not may_not_exist: + if filepath.is_file(): + srv_utils.compress_file(filepath) + filepath = f"{filepath}.b2" + + # Security check + abspath = cachedir / filepath + if cachedir not in abspath.parents: + srv_utils.raise_bad_request(f"Invalid path {filepath}") - return (abspath, dataprep) + # Existence check + if not abspath.is_file() and not may_not_exist: + srv_utils.raise_not_found() + + return abspath @app.get("/api/fetch/{path:path}") @@ -471,10 +461,7 @@ async def fetch_data( """ slice_ = api_utils.parse_slice(slice_) - # Download and update the necessary chunks of the schunk in cache - abspath, dataprep = abspath_and_dataprep(path, slice_, user=user) - # This is still needed and will only update the necessary chunks - await dataprep() + abspath = get_abspath(path, user) if filter: if field: @@ -593,15 +580,11 @@ async def get_chunk( nchunk: int, user: db.User = Depends(optional_user), ): - abspath, _ = abspath_and_dataprep(path, user=user) + abspath = get_abspath(path, user) lock = locks.setdefault(path, asyncio.Lock()) async with lock: root = path.parts[0] - if root not in {"@personal", "@shared", "@public"}: - raise ValueError(f"Unexpected root={root}") - - if root in {"@personal", "@shared"} and not user: - raise fastapi.HTTPException(status_code=401) # Unauthorized + get_rootdir_or_error(root, user) container = open_b2(abspath, path) if isinstance(container, blosc2.LazyArray): @@ -656,14 +639,7 @@ def make_expr(name: str, expr: str, operands: dict[str, str], user: db.User, com path = operands[var] # Detect special roots path = pathlib.Path(path) - if path.parts[0] == "@personal": - abspath = settings.personal / str(user.id) / pathlib.Path(*path.parts[1:]) - elif path.parts[0] == "@shared": - abspath = settings.shared / pathlib.Path(*path.parts[1:]) - elif path.parts[0] == "@public": - abspath = settings.public / pathlib.Path(*path.parts[1:]) - else: - abspath = settings.cache / path + abspath = get_writable_path(path, user) var_dict[var] = open_b2(abspath, path) # Create the lazy expression dataset @@ -745,8 +721,8 @@ async def move( ) namepath = pathlib.Path(payload.src) destpath = pathlib.Path(payload.dst) - abspath, _ = abspath_and_dataprep(namepath, user=user) - dest_abspath, _ = abspath_and_dataprep(destpath, user=user, may_not_exist=True) + abspath = get_abspath(namepath, user) + dest_abspath = get_abspath(destpath, user, may_not_exist=True) # If destination has not an extension, assume it is a directory # If user wants something without an extension, she can add a '.b2' extension :-) @@ -800,8 +776,8 @@ async def copy( ) namepath, destpath = pathlib.Path(src), pathlib.Path(dst) - abspath, _ = abspath_and_dataprep(namepath, user=user) - dest_abspath, _ = abspath_and_dataprep(destpath, user=user, may_not_exist=True) + abspath = get_abspath(namepath, user) + dest_abspath = get_abspath(destpath, user, may_not_exist=True) # If destination has not an extension, assume it is a directory # If user wants something without an extension, she should add a '.b2' extension @@ -839,15 +815,15 @@ def concatstackhelper(payload: models.ConcatStackPayload, user: db.User = Depend ) # dst should start with a special root and if not try and massage it if not dst.startswith(("@personal", "@shared", "@public")): - path = settings.personal / str(user.id) - path.mkdir(exist_ok=True, parents=True) - dest_abspath = pathlib.Path(f"{path / dst}") - destpath = pathlib.Path(f"@personal/{dst}") - else: - destpath = pathlib.Path(dst) - dest_abspath, _ = abspath_and_dataprep(destpath, user=user, may_not_exist=True) + raise fastapi.HTTPException( + status_code=400, + detail="Only stacking/concatenating to @personal or @shared or @public roots is allowed", + ) + + destpath = pathlib.Path(dst) + dest_abspath = get_abspath(destpath, user, may_not_exist=True) - abspaths = [abspath_and_dataprep(pathlib.Path(src), user=user)[0] for src in srcs] + abspaths = [get_abspath(pathlib.Path(src), user) for src in srcs] # dst should be a .b2nd array and if not try and massage it if not dest_abspath.suffix: @@ -920,16 +896,9 @@ def get_writable_path(path: pathlib.Path, user: db.User) -> pathlib.Path: fastapi.HTTPException If the path is not in a writable root """ - root = path.parts[0] - if root == "@personal": - return settings.personal / str(user.id) / pathlib.Path(*path.parts[1:]) - elif root == "@shared": - return settings.shared / pathlib.Path(*path.parts[1:]) - elif root == "@public": - return settings.public / pathlib.Path(*path.parts[1:]) - else: - detail = "Only @personal or @shared or @public roots can be modified" - raise fastapi.HTTPException(detail=detail, status_code=400) + root, *subpath = path.parts + rootdir = get_rootdir_or_error(root, user) + return rootdir / pathlib.Path(*subpath) @app.post("/api/upload/{path:path}") @@ -1436,21 +1405,38 @@ async def htmx_root_list( return templates.TemplateResponse(request, "root_list.html", context) -def _get_rootdir(user, root): - if root == "@personal": - if user: - return settings.personal / str(user.id) - elif root == "@shared": - if user: - return settings.shared - elif root == "@public": +def get_rootdir_or_error(root, user): + if root not in {"@personal", "@shared", "@public"}: + raise fastapi.HTTPException(status_code=404) # NotFound + + if root == "@public": return settings.public - else: - raise ValueError(f"Unexpected root={root}") + elif root == "@shared" and user: + return settings.shared + elif root == "@personal" and user: + return settings.personal / str(user.id) + + raise fastapi.HTTPException(status_code=401) # Unauthorized + + +def get_rootdir_or_none(root, user): + if root == "@public": + return settings.public + elif root == "@shared" and user: + return settings.shared + elif root == "@personal" and user: + return settings.personal / str(user.id) return None +def filter_roots(roots, user): + for root in roots: + rootdir = get_rootdir_or_none(root, user) + if rootdir is not None: + yield root, rootdir + + @app.get("/htmx/path-list/", response_class=HTMLResponse) async def htmx_path_list( request: Request, @@ -1489,8 +1475,7 @@ def add_dataset(path, abspath): } ) - for root in roots: - rootdir = _get_rootdir(user, root) + for root, rootdir in filter_roots(roots, user): for abspath, relpath in utils.walk_files(rootdir): if relpath.suffix == ".b2": relpath = relpath.with_suffix("") @@ -1508,14 +1493,15 @@ def add_dataset(path, abspath): break else: root = segments[1] - rootdir = _get_rootdir(user, root) - relpath = pathlib.Path(*segments[2:]) - abspath = rootdir / relpath - if abspath.suffix not in {".b2", ".b2nd", ".b2frame"}: - abspath = pathlib.Path(f"{abspath}.b2") + rootdir = get_rootdir_or_none(root, user) + if rootdir is not None: + relpath = pathlib.Path(*segments[2:]) + abspath = rootdir / relpath + if abspath.suffix not in {".b2", ".b2nd", ".b2frame"}: + abspath = pathlib.Path(f"{abspath}.b2") - with contextlib.suppress(FileNotFoundError): - add_dataset(path, abspath) + with contextlib.suppress(FileNotFoundError): + add_dataset(path, abspath) # Assign names to datasets datasets = sorted(datasets, key=lambda x: x["path"]) @@ -1567,10 +1553,8 @@ async def htmx_path_info( return response # Read metadata - abspath, _ = abspath_and_dataprep(path, user=user) - meta = srv_utils.read_metadata( - abspath, settings.cache, settings.personal, settings.shared, settings.public - ) + abspath = get_abspath(path, user) + meta = srv_utils.read_metadata(abspath) # Context tabs = [] @@ -1693,7 +1677,7 @@ async def htmx_path_view( # Depends user: db.User = Depends(optional_user), ): - abspath, _ = abspath_and_dataprep(path, user=user) + abspath = get_abspath(path, user) filter = filter.strip() if filter or sortby: try: @@ -2171,14 +2155,15 @@ async def htmx_upload( if not user: raise srv_utils.raise_unauthorized("Uploading files requires authentication") + if name not in {"@personal", "@shared", "@public"}: + raise fastapi.HTTPException(status_code=404) # NotFound + if name == "@personal": path = settings.personal / str(user.id) elif name == "@shared": path = settings.shared elif name == "@public": path = settings.public - else: - raise fastapi.HTTPException(status_code=404) # NotFound # Read the file and check quota data = await file.read() @@ -2269,20 +2254,21 @@ async def htmx_delete( user: db.User = Depends(current_active_user), ): # Find absolute path to file + root = path.parts[0] + if root not in {"@personal", "@shared", "@public"}: + return fastapi.HTTPException(status_code=400) + parts = list(path.parts) - name = parts[0] - if name == "@personal": + if root == "@personal": parts[0] = str(user.id) path = pathlib.Path(*parts) abspath = settings.personal / path - elif name == "@shared": + elif root == "@shared": path = pathlib.Path(*parts[1:]) abspath = settings.shared / path - elif name == "@public": + elif root == "@public": path = pathlib.Path(*parts[1:]) abspath = settings.public / path - else: - return fastapi.HTTPException(status_code=400) # Remove if abspath.suffix in [".h5", ".hdf5"]: @@ -2296,12 +2282,11 @@ async def htmx_delete( # Redirect to home url = make_url(request, "html_home") - return htmx_redirect(hx_current_url, url, root=name) + return htmx_redirect(hx_current_url, url, root=root) async def get_container(path, user): - abspath, dataprep = abspath_and_dataprep(path, user=user) - await dataprep() + abspath = get_abspath(path, user) return open_b2(abspath, path) @@ -2429,30 +2414,18 @@ def directory(abspath, relpath, content=None): content = [] if len(parts) == 0: - rootdir = _get_rootdir(user, "@personal") - if rootdir is not None: - rootdir.mkdir(exist_ok=True) - content.append(directory(rootdir, "@personal")) - - rootdir = _get_rootdir(user, "@shared") - if rootdir is not None: - content.append(directory(rootdir, "@shared")) + roots = {"@personal", "@shared", "@public"} + for root, rootdir in filter_roots(roots): + if root == "@personal": + rootdir.mkdir(exist_ok=True) - rootdir = _get_rootdir(user, "@public") - if rootdir is not None: - content.append(directory(rootdir, "@public")) + content.append(directory(rootdir, root)) dir_abspath = rootdir.parent dir_relpath = "" else: - # Check access to the root - root, *subpath = parts - rootdir = _get_rootdir(user, root) - if rootdir is None: - raise fastapi.HTTPException(status_code=404) # NotFound - # Get absolute and relative paths to the directory - dir_abspath = rootdir / pathlib.Path(*subpath) + dir_abspath = get_writable_path(path, user) dir_relpath = path for abspath, relpath in utils.iterdir(dir_abspath): @@ -2545,27 +2518,17 @@ def guess_dset_ctype(path: pathlib.Path, meta) -> str | None: def main(): - # Read configuration file - conf = utils.get_conf("subscriber", allow_id=True) - - # Parse command line arguments - _stdir = "_caterva2/sub" + (f".{conf.id}" if conf.id else "") + # Load configuration (args) + conf = utils.get_conf("subscriber") parser = utils.get_parser( - http=conf.get(".http", "localhost:8002"), + http=conf.get(".http", "localhost:8000"), loglevel=conf.get(".loglevel", "warning"), - statedir=conf.get(".statedir", _stdir), - id=conf.id, + statedir=conf.get(".statedir", "_caterva2/sub"), ) args = utils.run_parser(parser) - # Init cache + # Directories settings.statedir = args.statedir.resolve() - settings.cache = settings.statedir / "cache" - settings.cache.mkdir(exist_ok=True, parents=True) - # Use `download_cached()`, `StaticFiles` does not support authorization. - # app.mount("/files", StaticFiles(directory=cache), name="files") - - # Shared/Public dirs settings.shared = settings.statedir / "shared" settings.shared.mkdir(exist_ok=True, parents=True) settings.public = settings.statedir / "public" @@ -2589,7 +2552,7 @@ def main(): app.mount(f"/plugins/{tomography.name}", tomography.app) plugins[tomography.contenttype] = tomography - tomography.init(abspath_and_dataprep, settings.urlbase) + tomography.init(settings.urlbase) # Mount media media = settings.statedir / "media" @@ -2601,7 +2564,11 @@ def main(): # Run root_path = str(furl.furl(settings.urlbase).path) - srv_utils.uvicorn_run(app, args, root_path=root_path) + http = args.http + if http.uds: + uvicorn.run(app, uds=http.uds, root_path=root_path) + else: + uvicorn.run(app, host=http.host, port=http.port, root_path=root_path) if __name__ == "__main__": diff --git a/caterva2/tests/services.py b/caterva2/tests/services.py index f2ee4bf6..bd326c44 100644 --- a/caterva2/tests/services.py +++ b/caterva2/tests/services.py @@ -67,7 +67,7 @@ def get_service_ep(): return get_service_ep -get_sub_ep = service_ep_getter("localhost:8002") +get_sub_ep = service_ep_getter("localhost:8000") def make_get_http(host, path="/"): diff --git a/caterva2/tools/adduser.py b/caterva2/tools/adduser.py index 9ae5dead..887f263c 100644 --- a/caterva2/tools/adduser.py +++ b/caterva2/tools/adduser.py @@ -18,14 +18,17 @@ def main(): - conf = utils.get_conf("subscriber", allow_id=True) - _stdir = "_caterva2/sub" + (f".{conf.id}" if conf.id else "") - parser = utils.get_parser(statedir=conf.get(".statedir", _stdir), id=conf.id) + # Load configuration (args) + conf = utils.get_conf("subscriber") + parser = utils.get_parser( + statedir=conf.get(".statedir", "_caterva2/sub"), + ) parser.add_argument("username") parser.add_argument("password", nargs="?") parser.add_argument("--superuser", "-S", action="store_true", default=False) args = utils.run_parser(parser) + # Add user statedir = args.statedir.resolve() user = srv_utils.add_user(args.username, args.password, args.superuser, state_dir=statedir) print("Password:", user.password) diff --git a/caterva2/utils.py b/caterva2/utils.py index af2838c1..1534d98b 100644 --- a/caterva2/utils.py +++ b/caterva2/utils.py @@ -10,7 +10,6 @@ import argparse import datetime import logging -import os import pathlib import tomllib as toml @@ -78,9 +77,9 @@ def __init__(self, string): self.uds = string -def get_parser(loglevel="warning", statedir=None, id=None, http=None): +def get_parser(loglevel="warning", statedir=None, http=None): parser = argparse.ArgumentParser() - _add_preliminary_args(parser, id=id) # just for help purposes + _add_preliminary_args(parser) # just for help purposes if http is not None: parser.add_argument( "--http", default=http, type=Socket, help="Listen to given hostname:port or unix socket" @@ -107,12 +106,10 @@ def run_parser(parser): class Conf: - def __init__(self, conf, prefix=None, id=None): + def __init__(self, conf, prefix=None): self._conf = conf self.prefix = prefix - self.id = id - - self._pfx = (f"{prefix}.{id}" if id else prefix) if prefix else None + self._pfx = prefix if prefix else None def get(self, key, default=None): """Get configuration item with dot-separated `key`. @@ -133,20 +130,16 @@ def get(self, key, default=None): return conf -def _add_preliminary_args(parser, id=None): +def _add_preliminary_args(parser): parser.add_argument( "--conf", default="caterva2.toml", type=pathlib.Path, help=("path to alternative configuration file " "(may not exist)"), ) - if id is not None: # the empty string is a valid (default) ID - parser.add_argument( - "--id", default=id, help=("a string to distinguish services " "of the same category") - ) -def get_conf(prefix=None, allow_id=False): +def get_conf(prefix=None): """Get settings from the configuration file, if existing. If the configuration file does not exist, return an empty configuration. @@ -154,23 +147,15 @@ def get_conf(prefix=None, allow_id=False): You may get the value for a key from the returned configuration ``conf`` with ``conf.get('path.to.item'[, default])``. If a `prefix` is given and the key starts with a dot, like ``.path.to.item``, `prefix` is prepended - to it. If `allow_id` is true and command line arguments has a non-empty - value for the ``--id`` option, the value gets appended to `prefix`, - separated by a dot. - - For instance, with ``conf = get_conf('foo')`` and ``--id=bar``, - ``conf.get('.item')`` is equivalent to ``conf.get('foo.bar.item')``. + to it. """ parser = argparse.ArgumentParser(add_help=False) - _add_preliminary_args(parser, id="" if allow_id else None) + _add_preliminary_args(parser) opts = parser.parse_known_args()[0] - if allow_id and opts.id and any(p in opts.id for p in [os.curdir, os.pardir, os.sep]): - raise ValueError("invalid identifier", opts.id) - id_ = opts.id if allow_id else None try: with open(opts.conf, "rb") as conf_file: conf = toml.load(conf_file) - return Conf(conf, prefix=prefix, id=id_) + return Conf(conf, prefix=prefix) except FileNotFoundError: - return Conf({}, prefix=prefix, id=id_) + return Conf({}, prefix=prefix) diff --git a/examples/Caterva2Video.ipynb b/examples/Caterva2Video.ipynb index 28cd1808..05287366 100644 --- a/examples/Caterva2Video.ipynb +++ b/examples/Caterva2Video.ipynb @@ -10,17 +10,15 @@ }, { "cell_type": "code", + "execution_count": null, "id": "initial_id", "metadata": {}, + "outputs": [], "source": [ "#!pip install caterva2 blosc2 blosc2_grok matplotlib\n", "# Imports\n", - "import blosc2\n", - "import numpy as np\n", "import caterva2 as cat2" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -32,24 +30,24 @@ }, { "cell_type": "code", + "execution_count": null, "id": "d1da7cf5b8a87f6c", "metadata": {}, + "outputs": [], "source": [ "# First make sure a server is running via command line : CATERVA2_SECRET=c2sikrit cat2sub\n", - "client = cat2.Client(\"http://localhost:8002\", (\"user@example.com\", \"foobar11\"))\n", + "client = cat2.Client(\"http://localhost:8000\", (\"user@example.com\", \"foobar11\"))\n", "locpath, remote_path = \"localfile.b2nd\", \"@personal/localfile.b2nd\"\n", "\n", - "#DOWNLOAD\n", + "# DOWNLOAD\n", "client.download(remote_path, locpath)\n", "\n", - "#UPLOAD\n", + "# UPLOAD\n", "client.upload(locpath, remote_path)\n", "\n", - "#METADATA\n", + "# METADATA\n", "print(client.get_info(remote_path))" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -61,19 +59,19 @@ }, { "cell_type": "code", + "execution_count": null, "id": "2ecba921417f7292", "metadata": {}, + "outputs": [], "source": [ "ds = client.get(remote_path)\n", "\n", "# Download and decompress slice of dataset\n", - "temp = ds[:,5:9] #NumPy Array\n", + "temp = ds[:, 5:9] # NumPy Array\n", "\n", "# Download compressed slice of dataset\n", - "temp = ds.slice(None, slice(5, 9)) #Compressed, Blosc2 Array" - ], - "outputs": [], - "execution_count": null + "temp = ds.slice(None, slice(5, 9)) # Compressed, Blosc2 Array" + ] }, { "cell_type": "markdown", @@ -85,15 +83,15 @@ }, { "cell_type": "code", + "execution_count": null, "id": "ae8e53c4173fd414", "metadata": {}, + "outputs": [], "source": [ "arr = client.get(\"@personal/sa-1M.b2nd\")\n", "servered = arr[\"(A < - 500) & (B >= .1)\"][:]\n", "print(f\"Full array shape = {arr.shape}, filtered array shape = {servered.shape}\")" - ], - "outputs": [], - "execution_count": null + ] }, { "cell_type": "markdown", @@ -105,14 +103,14 @@ }, { "cell_type": "code", + "execution_count": null, "id": "bc6ee981d5db5c67", "metadata": {}, + "outputs": [], "source": [ "path_le = client.lazyexpr(\"pycli_expr\", \"sin(a) + 2\", {\"a\": ds.path})\n", "print(client.get(\"@personal\").file_list)" - ], - "outputs": [], - "execution_count": null + ] } ], "metadata": { diff --git a/examples/lazyexpr-chained.py b/examples/lazyexpr-chained.py index e940b21a..3ed4b3ec 100644 --- a/examples/lazyexpr-chained.py +++ b/examples/lazyexpr-chained.py @@ -19,7 +19,7 @@ import caterva2 as cat2 # Open a client to the local server -# client = cat2.Client("http://localhost:8002", ("user@example.com", "foobar11")) +# client = cat2.Client("http://localhost:8000", ("user@example.com", "foobar11")) # Open a client to the Cat2Cloud server client = cat2.Client("https://cat2.cloud/demo", ("user@example.com", "foobar11"))