From f9ba4700ed703df522dbde548df6a38e193cda04 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 13 Sep 2025 11:26:25 +0200 Subject: [PATCH 1/7] New CLI command tree --- caterva2/clients/cli.py | 56 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index c0ca8a62..4599875e 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -71,6 +71,55 @@ def cmd_list(client, args): print(f"{item}") +# New helpers for tree command +def build_tree(paths): + """Builds a nested dict representing directories and files from a list of paths.""" + tree = {} + for p in paths: + parts = p.strip("/").split("/") + node = tree + for part in parts[:-1]: + node = node.setdefault(part, {}) + last = parts[-1] + # files are represented by None, directories by dict + if last in node: + # If an entry exists as a dict (dir) and we are inserting a file, keep dir. + if node[last] is None: + node[last] = None + else: + node[last] = None + return tree + + +def _print_tree_node(node, prefix=""): + """Recursively prints a node (dict).""" + # Sort for deterministic output; directories and files mixed lexicographically. + items = sorted(node.items(), key=lambda kv: kv[0]) + for idx, (name, child) in enumerate(items): + is_last = idx == len(items) - 1 + connector = "└──" if is_last else "├──" + print(f"{prefix}{connector} {name}") + if isinstance(child, dict): + extension = " " if is_last else "│ " + _print_tree_node(child, prefix + extension) + + +@handle_errors +def cmd_tree(client, args): + """ + Print a hierarchical tree of datasets/files in the specified root/path. + """ + data = client.get_list(args.root) + if args.json: + print(json.dumps(data)) + return + + # Build a nested representation and print it + tree = build_tree(data) + # Print top-level entries without a leading root label (similar to unix `tree .`) + _print_tree_node(tree) + + @handle_errors def cmd_url(client, args): data = api_utils.get_download_url(args.dataset, args.urlbase) @@ -195,6 +244,13 @@ def main(): subparser.add_argument("root") subparser.set_defaults(func=cmd_list) + # tree (new) + help = "Show a tree view of datasets/files in a root (similar to unix tree)." + subparser = subparsers.add_parser("tree", aliases=["tr"], help=help) + subparser.add_argument("--json", action="store_true") + subparser.add_argument("root") + subparser.set_defaults(func=cmd_tree) + # copy help = "Copy a dataset to a different root." subparser = subparsers.add_parser("copy", aliases=["cp"], help=help) From b2460cb497f30cb5bdbc37a9cbc75a0d1f51aa7f Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Sat, 13 Sep 2025 13:08:25 +0200 Subject: [PATCH 2/7] Allow to pass slices as strings (happens in CLI mode) --- caterva2/client.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/caterva2/client.py b/caterva2/client.py index 8f60ebd6..6f2f9969 100644 --- a/caterva2/client.py +++ b/caterva2/client.py @@ -36,6 +36,18 @@ def _format_paths(urlbase, path=None): return urlbase, path +def _looks_like_slice(s: str) -> bool: + """Return True if `s` parses as an index/slice expression for `np.index_exp[...]`.""" + if not isinstance(s, str) or not s.strip(): + return False + try: + # restrict eval environment to only numpy (no builtins) + eval(f"np.index_exp[{s}]", {"np": np, "__builtins__": {}}, {}) + return True + except Exception: + return False + + class Root: def __init__(self, client, name): """ @@ -998,11 +1010,13 @@ def get_slice(self, path, key=None, as_blosc2=True, field=None): as_blosc2=as_blosc2, timeout=self.timeout, ) - if isinstance(key, str): # A filter has been passed + if isinstance(key, str): + # The key can still be a slice expression in string format (like for CLI utils) + params = {"slice_": key} if _looks_like_slice(key) else {"filter": key} return api_utils.fetch_data( path, urlbase, - {"filter": key}, + params=params, auth_cookie=self.cookie, as_blosc2=as_blosc2, timeout=self.timeout, From a9071d35e7e67fbfcdec191f4bbeccd5e190c032 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 16 Sep 2025 11:34:55 +0200 Subject: [PATCH 3/7] New handle and browse commands for CLI --- caterva2/api_utils.py | 6 ++++++ caterva2/clients/cli.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/caterva2/api_utils.py b/caterva2/api_utils.py index bfee2c6e..51b4ceeb 100644 --- a/caterva2/api_utils.py +++ b/caterva2/api_utils.py @@ -116,6 +116,12 @@ def get_download_url(path, urlbase): return f"{urlbase}/api/download/{path}" +def get_handle_url(path, urlbase): + # Get the root in path (first element in path) + root = path.split("/")[0] + return f"{urlbase}/roots/{path}?roots={root}" + + def b2_unpack(filepath): schunk = blosc2.open(filepath) outfile = filepath.with_suffix("") diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 4599875e..6160b349 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -12,6 +12,7 @@ import random import re import string +import webbrowser # Requirements import httpx @@ -120,6 +121,7 @@ def cmd_tree(client, args): _print_tree_node(tree) +# url command (returns download URL) @handle_errors def cmd_url(client, args): data = api_utils.get_download_url(args.dataset, args.urlbase) @@ -129,6 +131,29 @@ def cmd_url(client, args): print(data) +# handle command (returns handle URL meant for browser exploration) +@handle_errors +def cmd_handle(client, args): + data = api_utils.get_handle_url(args.dataset, args.urlbase) + if args.json: + print(json.dumps(data)) + return + print(data) + + +# browse command (opens local browser at the handle URL) +@handle_errors +def cmd_browse(client, args): + url = api_utils.get_handle_url(args.dataset, args.urlbase) + # Try to open in a new browser tab; still print the URL for logging + try: + webbrowser.open(url, new=2) + print(f"Opened browser at: {url}") + except Exception: + # Fallback: at least print the URL if opening fails + print(url) + + @handle_errors def cmd_info(client, args): print(f"Getting info for {args.dataset}") @@ -278,6 +303,19 @@ def main(): subparser.add_argument("dataset", type=str) subparser.set_defaults(func=cmd_url) + # handle + help = "Handle URL (resource handle) for a dataset (returns a URL for browser exploration)." + subparser = subparsers.add_parser("handle", help=help) + subparser.add_argument("--json", action="store_true") + subparser.add_argument("dataset", type=str) + subparser.set_defaults(func=cmd_handle) + + # browse + help = "Open a local web browser at the dataset handle URL." + subparser = subparsers.add_parser("browse", help=help) + subparser.add_argument("dataset", type=str) + subparser.set_defaults(func=cmd_browse) + # info help = "Get metadata about a dataset." subparser = subparsers.add_parser("info", help=help) From 8c66cb68595df3a7dbf75ee557a4800341f00ca0 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Tue, 16 Sep 2025 18:09:04 +0200 Subject: [PATCH 4/7] Fancy output for info command --- caterva2/clients/cli.py | 70 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 6160b349..c8ef0629 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -14,9 +14,10 @@ import string import webbrowser +import blosc2 + # Requirements import httpx -import rich import caterva2 as cat2 @@ -158,12 +159,73 @@ def cmd_browse(client, args): def cmd_info(client, args): print(f"Getting info for {args.dataset}") data = client.get_info(args.dataset) - - # Print if args.json: print(json.dumps(data)) return - rich.print(data) + + # Helpers + def _human_bytes(n): + if n is None: + return "N/A" + n = float(n) + for unit in ("B", "KiB", "MiB", "GiB", "TiB", "PiB"): + if n < 1024.0 or unit == "PiB": + return f"{n:.2f} {unit}" if unit != "B" else f"{int(n)} {unit}" + n /= 1024.0 + return None + + def _codec_name(cid): + try: + return blosc2.Codec(cid).name + except Exception: + return f"id({cid})" + + def _filter_names(fl): + names = [] + for f in fl or []: + if f == 0: + continue + try: + names.append(blosc2.Filter(f).name) + except Exception: + names.append(f"f{f}") + return names + + # Extract fields + schunk = data.get("schunk") or data + cparams = schunk.get("cparams") + shape = data.get("shape") + nchunks = data.get("nchunks") + chunks = data.get("chunks") + chunksize = data.get("chunksize") + blocks = data.get("blocks") + blocksize = cparams.get("blocksize") + dtype = data.get("dtype") + typesize = cparams.get("typesize") + nbytes = schunk.get("nbytes") or data.get("nbytes") + cbytes = schunk.get("cbytes") or data.get("cbytes") + codec = cparams.get("codec") + clevel = cparams.get("clevel") + filters = cparams.get("filters") + # Pretty print + # print() + # print("Dataset:") + print(f"nchunks : {nchunks}") if shape is None else print(f"shape : {shape}") + print(f"chunksize: {_human_bytes(chunksize)}") if chunks is None else print(f"chunks: {chunks}") + print(f"blocksize: {_human_bytes(blocksize)}") if blocks is None else print(f"blocks: {blocks}") + print(f"typesize : {_human_bytes(typesize)}") if dtype is None else print(f"dtype : {dtype}") + print(f"nbytes: {_human_bytes(nbytes)}") + print(f"cbytes: {_human_bytes(cbytes)}") + print(f"ratio : {nbytes / cbytes:.2f}x" if nbytes and cbytes else " ratio : N/A") + # print() + print("cparams:") + print(f" codec : {_codec_name(codec)} ({codec})") + print(f" clevel : {clevel}") + if filters is not None: + fnames = _filter_names(filters) + print(f" filters: [{', '.join(fnames)}]") + else: + print(" filters: None") @handle_errors From fe0e3dc6e76b120b6107e8b619fce34a77783001 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 17 Sep 2025 11:14:14 +0200 Subject: [PATCH 5/7] Paginate and colorize values for show cli --- caterva2/clients/cli.py | 153 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 8 deletions(-) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index c8ef0629..14320d44 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -6,18 +6,25 @@ # License: GNU Affero General Public License v3.0 # See LICENSE.txt for details about copyright and rights to use. ############################################################################### - +import contextlib import json +import os import pathlib import random import re +import shlex import string +import subprocess +import sys import webbrowser import blosc2 # Requirements import httpx +import numpy as np +from rich.console import Console +from rich.syntax import Syntax import caterva2 as cat2 @@ -234,16 +241,143 @@ def cmd_show(client, args): slice_ = params.get("slice_", None) data = client.fetch(path, slice_=slice_) - # Display + # JSON output requested -> convert numpy arrays to lists + if getattr(args, "json", False): + try: + # If numpy array, convert to list; bytes handled below + if isinstance(data, np.ndarray): + print(json.dumps(data.tolist())) + else: + print(json.dumps(data)) + except TypeError: + # Fallback for non-serializable objects + print(json.dumps(str(data))) + return + + # Display bytes as decoded text if possible, otherwise indicate binary data if isinstance(data, bytes): try: print(data.decode()) except UnicodeDecodeError: print("Binary data") - else: - print(data) - # TODO: make rich optional in command line - # rich.print(data) + return + + # If numpy array, produce full (non-truncated) string, colorize and use pager if needed + if isinstance(data, np.ndarray): + # Determine whether color is enabled (global flag) + color_enabled = not getattr(args, "nocolor", False) + + # Produce non-truncated representation but limit line width to ~80 cols so the output + # is readable. Use a large threshold to avoid numpy eliding values. + try: + # Console used for sizing and non-forced rendering + console = Console(color_system="truecolor" if color_enabled else None) + # Prefer a hard 80-column target, but avoid wider-than-terminal lines: + target_width = 80 + try: + term_width = console.size.width or target_width + except Exception: + term_width = target_width + wrap_width = min(target_width, term_width) + + array_str = np.array2string( + data, + threshold=sys.maxsize, + max_line_width=wrap_width, + ) + except Exception: + # Fallback to repr if array2string fails for any reason + console = Console(color_system="truecolor" if color_enabled else None) + array_str = repr(data) + + # Re-wrap long lines at comma+space token boundaries so we never split a numeric token. + def _wrap_preserving_tokens(s: str, width: int) -> str: + out_lines = [] + for orig in s.splitlines(): + if len(orig) <= width: + out_lines.append(orig) + continue + # Split at ', ' so numeric tokens remain whole + tokens = orig.split(", ") + line = "" + for i, tok in enumerate(tokens): + sep = ", " if i < len(tokens) - 1 else "" + piece = tok + sep + if not line: + line = piece + elif len(line) + len(piece) <= width: + line += piece + else: + out_lines.append(line) + line = piece + if line: + out_lines.append(line) + return "\n".join(out_lines) + + wrapped = _wrap_preserving_tokens(array_str, wrap_width) + + # Colorize and page + use_syntax = False + syntax = None + if color_enabled: + try: + syntax = Syntax(wrapped, "python", theme="monokai", word_wrap=False) + use_syntax = True + except Exception: + syntax = None + use_syntax = False + + # Decide whether to page: compare lines to terminal height + lines = wrapped.count("\n") + 1 + term_height = console.size.height or 24 + if lines > term_height: + pager_cmd = os.environ.get("PAGER", "less -R") + + # Preferred approach: run the pager subprocess and stream output to its stdin. + try: + args_split = shlex.split(pager_cmd) + proc = subprocess.Popen(args_split, stdin=subprocess.PIPE, text=True) + try: + if color_enabled: + # Use a Console that writes ANSI escapes into the pager stdin. + pager_console = Console( + file=proc.stdin, force_terminal=True, color_system="truecolor" + ) + if use_syntax: + pager_console.print(syntax) + else: + pager_console.print(wrapped) + else: + # Plain text: write directly to pager stdin to avoid ANSI sequences. + proc.stdin.write(wrapped) + finally: + with contextlib.suppress(Exception): + proc.stdin.close() + proc.wait() + except Exception: + # Fallback: try console.pager() while ensuring PAGER env var keeps -R option + prev_pager = os.environ.get("PAGER") + os.environ["PAGER"] = pager_cmd + try: + with console.pager(): + if color_enabled and use_syntax: + console.print(syntax) + else: + console.print(wrapped) + finally: + if prev_pager is None: + del os.environ["PAGER"] + else: + os.environ["PAGER"] = prev_pager + else: + if color_enabled and use_syntax: + console.print(syntax) + else: + console.print(wrapped) + return + + # For other objects, fallback to plain print + print(data) @handle_errors @@ -316,6 +450,8 @@ def main(): ) parser.add_argument("--username", default=conf.get("client.username")) parser.add_argument("--password", default=conf.get("client.password")) + # Make --json a global flag so it applies to all commands that support JSON output + parser.add_argument("--json", action="store_true", help="Output JSON when supported by the command") subparsers = parser.add_subparsers(required=True) # roots @@ -342,14 +478,14 @@ def main(): help = "Copy a dataset to a different root." subparser = subparsers.add_parser("copy", aliases=["cp"], help=help) subparser.add_argument("dataset", type=pathlib.Path) - subparser.add_argument("dest") + subparser.add_argument("dest", type=pathlib.Path) subparser.set_defaults(func=cmd_copy) # move help = "Move a dataset to a different root." subparser = subparsers.add_parser("move", aliases=["mv"], help=help) subparser.add_argument("dataset", type=pathlib.Path) - subparser.add_argument("dest") + subparser.add_argument("dest", type=pathlib.Path) subparser.set_defaults(func=cmd_move) # remove @@ -389,6 +525,7 @@ def main(): help = "Display a dataset." subparser = subparsers.add_parser("show", help=help) subparser.add_argument("--json", action="store_true") + subparser.add_argument("--nocolor", action="store_true", help="Disable ANSI color output for show") subparser.add_argument("dataset", type=dataset_with_slice) subparser.set_defaults(func=cmd_show) From 7b9713e012fa3fd5768f0b00a42cd69d20ffc997 Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Wed, 17 Sep 2025 11:23:40 +0200 Subject: [PATCH 6/7] Add mtime property to info cli --- caterva2/clients/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caterva2/clients/cli.py b/caterva2/clients/cli.py index 14320d44..cc06ca6d 100644 --- a/caterva2/clients/cli.py +++ b/caterva2/clients/cli.py @@ -211,11 +211,11 @@ def _filter_names(fl): typesize = cparams.get("typesize") nbytes = schunk.get("nbytes") or data.get("nbytes") cbytes = schunk.get("cbytes") or data.get("cbytes") + mtime = data.get("mtime") codec = cparams.get("codec") clevel = cparams.get("clevel") filters = cparams.get("filters") # Pretty print - # print() # print("Dataset:") print(f"nchunks : {nchunks}") if shape is None else print(f"shape : {shape}") print(f"chunksize: {_human_bytes(chunksize)}") if chunks is None else print(f"chunks: {chunks}") @@ -224,7 +224,7 @@ def _filter_names(fl): print(f"nbytes: {_human_bytes(nbytes)}") print(f"cbytes: {_human_bytes(cbytes)}") print(f"ratio : {nbytes / cbytes:.2f}x" if nbytes and cbytes else " ratio : N/A") - # print() + print(f"mtime : {mtime}") if mtime is not None else print("mtime : None") print("cparams:") print(f" codec : {_codec_name(codec)} ({codec})") print(f" clevel : {clevel}") From 9c12202d63d32392ce9a3aa54a9c04327c5ee5bf Mon Sep 17 00:00:00 2001 From: Francesc Alted Date: Thu, 18 Sep 2025 13:11:22 +0200 Subject: [PATCH 7/7] Don't show other datasets in the same root for browse command --- caterva2/api_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/caterva2/api_utils.py b/caterva2/api_utils.py index 51b4ceeb..d5aee412 100644 --- a/caterva2/api_utils.py +++ b/caterva2/api_utils.py @@ -118,8 +118,10 @@ def get_download_url(path, urlbase): def get_handle_url(path, urlbase): # Get the root in path (first element in path) - root = path.split("/")[0] - return f"{urlbase}/roots/{path}?roots={root}" + # root = path.split("/")[0] + # return f"{urlbase}/roots/{path}?roots={root}" + # We don't want to show other datasets in the same root + return f"{urlbase}/roots/{path}" def b2_unpack(filepath):