diff --git a/docs/backends.rst b/docs/backends.rst index ef9b63f..b9b8c0a 100644 --- a/docs/backends.rst +++ b/docs/backends.rst @@ -23,6 +23,8 @@ Use storage on a local POSIX filesystem: - Namespaces: directories - Values: in key-named files - atime: supported (require fs atime support) +- mtime: supported (stamped by the filesystem's clock - for a network filesystem + that is usually the file server's clock) - Quota: tracks backend storage size and rejects ``store`` if quota is exceeded. The current usage is persisted to a hidden file in the storage directory. @@ -121,6 +123,7 @@ Use storage on an SFTP server: - Values: in key-named files - store: a ``memoryview`` value is copied into a ``bytes`` object first, because paramiko does not accept a ``memoryview``. +- atime / mtime: supported (stamped by the SFTP server's filesystem) - hash: runs the hexdigest computation server-side (if server supports check-file). "blake3" is not part of the check-file extension, so it is always computed client-side. @@ -134,6 +137,8 @@ Use storage on any of the many cloud providers `rclone `_ s - The implementation primarily depends on the specific remote. - The rclone binary path can be set via the environment variable ``RCLONE_BINARY`` (default: "rclone"). - Debugging of HTTP requests/responses can be enabled by setting ``BORGSTORE_RCLONE_DEBUG=1``. +- atime / mtime: not supported (always 0). rclone's ModTime is the client-side mtime + preserved across upload, not a timestamp stamped by the storage side. s3 @@ -163,6 +168,8 @@ Use storage on an S3-compliant cloud service: - Values: in key-named files - store: a ``memoryview`` value is copied into a ``bytes`` object first, because boto3 does not accept a ``memoryview``. +- atime: not supported (always 0). +- mtime: supported (``LastModified``, stamped by the S3 service). REST (http/https) @@ -180,4 +187,5 @@ Use a storage backend running inside a BorgStore REST server process: - hash: runs the hexdigest computation server-side. Using algorithm "blake3" requires the optional ``blake3`` package to be installed **on the server**. - defrag: runs the defragmentation helper server-side. -- atime: supported (if backend used by server supports it). +- atime / mtime: supported (if backend used by server supports it). mtime is stamped + by the server side (the store operation executes there). diff --git a/docs/changes.rst b/docs/changes.rst index 24c65b8..3195214 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,17 @@ Changelog ========= +Version 0.6.1 (not released yet) +-------------------------------- + +New features: + +- ItemInfo: new ``mtime`` field - the last modification timestamp stamped by the + *storage side's* clock (0 if unknown). Implemented for posixfs, sftp, s3 and + rest; rclone reports 0 because its ModTime is a client-supplied timestamp. + sftp additionally reports atime now. + + Version 0.6.0 (2026-08-02) -------------------------- diff --git a/src/borgstore/backends/_base.py b/src/borgstore/backends/_base.py index 990720a..da9f69f 100644 --- a/src/borgstore/backends/_base.py +++ b/src/borgstore/backends/_base.py @@ -11,8 +11,11 @@ from ..constants import MAX_NAME_LENGTH, TMP_SUFFIX, HID_SUFFIX from ..utils import hashing -# atime is the last read access UNIX timestamp [s] or 0 if not implemented -ItemInfo = namedtuple("ItemInfo", "name exists size directory atime", defaults=(0,)) +# atime is the last read access UNIX timestamp [s] or 0 if not implemented. +# mtime is the last modification UNIX timestamp [s] or 0 if not implemented - it must be +# stamped by the *storage side's* clock; backends that would only echo a client-supplied +# timestamp (e.g. rclone) must report 0 (unknown) instead. +ItemInfo = namedtuple("ItemInfo", "name exists size directory atime mtime", defaults=(0, 0)) # type of a value given to store: a memoryview is accepted in addition to bytes, # so callers can avoid copying (e.g. give a slice of a big buffer they already have). diff --git a/src/borgstore/backends/posixfs.py b/src/borgstore/backends/posixfs.py index 7d4d3e2..d8c48c5 100644 --- a/src/borgstore/backends/posixfs.py +++ b/src/borgstore/backends/posixfs.py @@ -205,7 +205,9 @@ def info(self, name): return ItemInfo(name=path.name, exists=False, directory=False, size=0) else: is_dir = stat.S_ISDIR(st.st_mode) - return ItemInfo(name=path.name, exists=True, directory=is_dir, size=st.st_size, atime=st.st_atime) + return ItemInfo( + name=path.name, exists=True, directory=is_dir, size=st.st_size, atime=st.st_atime, mtime=st.st_mtime + ) def load(self, name, *, size=None, offset=0): if not self.opened: @@ -355,7 +357,14 @@ def list(self, name): pass else: is_dir = stat.S_ISDIR(st.st_mode) - yield ItemInfo(name=p.name, exists=True, size=st.st_size, directory=is_dir, atime=st.st_atime) + yield ItemInfo( + name=p.name, + exists=True, + size=st.st_size, + directory=is_dir, + atime=st.st_atime, + mtime=st.st_mtime, + ) def quota(self) -> dict: """Return quota information: limit and usage in bytes. -1 means not set / not tracked.""" diff --git a/src/borgstore/backends/rclone.py b/src/borgstore/backends/rclone.py index 66060e2..461b1ef 100644 --- a/src/borgstore/backends/rclone.py +++ b/src/borgstore/backends/rclone.py @@ -251,6 +251,10 @@ def _to_item_info(self, remote, item): name = item["Name"] size = item["Size"] directory = item["IsDir"] + # no atime/mtime: rclone's ModTime is the *client-side* mtime preserved across upload + # (not stamped by the storage side), so it must not be reported as ItemInfo.mtime; + # also, fetching it would need "noModTime": False, which on several remotes costs an + # extra metadata read per object during list(). return ItemInfo(name=name, exists=True, size=size, directory=directory) def info(self, name) -> ItemInfo: diff --git a/src/borgstore/backends/rest.py b/src/borgstore/backends/rest.py index c5970f0..6d4a2fa 100644 --- a/src/borgstore/backends/rest.py +++ b/src/borgstore/backends/rest.py @@ -526,8 +526,9 @@ def info(self, name: str) -> ItemInfo: exists = response.status_code == HTTP.OK is_dir = response.headers.get("X-BorgStore-Is-Directory") == "true" atime = float(response.headers.get("X-BorgStore-Atime", 0)) + mtime = float(response.headers.get("X-BorgStore-Mtime", 0)) # 0: old server without mtime support size = int(response.headers.get("Content-Length", 0)) if exists else 0 - return ItemInfo(name=name, exists=exists, size=size, directory=is_dir, atime=atime) + return ItemInfo(name=name, exists=exists, size=size, directory=is_dir, atime=atime, mtime=mtime) @with_reconnect def load(self, name: str, *, size=None, offset=0) -> bytes: @@ -633,4 +634,5 @@ def list(self, name: str) -> Iterator[ItemInfo]: size=entry["size"], directory=entry.get("directory", False), atime=entry.get("atime", 0), + mtime=entry.get("mtime", 0), # 0: old server without mtime support ) diff --git a/src/borgstore/backends/s3.py b/src/borgstore/backends/s3.py index 438e12b..eabb6d6 100644 --- a/src/borgstore/backends/s3.py +++ b/src/borgstore/backends/s3.py @@ -273,7 +273,13 @@ def list(self, name): pass # that file is likely not from us or is still uploading else: start_after = obj["Key"] - yield ItemInfo(name=obj_name, exists=True, size=obj["Size"], directory=False) + yield ItemInfo( + name=obj_name, + exists=True, + size=obj["Size"], + directory=False, + mtime=obj["LastModified"].timestamp(), + ) for prefix in objects.get("CommonPrefixes", []): dir_name = prefix["Prefix"][len(base_prefix) : -1] # Remove base_path prefix and trailing slash yield ItemInfo(name=dir_name, exists=True, size=0, directory=True) @@ -303,7 +309,13 @@ def info(self, name): key = self.base_path + name try: obj = self.s3.head_object(Bucket=self.bucket, Key=key) - return ItemInfo(name=name, exists=True, directory=False, size=obj["ContentLength"]) + return ItemInfo( + name=name, + exists=True, + directory=False, + size=obj["ContentLength"], + mtime=obj["LastModified"].timestamp(), + ) except self.s3.exceptions.ClientError as e: if e.response["Error"]["Code"] == "404": try: diff --git a/src/borgstore/backends/sftp.py b/src/borgstore/backends/sftp.py index a14aee2..9d46cb0 100644 --- a/src/borgstore/backends/sftp.py +++ b/src/borgstore/backends/sftp.py @@ -412,7 +412,9 @@ def info(self, name): return ItemInfo(name=name, exists=False, directory=False, size=0) else: is_dir = stat.S_ISDIR(st.st_mode) - return ItemInfo(name=name, exists=True, directory=is_dir, size=st.st_size) + return ItemInfo( + name=name, exists=True, directory=is_dir, size=st.st_size, atime=st.st_atime, mtime=st.st_mtime + ) @with_reconnect def load(self, name, *, size=None, offset=0): @@ -546,4 +548,11 @@ def list(self, name): pass # that file is likely not from us or is still uploading else: is_dir = stat.S_ISDIR(info.st_mode) - yield ItemInfo(name=info.filename, exists=True, size=info.st_size, directory=is_dir) + yield ItemInfo( + name=info.filename, + exists=True, + size=info.st_size, + directory=is_dir, + atime=info.st_atime, + mtime=info.st_mtime, + ) diff --git a/src/borgstore/server/rest.py b/src/borgstore/server/rest.py index fdc7a9f..1b456b9 100644 --- a/src/borgstore/server/rest.py +++ b/src/borgstore/server/rest.py @@ -310,6 +310,7 @@ def do_HEAD(self): "Content-Length": str(info.size), "X-BorgStore-Is-Directory": "true" if info.directory else "false", "X-BorgStore-Atime": str(info.atime), + "X-BorgStore-Mtime": str(info.mtime), }, ) except Exception as e: @@ -324,7 +325,13 @@ def do_GET(self): # [{"name": "...", "size": ...}, ...] with self.server.backend: items = ( - {"name": item.name, "size": item.size, "directory": item.directory, "atime": item.atime} + { + "name": item.name, + "size": item.size, + "directory": item.directory, + "atime": item.atime, + "mtime": item.mtime, + } for item in self.server.backend.list(self.name) ) json_data = json.dumps(list(items), indent=2) diff --git a/tests/test_backends.py b/tests/test_backends.py index 9faefe6..4f90389 100644 --- a/tests/test_backends.py +++ b/tests/test_backends.py @@ -540,6 +540,9 @@ def test_list(tested_backends, request): assert matching_k1[0].exists and not matching_k1[0].directory and matching_k1[0].size == len(v1) assert matching_k0[0].atime >= 0 assert matching_k1[0].atime >= 0 + # mtime is 0 for backends that can not provide a storage-side timestamp (e.g. rclone) + assert matching_k0[0].mtime >= 0 + assert matching_k1[0].mtime >= 0 # for "dir", we do not know what size the backend has returned. # that is rather OS / fs / backend specific. matching_items = [item for item in items if item.name == "dir"] diff --git a/tests/test_server_rest.py b/tests/test_server_rest.py index 0bc90b1..5bef820 100644 --- a/tests/test_server_rest.py +++ b/tests/test_server_rest.py @@ -699,12 +699,14 @@ def do_request(method, path, body=b""): assert status == 200 items = json.loads(body.decode("utf-8")) assert any(item["name"] == "item1" and item.get("atime", 0) > 0 for item in items) + assert any(item["name"] == "item1" and item.get("mtime", 0) > 0 for item in items) # 4. Info (HEAD) status, body, headers = do_request("HEAD", "/item1") assert status == 200 assert body == b"" assert float(headers.get("X-BorgStore-Atime", 0)) > 0 + assert float(headers.get("X-BorgStore-Mtime", 0)) > 0 # 5. Info for nonexistent (HEAD) status, body, headers = do_request("HEAD", "/nonexistent") @@ -740,12 +742,14 @@ def test_rest_url(tmp_path): assert info.exists assert info.size == len(item_data) assert info.atime > 0 + assert info.mtime > 0 # Test listing items = list(store.list("")) assert len(items) == 1 assert items[0].name == item_name assert items[0].atime > 0 + assert items[0].mtime > 0 # Test nonexistent item # This also used to hang if it returned a 404 with a body. diff --git a/tests/test_store.py b/tests/test_store.py index b82980a..9310299 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -4,6 +4,8 @@ import array import hashlib +import time + import pytest from . import key, list_store_names, list_store_names_sorted @@ -90,6 +92,9 @@ def test_basics(posixfs_store_created): assert items[0].size == len(v0) assert not items[0].directory assert items[0].atime >= 0 + # posixfs: mtime is the (local) filesystem clock at store() time + assert abs(items[0].mtime - time.time()) < 120 + assert abs(store.info(nsk0).mtime - time.time()) < 120 store.delete(nsk0)