diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8f50166..0fad4ce 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -30,18 +30,13 @@ jobs:
run: |
python -m pip install --upgrade pip
pip install --upgrade setuptools wheel setuptools-scm[toml]
- pip install flake8 pytest pytest-cov pytz
- if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
- pip install .
+ pip install ".[test,lint]"
pip list
- name: Test with pytest
run: |
pytest -sv
- - name: Lint with flake8
+ - name: Lint with ruff
run: |
- # stop the build if there are Python syntax errors or undefined names
- flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
- # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
- flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics
+ ruff check .
diff --git a/.gitignore b/.gitignore
index 3c4eb84..7db8ad1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,8 +9,11 @@
.tox
.cache
.pytest_cache
+.ruff_cache
.vscode
env/
+venv*/
+build/
dist/
dtoolcore/version.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..f3dc89a
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,6 @@
+repos:
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.6.9
+ hooks:
+ - id: ruff
+ args: [--fix]
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index b89c5a7..ad43f22 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -4,6 +4,39 @@ CHANGELOG
This project uses `semantic versioning `_.
This change log uses principles from `keep a changelog `_.
+
+[Unreleased]
+------------
+
+
+Added
+^^^^^
+
+- ``ruff`` linting (replacing ``flake8``), configured in ``pyproject.toml``,
+ with a ``.pre-commit-config.yaml`` and a ``lint`` optional-dependency group
+
+Changed
+^^^^^^^
+
+- Migrated deprecated ``datetime.utcnow()`` / ``datetime.utcfromtimestamp()``
+ to timezone-aware equivalents; ``utils.timestamp()`` now treats naive
+ datetimes as UTC and accepts timezone-aware datetimes
+- ``utils.write_config_value_to_file`` now sets config-file permissions to
+ ``0o600`` as documented (previously ``0o700``)
+
+Removed
+^^^^^^^
+
+- Python 2 import shims (``urlparse`` and ``importlib_metadata`` fallbacks)
+- Redundant ``requirements.txt``; ``tox.ini`` no longer references it
+
+Fixed
+^^^^^
+
+- ``raise(Error())`` rewritten to ``raise Error()`` throughout (statement, not
+ call), resolving the corresponding lint warnings
+
+
[3.20.0] - 2025-12-08
---------------------
diff --git a/CLAUDE.md b/CLAUDE.md
new file mode 100644
index 0000000..de0f186
--- /dev/null
+++ b/CLAUDE.md
@@ -0,0 +1,148 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Overview
+
+`dtoolcore` is the dependency-free Python core API for creating and reading
+**dtool datasets** — packages of data plus structural and descriptive metadata
+for managing (scientific) data. It is a library only (no CLI here; the CLI lives
+in separate `dtool` packages). Storage backends (disk, iRODS, Azure, S3, …) are
+pluggable via setuptools entry points; this repo ships only the disk backend.
+
+## Commands
+
+```bash
+# Run the full test suite (pytest config + coverage in pyproject.toml)
+pytest
+
+# Run a single test file / test
+pytest tests/test_copy_dataset.py
+pytest tests/test_copy_dataset.py::test_copy
+
+# Lint
+ruff check .
+ruff check . --fix # auto-fix
+
+# Run tests across all supported environments (py39..py314, lint)
+tox
+tox -e lint # lint only
+
+# Install for development
+pip install -e ".[test,lint]"
+
+# Build sdist + wheel (flit_scm backend)
+python -m build
+```
+
+Tests live in `tests/`, are pytest-based, and rely on shared fixtures defined in
+`tests/__init__.py` (`tmp_uri_fixture`, `tmp_dir_fixture`, `chdir_fixture`,
+`tmp_env_var`, etc.). Most tests are "functional": they create real datasets in
+temporary directories and read them back.
+
+## Architecture
+
+The codebase has two layers with a deliberate separation of concerns:
+
+1. **`dtoolcore/__init__.py`** — the storage-agnostic public API (dataset
+ lifecycle, metadata orchestration). It never touches the filesystem directly.
+2. **`dtoolcore/storagebroker.py`** — storage backends. All I/O happens here,
+ behind the `BaseStorageBroker` interface.
+
+These communicate through two key abstractions:
+
+- **handle**: a Unix-style relative path identifying an item *within* a dataset.
+- **identifier**: the SHA-1 hexdigest of a handle (`utils.generate_identifier`).
+ Identifiers are the canonical keys used throughout the manifest and overlays.
+
+### Dataset lifecycle: ProtoDataSet → DataSet
+
+A dataset is built as a **`ProtoDataSet`** (writable, `type: "protodataset"`),
+then **frozen** into a read-only **`DataSet`** (`type: "dataset"`). Both subclass
+`_BaseDataSet`. The `freeze()` method is the pivotal transition: it generates and
+persists the manifest, converts accumulated per-item metadata fragments into
+overlays, flips the `type` in admin metadata, and stamps `frozen_at`.
+
+`freeze_with_manifest(manifest, frozen_at=None)` is an alternative that freezes
+using a **client-supplied** manifest (pre-computed hashes) instead of hashing
+server-side. It validates that the README and every manifest item exist in
+storage but trusts the provided hashes — intended for server applications.
+
+`DataSetCreator` (and `DerivedDataSetCreator`) are context managers wrapping the
+normal flow: enter → work in a staging tempdir → exit auto-freezes (unless an
+exception propagated). Prefer these for new code over manual proto/freeze
+handling.
+
+Construct datasets from a URI via `DataSet.from_uri(uri)` /
+`ProtoDataSet.from_uri(uri)`, which type-check the admin metadata. Top-level
+helpers: `create_proto_dataset`, `create_derived_proto_dataset`, `copy`,
+`copy_resume`, `iter_datasets_in_base_uri`, `iter_proto_datasets_in_base_uri`.
+
+### Metadata model
+
+Four distinct metadata categories — do not conflate them:
+
+- **admin metadata** (`.dtool/dtool`): uuid, name, type, creator, timestamps.
+- **README** (`README.yml`): free-form descriptive metadata; client ensures
+ valid YAML. `update_readme` backs up the prior version with a timestamp suffix.
+- **manifest** (`.dtool/manifest.json`): per-item structural metadata
+ (size, hash, mtime, relpath), keyed by identifier. Generated at freeze.
+- **overlays** (`.dtool/overlays/*.json`): per-item key/value metadata, keyed by
+ identifier. On a frozen dataset, overlays must have exactly the dataset's
+ identifiers as keys. Pre-freeze, use `add_item_metadata(handle, key, value)`,
+ which writes fragment files that `freeze()` aggregates into overlays.
+- **annotations** (`.dtool/annotations/*.json`) and **tags** (`.dtool/tags/`):
+ dataset-level (not per-item) metadata.
+
+### Storage brokers (pluggable backends)
+
+`_get_storage_broker(uri, config_path)` parses the URI scheme and looks up the
+broker registered under the `dtool.storage_brokers` entry point group (see
+`pyproject.toml`). The scheme (e.g. `file`) maps to a broker's `key` attribute.
+
+To add a backend, subclass `BaseStorageBroker` and implement its
+`NotImplementedError` interface methods (`get_text`/`put_text`/`delete_key`, the
+`get_*_key` methods, `iter_item_handles`, `get_size_in_bytes`/`get_utc_timestamp`/
+`get_hash`, `has_admin_metadata`, `pre_freeze_hook`/`post_freeze_hook`, etc.).
+The base class provides reusable JSON serialization logic on top of the
+text/key primitives — most higher-level methods (`get_manifest`, `put_overlay`,
+…) are already implemented in terms of `get_text`/`put_text`. `DiskStorageBroker`
+(`key = "file"`) is the reference implementation; its on-disk layout is defined
+by `_STRUCTURE_PARAMETERS`.
+
+`pre_freeze_hook` is also a validation point: `DiskStorageBroker` rejects "rogue"
+files in the dataset root that aren't part of the known structure.
+
+### Cross-platform paths & URIs (`utils.py`)
+
+URIs are central. `generous_parse_uri` turns relative paths into fully-qualified
+`file://` URIs (and handles Windows drive letters); `sanitise_uri` normalizes any
+input. Handles are always Unix-style; `handle_to_osrelpath` / `relpath_to_handle`
+convert to/from OS-native paths. When editing path logic, preserve the
+`IS_WINDOWS` branches — Windows support is a maintained feature.
+
+### Configuration (`utils.get_config_value`)
+
+Lookup precedence: **environment variable → JSON config file → default**. Config
+file defaults to `~/.config/dtool/dtool.json`. Notable keys: `DTOOL_NUM_PROCESSES`
+(parallelize manifest hashing for `file`/`symlink` brokers via multiprocessing)
+and `DTOOL_STAGING_PREFIX` (staging tempdir location for `DataSetCreator`).
+
+## Conventions
+
+- **No runtime dependencies** — keep it that way; the dependency-free promise is
+ a core feature. Version discovery uses stdlib `importlib.metadata`.
+- **Python 3.9+** only (`requires-python = ">=3.9"`). CI in
+ `.github/workflows/test.yml` tests 3.9–3.14. Note the one remaining version
+ branch: `entry_points().select(...)` (3.10+) vs `.get(...)` (3.9) in
+ `_generate_storage_broker_lookup`.
+- **Build backend is `flit_scm`** (flit + setuptools_scm). The version is derived
+ from git tags and written to `dtoolcore/version.py` (generated, do not edit).
+ Releases are git tags; tagging triggers PyPI publish.
+- **Timestamps**: `utils.timestamp()` returns seconds since the Unix epoch and
+ treats naive datetimes as UTC. Pass timezone-aware UTC datetimes from new code.
+- Update `CHANGELOG.rst` (keep-a-changelog format, semantic versioning) for
+ user-facing changes.
+- The library raises `DtoolCore*` exception subclasses (`DtoolCoreTypeError`,
+ `DtoolCoreKeyError`, `DtoolCoreValueError`, `DtoolCoreInvalidNameError`,
+ `DtoolCoreBrokenStagingPromise`) — use these rather than bare builtins.
diff --git a/README.rst b/README.rst
index d8629ed..b868447 100644
--- a/README.rst
+++ b/README.rst
@@ -45,7 +45,7 @@ Features
- Ability to create plugins for custom storage solutions
- Plugins for iRODS and Microsoft Azure storage backends available
- Cross-platform: Linux, Mac and Windows are all supported
-- Works with Python 2.7, 3.5 and 3.6
+- Works with Python 3.9 and above
- No external dependencies
Overview
diff --git a/dtoolcore/__init__.py b/dtoolcore/__init__.py
index 69b1e94..c7641f5 100644
--- a/dtoolcore/__init__.py
+++ b/dtoolcore/__init__.py
@@ -2,38 +2,32 @@
"""
-import os
-
import datetime
import logging
import multiprocessing as mp
+import os
import shutil
import tempfile
import uuid
-
from collections import defaultdict
+from importlib.metadata import PackageNotFoundError, version
import dtoolcore.utils
logger = logging.getLogger(__name__)
-# workaround for diverging python versions:
-try:
- from importlib.metadata import version, PackageNotFoundError
- logger.debug("imported version, PackageNotFoundError from importlib.metadata")
-except ModuleNotFoundError:
- from importlib_metadata import version, PackageNotFoundError
- logger.debug("imported version, PackageNotFoundError from importlib_metadata")
-
# first, try to determine dynamic version at runtime
try:
__version__ = version(__name__)
- logger.debug("Determined version %s via importlib_metadata.version", __version__)
+ logger.debug("Determined version %s via importlib.metadata", __version__)
except PackageNotFoundError:
# if that fails, check for static version file written by setuptools_scm
try:
from .version import version as __version__
- logger.debug("Determined version %s from autogenerated dtoolcore/version.py", __version__)
+ logger.debug(
+ "Determined version %s from autogenerated dtoolcore/version.py",
+ __version__,
+ )
except Exception as e:
logger.debug("All efforts to determine version failed: %s", e)
__version__ = None
@@ -117,12 +111,12 @@ def generate_admin_metadata(name, creator_username=None):
logger.debug("In generate_admin_metadata...")
if not dtoolcore.utils.name_is_valid(name):
- raise(DtoolCoreInvalidNameError())
+ raise DtoolCoreInvalidNameError()
if creator_username is None:
creator_username = dtoolcore.utils.getuser()
- datetime_obj = datetime.datetime.utcnow()
+ datetime_obj = datetime.datetime.now(datetime.timezone.utc)
admin_metadata = {
"uuid": str(uuid.uuid4()),
@@ -167,7 +161,7 @@ def generate_proto_dataset(admin_metadata, base_uri, config_path=None):
logger.debug("In generate_proto_dataset...")
name = admin_metadata.get("name")
if name is None or not dtoolcore.utils.name_is_valid(name):
- raise(DtoolCoreInvalidNameError())
+ raise DtoolCoreInvalidNameError()
uri = _generate_uri(admin_metadata, base_uri)
return ProtoDataSet(uri, admin_metadata, config_path)
@@ -419,7 +413,7 @@ class DtoolCoreBrokenStagingPromise(IOError):
pass
-class _BaseDataSet(object):
+class _BaseDataSet:
"""Base class for datasets."""
def __init__(self, uri, admin_metadata, config_path=None):
@@ -435,7 +429,7 @@ def __init__(self, uri, admin_metadata, config_path=None):
def _from_uri_with_typecheck(cls, uri, config_path, type_name):
# Make sure that the URI refers to a dataset.
if not _is_dataset(uri, config_path):
- raise(DtoolCoreTypeError("{} is not a dataset".format(uri)))
+ raise DtoolCoreTypeError("{} is not a dataset".format(uri))
# Get the admin metadata out of the URI and type check.
admin_metadata = _admin_metadata_from_uri(uri, config_path)
@@ -484,7 +478,7 @@ def update_name(self, new_name):
))
if not dtoolcore.utils.name_is_valid(new_name):
- raise(DtoolCoreInvalidNameError())
+ raise DtoolCoreInvalidNameError()
self._admin_metadata['name'] = new_name
if self._storage_broker.has_admin_metadata():
@@ -513,7 +507,7 @@ def _put_overlay(self, overlay_name, overlay):
logger.debug("Put readme content {}".format(self))
if not dtoolcore.utils.name_is_valid(overlay_name):
- raise(DtoolCoreInvalidNameError())
+ raise DtoolCoreInvalidNameError()
if not isinstance(overlay, dict):
raise DtoolCoreTypeError("Overlay must be dict")
@@ -597,7 +591,7 @@ def get_annotation(self, annotation_name):
:returns: annotation
"""
if annotation_name not in self.list_annotation_names():
- raise(DtoolCoreKeyError())
+ raise DtoolCoreKeyError()
return self._storage_broker.get_annotation(annotation_name)
def put_annotation(self, annotation_name, annotation):
@@ -610,7 +604,7 @@ def put_annotation(self, annotation_name, annotation):
"""
logger.debug("Put annotation {} {}".format(annotation_name, self))
if not dtoolcore.utils.name_is_valid(annotation_name):
- raise(DtoolCoreInvalidNameError())
+ raise DtoolCoreInvalidNameError()
self._storage_broker.put_annotation(annotation_name, annotation)
def list_annotation_names(self):
@@ -635,10 +629,10 @@ def put_tag(self, tag):
:raises: DtoolCoreValueError if the tag is not a string
"""
if not isinstance(tag, str):
- raise(DtoolCoreValueError())
+ raise DtoolCoreValueError()
if not dtoolcore.utils.name_is_valid(tag):
- raise(DtoolCoreInvalidNameError())
+ raise DtoolCoreInvalidNameError()
self._storage_broker.put_tag(tag)
@@ -661,7 +655,7 @@ class DataSet(_BaseDataSet):
"""
def __init__(self, uri, admin_metadata, config_path=None):
- super(DataSet, self).__init__(uri, admin_metadata, config_path)
+ super().__init__(uri, admin_metadata, config_path)
self._manifest_cache = None
def _identifiers(self):
@@ -724,7 +718,7 @@ def get_overlay(self, overlay_name):
"""
logger.debug("Get overlay {} {}".format(overlay_name, self))
if overlay_name not in self.list_overlay_names():
- raise(DtoolCoreKeyError())
+ raise DtoolCoreKeyError()
return self._storage_broker.get_overlay(overlay_name)
def put_overlay(self, overlay_name, overlay):
@@ -855,7 +849,7 @@ def freeze(self, progressbar=None):
# administrative metadata. It is present in the administrative metadata
# if the dataset is being copied.
if "frozen_at" not in self._admin_metadata:
- datetime_obj = datetime.datetime.utcnow()
+ datetime_obj = datetime.datetime.now(datetime.timezone.utc)
metadata_update["frozen_at"] = dtoolcore.utils.timestamp(datetime_obj) # NOQA
# Apply the change(s) to the administrative metadata.
@@ -917,7 +911,8 @@ def freeze_with_manifest(self, manifest, frozen_at=None):
# Get identifiers of items that actually exist in storage
existing_handles = set(self._storage_broker.iter_item_handles())
existing_identifiers = set(
- dtoolcore.utils.generate_identifier(h) for h in existing_handles
+ dtoolcore.utils.generate_identifier(h)
+ for h in existing_handles
)
# Check for missing items
@@ -959,8 +954,10 @@ def freeze_with_manifest(self, manifest, frozen_at=None):
if frozen_at is not None:
metadata_update["frozen_at"] = frozen_at
elif "frozen_at" not in self._admin_metadata:
- datetime_obj = datetime.datetime.utcnow()
- metadata_update["frozen_at"] = dtoolcore.utils.timestamp(datetime_obj)
+ datetime_obj = datetime.datetime.now(datetime.timezone.utc)
+ metadata_update["frozen_at"] = dtoolcore.utils.timestamp(
+ datetime_obj
+ )
# Apply the change(s) to the administrative metadata.
self._admin_metadata.update(metadata_update)
@@ -970,7 +967,7 @@ def freeze_with_manifest(self, manifest, frozen_at=None):
self._storage_broker.post_freeze_hook()
-class DataSetCreator(object):
+class DataSetCreator:
"""Context manager for creating a dataset.
Inside the context manager one works on a proto dataset. When exiting the
@@ -1015,9 +1012,9 @@ def __exit__(self, exception_type, exception_value, traceback):
# Add any staged files to the dataset.
for abspath, relpath in self._to_stage:
if not os.path.isfile(abspath):
- raise(DtoolCoreBrokenStagingPromise(
+ raise DtoolCoreBrokenStagingPromise(
"No such file: {}".format(abspath)
- ))
+ )
self.proto_dataset.put_item(abspath, relpath)
# If everything has been successful freeze the dataset.
diff --git a/dtoolcore/filehasher.py b/dtoolcore/filehasher.py
index dfc41d6..164c9e0 100644
--- a/dtoolcore/filehasher.py
+++ b/dtoolcore/filehasher.py
@@ -3,7 +3,7 @@
import hashlib
-class FileHasher(object):
+class FileHasher:
"""Class for associating hash functions with names."""
def __init__(self, hash_func):
diff --git a/dtoolcore/storagebroker.py b/dtoolcore/storagebroker.py
index 815efc5..4eda4c1 100644
--- a/dtoolcore/storagebroker.py
+++ b/dtoolcore/storagebroker.py
@@ -1,24 +1,24 @@
"""Disk storage broker."""
-import os
+import datetime
import json
-import shutil
import logging
-import datetime
+import os
+import shutil
import socket
from dtoolcore import __version__
+from dtoolcore.filehasher import FileHasher, md5sum_hexdigest
from dtoolcore.utils import (
- mkdir_parents,
+ IS_WINDOWS,
generate_identifier,
generous_parse_uri,
+ handle_to_osrelpath,
+ mkdir_parents,
timestamp,
- IS_WINDOWS,
- windows_to_unix_path,
unix_to_windows_path,
- handle_to_osrelpath,
+ windows_to_unix_path,
)
-from dtoolcore.filehasher import FileHasher, md5sum_hexdigest
logger = logging.getLogger(__name__)
@@ -70,7 +70,7 @@ class DiskStorageBrokerValidationWarning(Warning):
pass
-class BaseStorageBroker(object):
+class BaseStorageBroker:
"""Base storage broker class defining the required interface."""
# Class methods to override.
@@ -78,62 +78,62 @@ class BaseStorageBroker(object):
@classmethod
def list_dataset_uris(cls, base_uri, config_path):
"""Return list containing URIs in location given by base_uri."""
- raise(NotImplementedError())
+ raise NotImplementedError()
@classmethod
def generate_uri(cls, name, uuid, base_uri):
"""Return dataset URI."""
- raise(NotImplementedError())
+ raise NotImplementedError()
# Methods to override.
def get_text(self, key):
"""Return the text associated with the key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def put_text(self, key, text):
"""Put the text into the storage associated with the key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def delete_key(self, key):
"""Delete the file/object associated with the key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_admin_metadata_key(self):
"""Return the admin metadata key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_readme_key(self):
"""Return the admin metadata key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_manifest_key(self):
"""Return the manifest key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_overlay_key(self, overlay_name):
"""Return the overlay key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_annotation_key(self, annotation_name):
"""Return the annotation key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_tag_key(self, tag):
"""Return the tag key."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def list_overlay_names(self):
"""Return list of overlay names."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def list_annotation_names(self):
"""Return list of annotation names."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def list_tags(self):
"""Return list of tags."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_item_abspath(self, identifier):
"""Return absolute path at which item content can be accessed.
@@ -141,11 +141,11 @@ def get_item_abspath(self, identifier):
:param identifier: item identifier
:returns: absolute path from which the item content can be accessed
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def _create_structure(self):
"""Create necessary structure to hold a dataset."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def put_item(self, fpath, relpath):
"""Put item with content from fpath at relpath in dataset.
@@ -157,30 +157,30 @@ def put_item(self, fpath, relpath):
a handle
:returns: the handle given to the item
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def iter_item_handles(self):
"""Return iterator over item handles."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_size_in_bytes(self, handle):
"""Return the size in bytes."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_utc_timestamp(self, handle):
"""Return the UTC timestamp."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_hash(self, handle):
"""Return the hash."""
- raise(NotImplementedError())
+ raise NotImplementedError()
def has_admin_metadata(self):
"""Return True if the administrative metadata exists.
This is the definition of being a "dataset".
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def add_item_metadata(self, handle, key, value):
"""Store the given key:value pair for the item associated with handle.
@@ -190,7 +190,7 @@ def add_item_metadata(self, handle, key, value):
:param key: metadata key
:param value: metadata value
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def get_item_metadata(self, handle):
"""Return dictionary containing all metadata associated with handle.
@@ -202,7 +202,7 @@ def get_item_metadata(self, handle):
frozen
:returns: dictionary containing item metadata
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def pre_freeze_hook(self):
"""Pre :meth:`dtoolcore.ProtoDataSet.freeze` actions.
@@ -213,7 +213,7 @@ def pre_freeze_hook(self):
It may be useful for remote storage backends to generate
caches to remove repetitive time consuming calls
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def post_freeze_hook(self):
"""Post :meth:`dtoolcore.ProtoDataSet.freeze` cleanup actions.
@@ -224,11 +224,11 @@ def post_freeze_hook(self):
In the :class:`dtoolcore.storage_broker.DiskStorageBroker` it removes
the temporary directory for storing item metadata fragment files.
"""
- raise(NotImplementedError())
+ raise NotImplementedError()
def _list_historical_readme_keys(self):
"""Return list of historical README.yml keys."""
- raise(NotImplementedError())
+ raise NotImplementedError()
# Reusable methods.
@@ -349,7 +349,9 @@ def delete_annotation(self, annotation_name):
:param annotation_name: annotation name
"""
- logger.debug("Deleting annotation: {} {}".format(annotation_name, self))
+ logger.debug(
+ "Deleting annotation: {} {}".format(annotation_name, self)
+ )
key = self.get_annotation_key(annotation_name)
self.delete_key(key)
@@ -583,8 +585,8 @@ def get_size_in_bytes(self, handle):
def get_utc_timestamp(self, handle):
"""Return the UTC timestamp."""
fpath = self._fpath_from_handle(handle)
- datetime_obj = datetime.datetime.utcfromtimestamp(
- os.stat(fpath).st_mtime
+ datetime_obj = datetime.datetime.fromtimestamp(
+ os.stat(fpath).st_mtime, datetime.timezone.utc
)
return timestamp(datetime_obj)
@@ -646,15 +648,15 @@ def _create_structure(self):
# Ensure that the specified path does not exist and create it.
if os.path.exists(self._abspath):
- raise(StorageBrokerOSError(
+ raise StorageBrokerOSError(
"Path already exists: {}".format(self._abspath)
- ))
+ )
# Make sure the parent directory exists.
parent, _ = os.path.split(self._abspath)
if not os.path.isdir(parent):
- raise(StorageBrokerOSError(
- "No such directory: {}".format(parent)))
+ raise StorageBrokerOSError(
+ "No such directory: {}".format(parent))
os.mkdir(self._abspath)
@@ -761,7 +763,7 @@ def pre_freeze_hook(self):
for d in os.listdir(self._abspath):
if d not in allowed:
msg = "Rogue content in base of dataset: {}".format(d)
- raise(DiskStorageBrokerValidationWarning(msg))
+ raise DiskStorageBrokerValidationWarning(msg)
def post_freeze_hook(self):
"""Post :meth:`dtoolcore.ProtoDataSet.freeze` cleanup actions.
diff --git a/dtoolcore/utils.py b/dtoolcore/utils.py
index b8008f4..10c116b 100644
--- a/dtoolcore/utils.py
+++ b/dtoolcore/utils.py
@@ -1,22 +1,19 @@
"""Utility functions for dtoolcore."""
-import os
+import base64
+import binascii
+import datetime
import errno
import getpass
import hashlib
import json
+import logging
+import os
import platform
-import binascii
-import base64
-import datetime
import re
import socket
-import logging
-
-try:
- from urlparse import urlparse, urlunparse
-except ImportError:
- from urllib.parse import urlparse, urlunparse
+import stat
+from urllib.parse import urlparse, urlunparse
logger = logging.getLogger(__name__)
@@ -161,7 +158,7 @@ def write_config_value_to_file(key, value, config_path=None):
json.dump(config, fh, sort_keys=True, indent=2)
# Set 600 permissions on the config file.
- os.chmod(config_path, 33216)
+ os.chmod(config_path, stat.S_IRUSR | stat.S_IWUSR)
return get_config_value_from_file(key, config_path)
@@ -247,8 +244,12 @@ def timestamp(datetime_obj):
"""Return Unix timestamp as float.
The number of seconds that have elapsed since January 1, 1970.
+
+ Naive datetimes are assumed to be in UTC.
"""
- start_of_time = datetime.datetime(1970, 1, 1)
+ start_of_time = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
+ if datetime_obj.tzinfo is None:
+ datetime_obj = datetime_obj.replace(tzinfo=datetime.timezone.utc)
diff = datetime_obj - start_of_time
return diff.total_seconds()
diff --git a/pyproject.toml b/pyproject.toml
index b0281df..271039b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,6 +20,9 @@ test = [
"pytest-cov",
"pytz"
]
+lint = [
+ "ruff"
+]
docs = [
"sphinx",
"sphinx_rtd_theme"
@@ -45,5 +48,10 @@ write_to = "dtoolcore/version.py"
testpaths = ["tests"]
addopts = "--cov=dtoolcore --cov-report=term-missing"
-[tool.flake8]
-exclude = ["env*", ".tox", ".git", "*.egg", "build", "docs", "venv"]
+[tool.ruff]
+line-length = 79
+extend-exclude = ["dtoolcore/version.py", "docs", "venv*"]
+
+[tool.ruff.lint]
+# pycodestyle (E, W), pyflakes (F), isort (I)
+select = ["E", "W", "F", "I"]
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index d6e1198..0000000
--- a/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
--e .
diff --git a/tests/__init__.py b/tests/__init__.py
index e2b3af8..71c0753 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -9,9 +9,9 @@
from dtoolcore.utils import (
IS_WINDOWS,
- windows_to_unix_path,
- unix_to_windows_path,
generous_parse_uri,
+ unix_to_windows_path,
+ windows_to_unix_path,
)
_HERE = os.path.dirname(__file__)
diff --git a/tests/test_DiskStorageBroker.py b/tests/test_DiskStorageBroker.py
index 76e3fa1..701d8b9 100644
--- a/tests/test_DiskStorageBroker.py
+++ b/tests/test_DiskStorageBroker.py
@@ -1,15 +1,17 @@
"""Tests for disk storage broker."""
-import os
-import pytz
import datetime
+import os
import time
import pytest
+import pytz
-from . import tmp_dir_fixture # NOQA
-from . import tmp_uri_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ tmp_dir_fixture, # NOQA
+ tmp_uri_fixture, # NOQA
+)
def test_initialise():
@@ -22,8 +24,7 @@ def test_initialise():
def test_create_structure(tmp_dir_fixture): # NOQA
- from dtoolcore.storagebroker import DiskStorageBroker
- from dtoolcore.storagebroker import StorageBrokerOSError
+ from dtoolcore.storagebroker import DiskStorageBroker, StorageBrokerOSError
storagebroker = DiskStorageBroker(tmp_dir_fixture)
diff --git a/tests/test_abspath_uri.py b/tests/test_abspath_uri.py
index 3c3adc0..6423851 100644
--- a/tests/test_abspath_uri.py
+++ b/tests/test_abspath_uri.py
@@ -3,9 +3,11 @@
import pytest
-from . import TEST_SAMPLE_DATA
-from . import tmp_dir_fixture # NOQA
-from . import chdir_fixture # NOQA
+from . import (
+ TEST_SAMPLE_DATA,
+ chdir_fixture, # NOQA
+ tmp_dir_fixture, # NOQA
+)
def test_abspath_uri():
diff --git a/tests/test_annotation_functional.py b/tests/test_annotation_functional.py
index 86f1d73..085c5e2 100644
--- a/tests/test_annotation_functional.py
+++ b/tests/test_annotation_functional.py
@@ -4,21 +4,22 @@
import pytest
-from . import tmp_dir_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ tmp_dir_fixture, # NOQA
+)
def test_annotation_functional(tmp_dir_fixture): # NOQA
from dtoolcore import (
DataSet,
- ProtoDataSet,
- DtoolCoreKeyError,
DtoolCoreInvalidNameError,
- generate_admin_metadata,
+ DtoolCoreKeyError,
+ ProtoDataSet,
copy,
+ generate_admin_metadata,
)
-
from dtoolcore.storagebroker import DiskStorageBroker
name = "my_dataset"
diff --git a/tests/test_audit_fixes.py b/tests/test_audit_fixes.py
index 253afb0..3a76eca 100644
--- a/tests/test_audit_fixes.py
+++ b/tests/test_audit_fixes.py
@@ -11,14 +11,14 @@
import pytest
-from . import tmp_dir_fixture # NOQA
-
from dtoolcore.utils import (
IS_WINDOWS,
generous_parse_uri,
windows_to_unix_path,
)
+from . import tmp_dir_fixture # NOQA
+
def _sanitise_base_uri(tmp_dir):
base_uri = tmp_dir
@@ -79,8 +79,6 @@ def test_generate_proto_dataset_accepts_valid_name(tmp_dir_fixture): # NOQA
def test_copy_resume_missing_frozen_at_gives_helpful_error(tmp_dir_fixture): # NOQA
import dtoolcore
- base_uri = _sanitise_base_uri(tmp_dir_fixture)
-
src_dir = os.path.join(tmp_dir_fixture, "src")
dest_dir = os.path.join(tmp_dir_fixture, "dest")
os.mkdir(src_dir)
diff --git a/tests/test_compare_module.py b/tests/test_compare_module.py
index 9dbd171..595fb35 100644
--- a/tests/test_compare_module.py
+++ b/tests/test_compare_module.py
@@ -2,8 +2,10 @@
import os
-from . import uri_to_path
-from . import tmp_uri_fixture # NOQA
+from . import (
+ tmp_uri_fixture, # NOQA
+ uri_to_path,
+)
def create_test_files(uri):
@@ -23,8 +25,8 @@ def test_diff_identifiers(tmp_uri_fixture): # NOQA
generate_admin_metadata,
generate_proto_dataset,
)
- from dtoolcore.utils import generate_identifier
from dtoolcore.compare import diff_identifiers
+ from dtoolcore.utils import generate_identifier
fpaths = create_test_files(tmp_uri_fixture)
@@ -63,8 +65,8 @@ def test_diff_sizes(tmp_uri_fixture): # NOQA
generate_admin_metadata,
generate_proto_dataset,
)
- from dtoolcore.utils import generate_identifier
from dtoolcore.compare import diff_sizes
+ from dtoolcore.utils import generate_identifier
fpaths = create_test_files(tmp_uri_fixture)
@@ -102,9 +104,9 @@ def test_diff_content(tmp_uri_fixture): # NOQA
generate_admin_metadata,
generate_proto_dataset,
)
- from dtoolcore.utils import generate_identifier
from dtoolcore.compare import diff_content
from dtoolcore.storagebroker import DiskStorageBroker
+ from dtoolcore.utils import generate_identifier
fpaths = create_test_files(tmp_uri_fixture)
diff --git a/tests/test_copy_dataset.py b/tests/test_copy_dataset.py
index d5d13d1..5982d3c 100644
--- a/tests/test_copy_dataset.py
+++ b/tests/test_copy_dataset.py
@@ -4,9 +4,11 @@
import pytest
-from . import uri_to_path
-from . import tmp_uri_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ tmp_uri_fixture, # NOQA
+ uri_to_path,
+)
def test_copy(tmp_uri_fixture): # NOQA
@@ -76,7 +78,8 @@ def test_copy(tmp_uri_fixture): # NOQA
assert src_ds.list_annotation_names() == dest_ds.list_annotation_names()
for annotation_name in dest_ds.list_annotation_names():
- assert src_ds.get_annotation(annotation_name) == dest_ds.get_annotation(annotation_name)
+ assert src_ds.get_annotation(annotation_name) \
+ == dest_ds.get_annotation(annotation_name)
def test_copy_resume(tmp_uri_fixture): # NOQA
diff --git a/tests/test_create_dataset_helpers_functional.py b/tests/test_create_dataset_helpers_functional.py
index c25f792..e252a2e 100644
--- a/tests/test_create_dataset_helpers_functional.py
+++ b/tests/test_create_dataset_helpers_functional.py
@@ -4,15 +4,17 @@
import pytest
-from . import tmp_dir_fixture # NOQA
-from . import TEST_SAMPLE_DATA
-
from dtoolcore.utils import (
IS_WINDOWS,
generous_parse_uri,
windows_to_unix_path,
)
+from . import (
+ TEST_SAMPLE_DATA,
+ tmp_dir_fixture, # NOQA
+)
+
def _sanitise_base_uri(tmp_dir):
base_uri = tmp_dir
diff --git a/tests/test_creation_and_reading_functional.py b/tests/test_creation_and_reading_functional.py
index 2c7b6bc..3f059a4 100644
--- a/tests/test_creation_and_reading_functional.py
+++ b/tests/test_creation_and_reading_functional.py
@@ -1,23 +1,28 @@
"""Functional tests for creation and reading of a disk based DataSet."""
-import os
import datetime
+import os
import time
-import pytz
import pytest
+import pytz
-from . import tmp_dir_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ tmp_dir_fixture, # NOQA
+)
def test_basic_workflow(tmp_dir_fixture): # NOQA
- from dtoolcore import ProtoDataSet, generate_admin_metadata
- from dtoolcore import DataSet
- from dtoolcore import __version__
- from dtoolcore.utils import generate_identifier
+ from dtoolcore import (
+ DataSet,
+ ProtoDataSet,
+ __version__,
+ generate_admin_metadata,
+ )
from dtoolcore.storagebroker import DiskStorageBroker
+ from dtoolcore.utils import generate_identifier
name = "my_dataset"
admin_metadata = generate_admin_metadata(name)
@@ -59,13 +64,13 @@ def test_basic_workflow(tmp_dir_fixture): # NOQA
def test_proto_dataset_freeze_functional(tmp_dir_fixture): # NOQA
from dtoolcore import (
- generate_admin_metadata,
DataSet,
+ DtoolCoreTypeError,
ProtoDataSet,
- DtoolCoreTypeError
+ generate_admin_metadata,
)
- from dtoolcore.utils import generate_identifier
from dtoolcore.storagebroker import DiskStorageBroker
+ from dtoolcore.utils import generate_identifier
name = "func_test_dataset_freeze"
admin_metadata = generate_admin_metadata(name)
@@ -241,10 +246,9 @@ def test_creation_and_reading(tmp_dir_fixture): # NOQA
def test_basic_workflow_with_nested_handle(tmp_dir_fixture): # NOQA
- from dtoolcore import ProtoDataSet, generate_admin_metadata
- from dtoolcore import DataSet
- from dtoolcore.utils import generate_identifier
+ from dtoolcore import DataSet, ProtoDataSet, generate_admin_metadata
from dtoolcore.storagebroker import DiskStorageBroker
+ from dtoolcore.utils import generate_identifier
name = "my_dataset"
admin_metadata = generate_admin_metadata(name)
diff --git a/tests/test_dtool_list_overlays_when_dir_missing.py b/tests/test_dtool_list_overlays_when_dir_missing.py
index 02e071d..97dc3ac 100644
--- a/tests/test_dtool_list_overlays_when_dir_missing.py
+++ b/tests/test_dtool_list_overlays_when_dir_missing.py
@@ -1,7 +1,9 @@
import os
-from . import chdir_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ chdir_fixture, # NOQA
+)
def test_list_overlays_when_dir_missing(chdir_fixture): # NOQA
@@ -14,8 +16,7 @@ def test_list_overlays_when_dir_missing(chdir_fixture): # NOQA
https://github.com/jic-dtool/dtoolcore/issues/3
"""
- from dtoolcore import ProtoDataSet, generate_admin_metadata
- from dtoolcore import DataSet
+ from dtoolcore import DataSet, ProtoDataSet, generate_admin_metadata
from dtoolcore.storagebroker import DiskStorageBroker
name = "my_dataset"
diff --git a/tests/test_filehasher_api.py b/tests/test_filehasher_api.py
index 8264d26..e981338 100644
--- a/tests/test_filehasher_api.py
+++ b/tests/test_filehasher_api.py
@@ -22,8 +22,9 @@ def test_md5sum_hexdigest():
def test_md5sum_digest():
- from dtoolcore.filehasher import md5sum_digest
from base64 import b64encode
+
+ from dtoolcore.filehasher import md5sum_digest
expected = "3HMZLS+B1wCc5aHue61XVQ=="
test_file = os.path.join(TEST_SAMPLE_DATA, 'tiny.png')
actual = b64encode(md5sum_digest(test_file)).decode("utf-8")
diff --git a/tests/test_freeze_with_manifest.py b/tests/test_freeze_with_manifest.py
index db35550..948ffa2 100644
--- a/tests/test_freeze_with_manifest.py
+++ b/tests/test_freeze_with_manifest.py
@@ -1,19 +1,18 @@
"""Test the freeze_with_manifest method of ProtoDataSet."""
import os
-import uuid as uuid_module
import pytest
-from . import tmp_dir_fixture # NOQA
-
from dtoolcore.utils import (
IS_WINDOWS,
+ generate_identifier,
generous_parse_uri,
windows_to_unix_path,
- generate_identifier,
)
+from . import tmp_dir_fixture # NOQA
+
def _sanitise_base_uri(tmp_dir):
base_uri = tmp_dir
@@ -61,9 +60,9 @@ def test_freeze_with_manifest_basic(tmp_dir_fixture): # NOQA
def test_freeze_with_manifest_with_items(tmp_dir_fixture): # NOQA
"""Test freezing with manifest containing items."""
- import dtoolcore
import tempfile
- import os
+
+ import dtoolcore
base_uri = _sanitise_base_uri(tmp_dir_fixture)
name = "test-items"
@@ -100,7 +99,9 @@ def test_freeze_with_manifest_with_items(tmp_dir_fixture): # NOQA
# Add items to storage
temp_files = []
for relpath in ["data/file1.txt", "data/file2.csv"]:
- with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
+ with tempfile.NamedTemporaryFile(
+ mode='w', delete=False, suffix='.txt'
+ ) as f:
f.write(f"content for {relpath}")
temp_files.append((f.name, relpath))
@@ -127,9 +128,10 @@ def test_freeze_with_manifest_with_items(tmp_dir_fixture): # NOQA
def test_freeze_with_manifest_auto_frozen_at(tmp_dir_fixture): # NOQA
"""Test that frozen_at is auto-generated if not provided."""
- import dtoolcore
import time
+ import dtoolcore
+
base_uri = _sanitise_base_uri(tmp_dir_fixture)
name = "test-auto-frozen-at"
@@ -250,9 +252,9 @@ def test_freeze_with_manifest_with_annotations(tmp_dir_fixture): # NOQA
def test_freeze_with_manifest_full(tmp_dir_fixture): # NOQA
"""Test freezing with all features combined."""
- import dtoolcore
import tempfile
- import os
+
+ import dtoolcore
base_uri = _sanitise_base_uri(tmp_dir_fixture)
name = "full-test-dataset"
@@ -293,7 +295,9 @@ def test_freeze_with_manifest_full(tmp_dir_fixture): # NOQA
proto_dataset.put_annotation(ann_name, ann_value)
# Add item to storage
- with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f:
+ with tempfile.NamedTemporaryFile(
+ mode='w', delete=False, suffix='.json'
+ ) as f:
f.write('{"results": "test data"}')
temp_path = f.name
@@ -393,7 +397,6 @@ def test_dataset_type_after_freeze(tmp_dir_fixture): # NOQA
def test_freeze_with_manifest_missing_readme(tmp_dir_fixture): # NOQA
"""Test that freezing fails if README is missing."""
import dtoolcore
- import os
base_uri = _sanitise_base_uri(tmp_dir_fixture)
name = "test-missing-readme"
@@ -468,9 +471,9 @@ def test_freeze_with_manifest_missing_items(tmp_dir_fixture): # NOQA
def test_freeze_with_manifest_partial_items(tmp_dir_fixture): # NOQA
"""Test that freezing fails if some manifest items are missing."""
- import dtoolcore
import tempfile
- import os
+
+ import dtoolcore
base_uri = _sanitise_base_uri(tmp_dir_fixture)
name = "test-partial-items"
@@ -504,7 +507,9 @@ def test_freeze_with_manifest_partial_items(tmp_dir_fixture): # NOQA
)
# Create a temporary file to add to the dataset
- with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
+ with tempfile.NamedTemporaryFile(
+ mode='w', delete=False, suffix='.txt'
+ ) as f:
f.write("test content")
temp_path = f.name
@@ -525,9 +530,9 @@ def test_freeze_with_manifest_partial_items(tmp_dir_fixture): # NOQA
def test_freeze_with_manifest_items_exist(tmp_dir_fixture): # NOQA
"""Test that freezing succeeds when all items exist."""
- import dtoolcore
import tempfile
- import os
+
+ import dtoolcore
base_uri = _sanitise_base_uri(tmp_dir_fixture)
name = "test-items-exist"
@@ -563,7 +568,9 @@ def test_freeze_with_manifest_items_exist(tmp_dir_fixture): # NOQA
# Create temporary files and add them to the dataset
temp_files = []
for relpath in ["data/file1.txt", "data/file2.txt"]:
- with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f:
+ with tempfile.NamedTemporaryFile(
+ mode='w', delete=False, suffix='.txt'
+ ) as f:
f.write(f"content for {relpath}")
temp_files.append((f.name, relpath))
diff --git a/tests/test_frozen_at_when_copying_dataset.py b/tests/test_frozen_at_when_copying_dataset.py
index caf453c..fbda614 100644
--- a/tests/test_frozen_at_when_copying_dataset.py
+++ b/tests/test_frozen_at_when_copying_dataset.py
@@ -5,9 +5,10 @@
import os
import time
+import dtoolcore as dc
+
from . import tmp_dir_fixture # NOQA
-import dtoolcore as dc
def test_frozen_at_value_when_copying_dataset(tmp_dir_fixture): # NOQA
diff --git a/tests/test_generate_admin_metadata.py b/tests/test_generate_admin_metadata.py
index fe6c26a..f6d3a2b 100644
--- a/tests/test_generate_admin_metadata.py
+++ b/tests/test_generate_admin_metadata.py
@@ -1,9 +1,9 @@
"""Test the generate_admin_metadata helper function."""
import datetime
-import pytz
import pytest
+import pytz
def test_generate_admin_metadata():
@@ -28,8 +28,7 @@ def test_generate_admin_metadata():
def test_generate_admin_metadata_raises_invalid_name_error():
- from dtoolcore import generate_admin_metadata
- from dtoolcore import DtoolCoreInvalidNameError
+ from dtoolcore import DtoolCoreInvalidNameError, generate_admin_metadata
with pytest.raises(DtoolCoreInvalidNameError):
generate_admin_metadata("/root/bad-name")
diff --git a/tests/test_iter_datasets_in_base_uri.py b/tests/test_iter_datasets_in_base_uri.py
index 95912bb..c2ea698 100644
--- a/tests/test_iter_datasets_in_base_uri.py
+++ b/tests/test_iter_datasets_in_base_uri.py
@@ -6,9 +6,9 @@
def test_iter_datasets_in_base_uri(tmp_uri_fixture): # NOQA
from dtoolcore import (
- create_proto_dataset,
DataSet,
ProtoDataSet,
+ create_proto_dataset,
iter_datasets_in_base_uri,
iter_proto_datasets_in_base_uri,
)
diff --git a/tests/test_overlays_functional.py b/tests/test_overlays_functional.py
index efa10ee..56481ab 100644
--- a/tests/test_overlays_functional.py
+++ b/tests/test_overlays_functional.py
@@ -4,25 +4,26 @@
import pytest
-from . import tmp_dir_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ tmp_dir_fixture, # NOQA
+)
def test_overlays_functional(tmp_dir_fixture): # NOQA
from dtoolcore import (
DataSet,
- ProtoDataSet,
+ DtoolCoreInvalidNameError,
DtoolCoreKeyError,
DtoolCoreTypeError,
DtoolCoreValueError,
- DtoolCoreInvalidNameError,
- generate_admin_metadata,
+ ProtoDataSet,
copy,
+ generate_admin_metadata,
)
-
- from dtoolcore.utils import generate_identifier
from dtoolcore.storagebroker import DiskStorageBroker
+ from dtoolcore.utils import generate_identifier
name = "my_dataset"
admin_metadata = generate_admin_metadata(name)
diff --git a/tests/test_self_description.py b/tests/test_self_description.py
index b4440e1..0ea705b 100644
--- a/tests/test_self_description.py
+++ b/tests/test_self_description.py
@@ -1,12 +1,14 @@
"""Test the DiskStorageBroker self description metadata."""
-import os
import json
+import os
from dtoolcore import __version__
-from . import uri_to_path
-from . import tmp_uri_fixture # NOQA
+from . import (
+ tmp_uri_fixture, # NOQA
+ uri_to_path,
+)
def test_writing_of_dtool_structure_file(tmp_uri_fixture): # NOQA
diff --git a/tests/test_tags_functional.py b/tests/test_tags_functional.py
index 74bdc1e..7be2510 100644
--- a/tests/test_tags_functional.py
+++ b/tests/test_tags_functional.py
@@ -7,7 +7,7 @@
def test_tags_functional(tmp_dir_fixture): # NOQA
- from dtoolcore import DataSetCreator, DataSet
+ from dtoolcore import DataSet, DataSetCreator
with DataSetCreator(name="empty-test-ds", base_uri=tmp_dir_fixture) as c:
# Test put on proto dataset.
diff --git a/tests/test_timestamp.py b/tests/test_timestamp.py
index c3e9417..4ef24de 100644
--- a/tests/test_timestamp.py
+++ b/tests/test_timestamp.py
@@ -1,6 +1,6 @@
"""Test timestamp functionality."""
-from datetime import datetime
+from datetime import datetime, timezone
def test_timestamp_returns_float():
@@ -33,7 +33,7 @@ def test_subsection_precision():
import dtoolcore.utils
time_as_float = 946684800.513
- into_new_millenium = datetime.utcfromtimestamp(time_as_float)
+ into_new_millenium = datetime.fromtimestamp(time_as_float, timezone.utc)
tolerance = 0.000001
actual = dtoolcore.utils.timestamp(into_new_millenium)
diff --git a/tests/test_uri_property_when_using_relpath.py b/tests/test_uri_property_when_using_relpath.py
index d59afdd..7079097 100644
--- a/tests/test_uri_property_when_using_relpath.py
+++ b/tests/test_uri_property_when_using_relpath.py
@@ -1,19 +1,16 @@
import os
-from . import chdir_fixture # NOQA
-from . import TEST_SAMPLE_DATA
+from . import (
+ TEST_SAMPLE_DATA,
+ chdir_fixture, # NOQA
+)
def test_uri_property_when_using_relpath(chdir_fixture): # NOQA
- from dtoolcore import ProtoDataSet, generate_admin_metadata
- from dtoolcore import DataSet
+ from dtoolcore import DataSet, ProtoDataSet, generate_admin_metadata
from dtoolcore.storagebroker import DiskStorageBroker
- from dtoolcore.utils import (
- IS_WINDOWS,
- windows_to_unix_path,
- urlparse
- )
+ from dtoolcore.utils import IS_WINDOWS, urlparse, windows_to_unix_path
name = "my_dataset"
admin_metadata = generate_admin_metadata(name)
diff --git a/tests/test_uri_sanitisation.py b/tests/test_uri_sanitisation.py
index ab908a1..48ddb3f 100644
--- a/tests/test_uri_sanitisation.py
+++ b/tests/test_uri_sanitisation.py
@@ -43,7 +43,7 @@ def test_generous_parse_uri():
def test_sanitise_uri():
- from dtoolcore.utils import sanitise_uri, IS_WINDOWS, windows_to_unix_path
+ from dtoolcore.utils import IS_WINDOWS, sanitise_uri, windows_to_unix_path
relpath = "./my_data"
diff --git a/tests/test_utils_module.py b/tests/test_utils_module.py
index d4619a4..0625554 100644
--- a/tests/test_utils_module.py
+++ b/tests/test_utils_module.py
@@ -1,16 +1,15 @@
"""Test the dtoolcore.utils module."""
+import json
import os
+import stat
import sys
-import json
+from unittest.mock import MagicMock
-try:
- from unittest.mock import MagicMock
-except ImportError:
- from mock import MagicMock
-
-from . import tmp_dir_fixture # NOQA
-from . import tmp_env_var
+from . import (
+ tmp_dir_fixture, # NOQA
+ tmp_env_var,
+)
def test_sha1_hexdigest():
@@ -29,32 +28,36 @@ def test_generate_identifier():
def test_cross_platform_getuser_windows_and_no_username_env_var():
- from dtoolcore.utils import cross_platform_getuser
import getpass
+
+ from dtoolcore.utils import cross_platform_getuser
getpass.getuser = MagicMock(return_value="user1")
assert cross_platform_getuser(True, True) == "unknown"
getpass.getuser.assert_not_called()
def test_cross_platform_getuser_windows_and_username_env_var():
- from dtoolcore.utils import cross_platform_getuser
import getpass
+
+ from dtoolcore.utils import cross_platform_getuser
getpass.getuser = MagicMock(return_value="user1")
assert cross_platform_getuser(True, False) == "user1"
getpass.getuser.assert_called_once()
def test_cross_platform_getuser_not_windows_and_username_env_var():
- from dtoolcore.utils import cross_platform_getuser
import getpass
+
+ from dtoolcore.utils import cross_platform_getuser
getpass.getuser = MagicMock(return_value="user1")
assert cross_platform_getuser(False, False) == "user1"
getpass.getuser.assert_called_once()
def test_cross_platform_getuser_not_windows_and_no_username_env_var():
- from dtoolcore.utils import cross_platform_getuser
import getpass
+
+ from dtoolcore.utils import cross_platform_getuser
getpass.getuser = MagicMock(return_value="user1")
assert cross_platform_getuser(False, True) == "user1"
getpass.getuser.assert_called_once()
@@ -108,7 +111,7 @@ def test_write_config_value_to_file(tmp_dir_fixture): # NOQA
# Ensure that the file has 600 permissions.
if sys.platform != "win32":
# Don't do this check on windows
- assert os.stat(config_path).st_mode == 33216
+ assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600
def test_get_config_value():
@@ -206,7 +209,7 @@ def test_name_is_valid():
assert not name_is_valid("x" * 81)
assert not name_is_valid("/root/this-is-a-bad-name")
- assert not name_is_valid("th\is-is-a-bad-name") # NOQA
+ assert not name_is_valid(r"th\is-is-a-bad-name") # NOQA
assert not name_is_valid("th\\is-is-a-bad-name")
assert not name_is_valid("th\nis-is-a-bad-name")
assert not name_is_valid("{this-is-a-bad-name}")
diff --git a/tox.ini b/tox.ini
index d6a253a..034406b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,16 +1,10 @@
[tox]
-envlist=py27,py3,flake8
+envlist = py39,py310,py311,py312,py313,py314,lint
[testenv]
-deps=pytest
- pytest-cov
- mock
- pytest-mock
- coverage
- pytz
- -r{toxinidir}/requirements.txt
-commands=py.test
+extras = test
+commands = pytest {posargs}
-[testenv:flake8]
-deps=flake8
-commands=flake8
+[testenv:lint]
+extras = lint
+commands = ruff check .