From 8cfe782807d426025c43a4680de65c7fd24cf0ca Mon Sep 17 00:00:00 2001 From: Johannes Laurin Hoermann Date: Tue, 16 Jun 2026 17:29:00 +0900 Subject: [PATCH] Modernize tooling and fix latent bugs (rebased on flit master) Re-applies the PR #43 modernization on top of the current master, which has since adopted the flit_scm build system (PR #42). Keeps flit; layers on the source/tooling cleanups that master does not yet have. Bugs: - write_config_value_to_file set 0o700 permissions despite documenting 0o600; now sets 0o600 via stat constants - migrate deprecated datetime.utcnow()/utcfromtimestamp() (including the new freeze_with_manifest code) to timezone-aware equivalents; utils.timestamp() treats naive datetimes as UTC and accepts aware ones Removals: - Python 2 import shims (urlparse, importlib_metadata fallbacks) - redundant requirements.txt (tox.ini no longer references it) Tooling: - replace flake8 with ruff (config in pyproject, lint extra, .pre-commit-config.yaml); rewrite raise(Error()) -> raise Error() throughout, including the raise() added by PR #42 - update tox (py39..py314, lint) and the GitHub Actions workflow (ruff step, .[test,lint] install) - super(Cls, self) -> super(); drop (object) base classes - refresh README supported-versions, CHANGELOG, .gitignore; add CLAUDE.md Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/test.yml | 11 +- .gitignore | 3 + .pre-commit-config.yaml | 6 + CHANGELOG.rst | 33 ++++ CLAUDE.md | 148 ++++++++++++++++++ README.rst | 2 +- dtoolcore/__init__.py | 63 ++++---- dtoolcore/filehasher.py | 2 +- dtoolcore/storagebroker.py | 90 +++++------ dtoolcore/utils.py | 25 +-- pyproject.toml | 12 +- requirements.txt | 1 - tests/__init__.py | 4 +- tests/test_DiskStorageBroker.py | 15 +- tests/test_abspath_uri.py | 8 +- tests/test_annotation_functional.py | 13 +- tests/test_audit_fixes.py | 6 +- tests/test_compare_module.py | 12 +- tests/test_copy_dataset.py | 11 +- .../test_create_dataset_helpers_functional.py | 8 +- tests/test_creation_and_reading_functional.py | 32 ++-- ...st_dtool_list_overlays_when_dir_missing.py | 9 +- tests/test_filehasher_api.py | 3 +- tests/test_freeze_with_manifest.py | 43 ++--- tests/test_frozen_at_when_copying_dataset.py | 3 +- tests/test_generate_admin_metadata.py | 5 +- tests/test_iter_datasets_in_base_uri.py | 2 +- tests/test_overlays_functional.py | 15 +- tests/test_self_description.py | 8 +- tests/test_tags_functional.py | 2 +- tests/test_timestamp.py | 4 +- tests/test_uri_property_when_using_relpath.py | 15 +- tests/test_uri_sanitisation.py | 2 +- tests/test_utils_module.py | 31 ++-- tox.ini | 18 +-- 35 files changed, 438 insertions(+), 227 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 CLAUDE.md delete mode 100644 requirements.txt diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f50166..0fad4ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,18 +30,13 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade setuptools wheel setuptools-scm[toml] - pip install flake8 pytest pytest-cov pytz - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip install . + pip install ".[test,lint]" pip list - name: Test with pytest run: | pytest -sv - - name: Lint with flake8 + - name: Lint with ruff run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=12 --max-line-length=127 --statistics + ruff check . diff --git a/.gitignore b/.gitignore index 3c4eb84..7db8ad1 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,11 @@ .tox .cache .pytest_cache +.ruff_cache .vscode env/ +venv*/ +build/ dist/ dtoolcore/version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f3dc89a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,6 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.9 + hooks: + - id: ruff + args: [--fix] diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b89c5a7..ad43f22 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,39 @@ CHANGELOG This project uses `semantic versioning `_. This change log uses principles from `keep a changelog `_. + +[Unreleased] +------------ + + +Added +^^^^^ + +- ``ruff`` linting (replacing ``flake8``), configured in ``pyproject.toml``, + with a ``.pre-commit-config.yaml`` and a ``lint`` optional-dependency group + +Changed +^^^^^^^ + +- Migrated deprecated ``datetime.utcnow()`` / ``datetime.utcfromtimestamp()`` + to timezone-aware equivalents; ``utils.timestamp()`` now treats naive + datetimes as UTC and accepts timezone-aware datetimes +- ``utils.write_config_value_to_file`` now sets config-file permissions to + ``0o600`` as documented (previously ``0o700``) + +Removed +^^^^^^^ + +- Python 2 import shims (``urlparse`` and ``importlib_metadata`` fallbacks) +- Redundant ``requirements.txt``; ``tox.ini`` no longer references it + +Fixed +^^^^^ + +- ``raise(Error())`` rewritten to ``raise Error()`` throughout (statement, not + call), resolving the corresponding lint warnings + + [3.20.0] - 2025-12-08 --------------------- diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..de0f186 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,148 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +`dtoolcore` is the dependency-free Python core API for creating and reading +**dtool datasets** — packages of data plus structural and descriptive metadata +for managing (scientific) data. It is a library only (no CLI here; the CLI lives +in separate `dtool` packages). Storage backends (disk, iRODS, Azure, S3, …) are +pluggable via setuptools entry points; this repo ships only the disk backend. + +## Commands + +```bash +# Run the full test suite (pytest config + coverage in pyproject.toml) +pytest + +# Run a single test file / test +pytest tests/test_copy_dataset.py +pytest tests/test_copy_dataset.py::test_copy + +# Lint +ruff check . +ruff check . --fix # auto-fix + +# Run tests across all supported environments (py39..py314, lint) +tox +tox -e lint # lint only + +# Install for development +pip install -e ".[test,lint]" + +# Build sdist + wheel (flit_scm backend) +python -m build +``` + +Tests live in `tests/`, are pytest-based, and rely on shared fixtures defined in +`tests/__init__.py` (`tmp_uri_fixture`, `tmp_dir_fixture`, `chdir_fixture`, +`tmp_env_var`, etc.). Most tests are "functional": they create real datasets in +temporary directories and read them back. + +## Architecture + +The codebase has two layers with a deliberate separation of concerns: + +1. **`dtoolcore/__init__.py`** — the storage-agnostic public API (dataset + lifecycle, metadata orchestration). It never touches the filesystem directly. +2. **`dtoolcore/storagebroker.py`** — storage backends. All I/O happens here, + behind the `BaseStorageBroker` interface. + +These communicate through two key abstractions: + +- **handle**: a Unix-style relative path identifying an item *within* a dataset. +- **identifier**: the SHA-1 hexdigest of a handle (`utils.generate_identifier`). + Identifiers are the canonical keys used throughout the manifest and overlays. + +### Dataset lifecycle: ProtoDataSet → DataSet + +A dataset is built as a **`ProtoDataSet`** (writable, `type: "protodataset"`), +then **frozen** into a read-only **`DataSet`** (`type: "dataset"`). Both subclass +`_BaseDataSet`. The `freeze()` method is the pivotal transition: it generates and +persists the manifest, converts accumulated per-item metadata fragments into +overlays, flips the `type` in admin metadata, and stamps `frozen_at`. + +`freeze_with_manifest(manifest, frozen_at=None)` is an alternative that freezes +using a **client-supplied** manifest (pre-computed hashes) instead of hashing +server-side. It validates that the README and every manifest item exist in +storage but trusts the provided hashes — intended for server applications. + +`DataSetCreator` (and `DerivedDataSetCreator`) are context managers wrapping the +normal flow: enter → work in a staging tempdir → exit auto-freezes (unless an +exception propagated). Prefer these for new code over manual proto/freeze +handling. + +Construct datasets from a URI via `DataSet.from_uri(uri)` / +`ProtoDataSet.from_uri(uri)`, which type-check the admin metadata. Top-level +helpers: `create_proto_dataset`, `create_derived_proto_dataset`, `copy`, +`copy_resume`, `iter_datasets_in_base_uri`, `iter_proto_datasets_in_base_uri`. + +### Metadata model + +Four distinct metadata categories — do not conflate them: + +- **admin metadata** (`.dtool/dtool`): uuid, name, type, creator, timestamps. +- **README** (`README.yml`): free-form descriptive metadata; client ensures + valid YAML. `update_readme` backs up the prior version with a timestamp suffix. +- **manifest** (`.dtool/manifest.json`): per-item structural metadata + (size, hash, mtime, relpath), keyed by identifier. Generated at freeze. +- **overlays** (`.dtool/overlays/*.json`): per-item key/value metadata, keyed by + identifier. On a frozen dataset, overlays must have exactly the dataset's + identifiers as keys. Pre-freeze, use `add_item_metadata(handle, key, value)`, + which writes fragment files that `freeze()` aggregates into overlays. +- **annotations** (`.dtool/annotations/*.json`) and **tags** (`.dtool/tags/`): + dataset-level (not per-item) metadata. + +### Storage brokers (pluggable backends) + +`_get_storage_broker(uri, config_path)` parses the URI scheme and looks up the +broker registered under the `dtool.storage_brokers` entry point group (see +`pyproject.toml`). The scheme (e.g. `file`) maps to a broker's `key` attribute. + +To add a backend, subclass `BaseStorageBroker` and implement its +`NotImplementedError` interface methods (`get_text`/`put_text`/`delete_key`, the +`get_*_key` methods, `iter_item_handles`, `get_size_in_bytes`/`get_utc_timestamp`/ +`get_hash`, `has_admin_metadata`, `pre_freeze_hook`/`post_freeze_hook`, etc.). +The base class provides reusable JSON serialization logic on top of the +text/key primitives — most higher-level methods (`get_manifest`, `put_overlay`, +…) are already implemented in terms of `get_text`/`put_text`. `DiskStorageBroker` +(`key = "file"`) is the reference implementation; its on-disk layout is defined +by `_STRUCTURE_PARAMETERS`. + +`pre_freeze_hook` is also a validation point: `DiskStorageBroker` rejects "rogue" +files in the dataset root that aren't part of the known structure. + +### Cross-platform paths & URIs (`utils.py`) + +URIs are central. `generous_parse_uri` turns relative paths into fully-qualified +`file://` URIs (and handles Windows drive letters); `sanitise_uri` normalizes any +input. Handles are always Unix-style; `handle_to_osrelpath` / `relpath_to_handle` +convert to/from OS-native paths. When editing path logic, preserve the +`IS_WINDOWS` branches — Windows support is a maintained feature. + +### Configuration (`utils.get_config_value`) + +Lookup precedence: **environment variable → JSON config file → default**. Config +file defaults to `~/.config/dtool/dtool.json`. Notable keys: `DTOOL_NUM_PROCESSES` +(parallelize manifest hashing for `file`/`symlink` brokers via multiprocessing) +and `DTOOL_STAGING_PREFIX` (staging tempdir location for `DataSetCreator`). + +## Conventions + +- **No runtime dependencies** — keep it that way; the dependency-free promise is + a core feature. Version discovery uses stdlib `importlib.metadata`. +- **Python 3.9+** only (`requires-python = ">=3.9"`). CI in + `.github/workflows/test.yml` tests 3.9–3.14. Note the one remaining version + branch: `entry_points().select(...)` (3.10+) vs `.get(...)` (3.9) in + `_generate_storage_broker_lookup`. +- **Build backend is `flit_scm`** (flit + setuptools_scm). The version is derived + from git tags and written to `dtoolcore/version.py` (generated, do not edit). + Releases are git tags; tagging triggers PyPI publish. +- **Timestamps**: `utils.timestamp()` returns seconds since the Unix epoch and + treats naive datetimes as UTC. Pass timezone-aware UTC datetimes from new code. +- Update `CHANGELOG.rst` (keep-a-changelog format, semantic versioning) for + user-facing changes. +- The library raises `DtoolCore*` exception subclasses (`DtoolCoreTypeError`, + `DtoolCoreKeyError`, `DtoolCoreValueError`, `DtoolCoreInvalidNameError`, + `DtoolCoreBrokenStagingPromise`) — use these rather than bare builtins. diff --git a/README.rst b/README.rst index d8629ed..b868447 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,7 @@ Features - Ability to create plugins for custom storage solutions - Plugins for iRODS and Microsoft Azure storage backends available - Cross-platform: Linux, Mac and Windows are all supported -- Works with Python 2.7, 3.5 and 3.6 +- Works with Python 3.9 and above - No external dependencies Overview diff --git a/dtoolcore/__init__.py b/dtoolcore/__init__.py index 69b1e94..c7641f5 100644 --- a/dtoolcore/__init__.py +++ b/dtoolcore/__init__.py @@ -2,38 +2,32 @@ """ -import os - import datetime import logging import multiprocessing as mp +import os import shutil import tempfile import uuid - from collections import defaultdict +from importlib.metadata import PackageNotFoundError, version import dtoolcore.utils logger = logging.getLogger(__name__) -# workaround for diverging python versions: -try: - from importlib.metadata import version, PackageNotFoundError - logger.debug("imported version, PackageNotFoundError from importlib.metadata") -except ModuleNotFoundError: - from importlib_metadata import version, PackageNotFoundError - logger.debug("imported version, PackageNotFoundError from importlib_metadata") - # first, try to determine dynamic version at runtime try: __version__ = version(__name__) - logger.debug("Determined version %s via importlib_metadata.version", __version__) + logger.debug("Determined version %s via importlib.metadata", __version__) except PackageNotFoundError: # if that fails, check for static version file written by setuptools_scm try: from .version import version as __version__ - logger.debug("Determined version %s from autogenerated dtoolcore/version.py", __version__) + logger.debug( + "Determined version %s from autogenerated dtoolcore/version.py", + __version__, + ) except Exception as e: logger.debug("All efforts to determine version failed: %s", e) __version__ = None @@ -117,12 +111,12 @@ def generate_admin_metadata(name, creator_username=None): logger.debug("In generate_admin_metadata...") if not dtoolcore.utils.name_is_valid(name): - raise(DtoolCoreInvalidNameError()) + raise DtoolCoreInvalidNameError() if creator_username is None: creator_username = dtoolcore.utils.getuser() - datetime_obj = datetime.datetime.utcnow() + datetime_obj = datetime.datetime.now(datetime.timezone.utc) admin_metadata = { "uuid": str(uuid.uuid4()), @@ -167,7 +161,7 @@ def generate_proto_dataset(admin_metadata, base_uri, config_path=None): logger.debug("In generate_proto_dataset...") name = admin_metadata.get("name") if name is None or not dtoolcore.utils.name_is_valid(name): - raise(DtoolCoreInvalidNameError()) + raise DtoolCoreInvalidNameError() uri = _generate_uri(admin_metadata, base_uri) return ProtoDataSet(uri, admin_metadata, config_path) @@ -419,7 +413,7 @@ class DtoolCoreBrokenStagingPromise(IOError): pass -class _BaseDataSet(object): +class _BaseDataSet: """Base class for datasets.""" def __init__(self, uri, admin_metadata, config_path=None): @@ -435,7 +429,7 @@ def __init__(self, uri, admin_metadata, config_path=None): def _from_uri_with_typecheck(cls, uri, config_path, type_name): # Make sure that the URI refers to a dataset. if not _is_dataset(uri, config_path): - raise(DtoolCoreTypeError("{} is not a dataset".format(uri))) + raise DtoolCoreTypeError("{} is not a dataset".format(uri)) # Get the admin metadata out of the URI and type check. admin_metadata = _admin_metadata_from_uri(uri, config_path) @@ -484,7 +478,7 @@ def update_name(self, new_name): )) if not dtoolcore.utils.name_is_valid(new_name): - raise(DtoolCoreInvalidNameError()) + raise DtoolCoreInvalidNameError() self._admin_metadata['name'] = new_name if self._storage_broker.has_admin_metadata(): @@ -513,7 +507,7 @@ def _put_overlay(self, overlay_name, overlay): logger.debug("Put readme content {}".format(self)) if not dtoolcore.utils.name_is_valid(overlay_name): - raise(DtoolCoreInvalidNameError()) + raise DtoolCoreInvalidNameError() if not isinstance(overlay, dict): raise DtoolCoreTypeError("Overlay must be dict") @@ -597,7 +591,7 @@ def get_annotation(self, annotation_name): :returns: annotation """ if annotation_name not in self.list_annotation_names(): - raise(DtoolCoreKeyError()) + raise DtoolCoreKeyError() return self._storage_broker.get_annotation(annotation_name) def put_annotation(self, annotation_name, annotation): @@ -610,7 +604,7 @@ def put_annotation(self, annotation_name, annotation): """ logger.debug("Put annotation {} {}".format(annotation_name, self)) if not dtoolcore.utils.name_is_valid(annotation_name): - raise(DtoolCoreInvalidNameError()) + raise DtoolCoreInvalidNameError() self._storage_broker.put_annotation(annotation_name, annotation) def list_annotation_names(self): @@ -635,10 +629,10 @@ def put_tag(self, tag): :raises: DtoolCoreValueError if the tag is not a string """ if not isinstance(tag, str): - raise(DtoolCoreValueError()) + raise DtoolCoreValueError() if not dtoolcore.utils.name_is_valid(tag): - raise(DtoolCoreInvalidNameError()) + raise DtoolCoreInvalidNameError() self._storage_broker.put_tag(tag) @@ -661,7 +655,7 @@ class DataSet(_BaseDataSet): """ def __init__(self, uri, admin_metadata, config_path=None): - super(DataSet, self).__init__(uri, admin_metadata, config_path) + super().__init__(uri, admin_metadata, config_path) self._manifest_cache = None def _identifiers(self): @@ -724,7 +718,7 @@ def get_overlay(self, overlay_name): """ logger.debug("Get overlay {} {}".format(overlay_name, self)) if overlay_name not in self.list_overlay_names(): - raise(DtoolCoreKeyError()) + raise DtoolCoreKeyError() return self._storage_broker.get_overlay(overlay_name) def put_overlay(self, overlay_name, overlay): @@ -855,7 +849,7 @@ def freeze(self, progressbar=None): # administrative metadata. It is present in the administrative metadata # if the dataset is being copied. if "frozen_at" not in self._admin_metadata: - datetime_obj = datetime.datetime.utcnow() + datetime_obj = datetime.datetime.now(datetime.timezone.utc) metadata_update["frozen_at"] = dtoolcore.utils.timestamp(datetime_obj) # NOQA # Apply the change(s) to the administrative metadata. @@ -917,7 +911,8 @@ def freeze_with_manifest(self, manifest, frozen_at=None): # Get identifiers of items that actually exist in storage existing_handles = set(self._storage_broker.iter_item_handles()) existing_identifiers = set( - dtoolcore.utils.generate_identifier(h) for h in existing_handles + dtoolcore.utils.generate_identifier(h) + for h in existing_handles ) # Check for missing items @@ -959,8 +954,10 @@ def freeze_with_manifest(self, manifest, frozen_at=None): if frozen_at is not None: metadata_update["frozen_at"] = frozen_at elif "frozen_at" not in self._admin_metadata: - datetime_obj = datetime.datetime.utcnow() - metadata_update["frozen_at"] = dtoolcore.utils.timestamp(datetime_obj) + datetime_obj = datetime.datetime.now(datetime.timezone.utc) + metadata_update["frozen_at"] = dtoolcore.utils.timestamp( + datetime_obj + ) # Apply the change(s) to the administrative metadata. self._admin_metadata.update(metadata_update) @@ -970,7 +967,7 @@ def freeze_with_manifest(self, manifest, frozen_at=None): self._storage_broker.post_freeze_hook() -class DataSetCreator(object): +class DataSetCreator: """Context manager for creating a dataset. Inside the context manager one works on a proto dataset. When exiting the @@ -1015,9 +1012,9 @@ def __exit__(self, exception_type, exception_value, traceback): # Add any staged files to the dataset. for abspath, relpath in self._to_stage: if not os.path.isfile(abspath): - raise(DtoolCoreBrokenStagingPromise( + raise DtoolCoreBrokenStagingPromise( "No such file: {}".format(abspath) - )) + ) self.proto_dataset.put_item(abspath, relpath) # If everything has been successful freeze the dataset. diff --git a/dtoolcore/filehasher.py b/dtoolcore/filehasher.py index dfc41d6..164c9e0 100644 --- a/dtoolcore/filehasher.py +++ b/dtoolcore/filehasher.py @@ -3,7 +3,7 @@ import hashlib -class FileHasher(object): +class FileHasher: """Class for associating hash functions with names.""" def __init__(self, hash_func): diff --git a/dtoolcore/storagebroker.py b/dtoolcore/storagebroker.py index 815efc5..4eda4c1 100644 --- a/dtoolcore/storagebroker.py +++ b/dtoolcore/storagebroker.py @@ -1,24 +1,24 @@ """Disk storage broker.""" -import os +import datetime import json -import shutil import logging -import datetime +import os +import shutil import socket from dtoolcore import __version__ +from dtoolcore.filehasher import FileHasher, md5sum_hexdigest from dtoolcore.utils import ( - mkdir_parents, + IS_WINDOWS, generate_identifier, generous_parse_uri, + handle_to_osrelpath, + mkdir_parents, timestamp, - IS_WINDOWS, - windows_to_unix_path, unix_to_windows_path, - handle_to_osrelpath, + windows_to_unix_path, ) -from dtoolcore.filehasher import FileHasher, md5sum_hexdigest logger = logging.getLogger(__name__) @@ -70,7 +70,7 @@ class DiskStorageBrokerValidationWarning(Warning): pass -class BaseStorageBroker(object): +class BaseStorageBroker: """Base storage broker class defining the required interface.""" # Class methods to override. @@ -78,62 +78,62 @@ class BaseStorageBroker(object): @classmethod def list_dataset_uris(cls, base_uri, config_path): """Return list containing URIs in location given by base_uri.""" - raise(NotImplementedError()) + raise NotImplementedError() @classmethod def generate_uri(cls, name, uuid, base_uri): """Return dataset URI.""" - raise(NotImplementedError()) + raise NotImplementedError() # Methods to override. def get_text(self, key): """Return the text associated with the key.""" - raise(NotImplementedError()) + raise NotImplementedError() def put_text(self, key, text): """Put the text into the storage associated with the key.""" - raise(NotImplementedError()) + raise NotImplementedError() def delete_key(self, key): """Delete the file/object associated with the key.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_admin_metadata_key(self): """Return the admin metadata key.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_readme_key(self): """Return the admin metadata key.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_manifest_key(self): """Return the manifest key.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_overlay_key(self, overlay_name): """Return the overlay key.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_annotation_key(self, annotation_name): """Return the annotation key.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_tag_key(self, tag): """Return the tag key.""" - raise(NotImplementedError()) + raise NotImplementedError() def list_overlay_names(self): """Return list of overlay names.""" - raise(NotImplementedError()) + raise NotImplementedError() def list_annotation_names(self): """Return list of annotation names.""" - raise(NotImplementedError()) + raise NotImplementedError() def list_tags(self): """Return list of tags.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_item_abspath(self, identifier): """Return absolute path at which item content can be accessed. @@ -141,11 +141,11 @@ def get_item_abspath(self, identifier): :param identifier: item identifier :returns: absolute path from which the item content can be accessed """ - raise(NotImplementedError()) + raise NotImplementedError() def _create_structure(self): """Create necessary structure to hold a dataset.""" - raise(NotImplementedError()) + raise NotImplementedError() def put_item(self, fpath, relpath): """Put item with content from fpath at relpath in dataset. @@ -157,30 +157,30 @@ def put_item(self, fpath, relpath): a handle :returns: the handle given to the item """ - raise(NotImplementedError()) + raise NotImplementedError() def iter_item_handles(self): """Return iterator over item handles.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_size_in_bytes(self, handle): """Return the size in bytes.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_utc_timestamp(self, handle): """Return the UTC timestamp.""" - raise(NotImplementedError()) + raise NotImplementedError() def get_hash(self, handle): """Return the hash.""" - raise(NotImplementedError()) + raise NotImplementedError() def has_admin_metadata(self): """Return True if the administrative metadata exists. This is the definition of being a "dataset". """ - raise(NotImplementedError()) + raise NotImplementedError() def add_item_metadata(self, handle, key, value): """Store the given key:value pair for the item associated with handle. @@ -190,7 +190,7 @@ def add_item_metadata(self, handle, key, value): :param key: metadata key :param value: metadata value """ - raise(NotImplementedError()) + raise NotImplementedError() def get_item_metadata(self, handle): """Return dictionary containing all metadata associated with handle. @@ -202,7 +202,7 @@ def get_item_metadata(self, handle): frozen :returns: dictionary containing item metadata """ - raise(NotImplementedError()) + raise NotImplementedError() def pre_freeze_hook(self): """Pre :meth:`dtoolcore.ProtoDataSet.freeze` actions. @@ -213,7 +213,7 @@ def pre_freeze_hook(self): It may be useful for remote storage backends to generate caches to remove repetitive time consuming calls """ - raise(NotImplementedError()) + raise NotImplementedError() def post_freeze_hook(self): """Post :meth:`dtoolcore.ProtoDataSet.freeze` cleanup actions. @@ -224,11 +224,11 @@ def post_freeze_hook(self): In the :class:`dtoolcore.storage_broker.DiskStorageBroker` it removes the temporary directory for storing item metadata fragment files. """ - raise(NotImplementedError()) + raise NotImplementedError() def _list_historical_readme_keys(self): """Return list of historical README.yml keys.""" - raise(NotImplementedError()) + raise NotImplementedError() # Reusable methods. @@ -349,7 +349,9 @@ def delete_annotation(self, annotation_name): :param annotation_name: annotation name """ - logger.debug("Deleting annotation: {} {}".format(annotation_name, self)) + logger.debug( + "Deleting annotation: {} {}".format(annotation_name, self) + ) key = self.get_annotation_key(annotation_name) self.delete_key(key) @@ -583,8 +585,8 @@ def get_size_in_bytes(self, handle): def get_utc_timestamp(self, handle): """Return the UTC timestamp.""" fpath = self._fpath_from_handle(handle) - datetime_obj = datetime.datetime.utcfromtimestamp( - os.stat(fpath).st_mtime + datetime_obj = datetime.datetime.fromtimestamp( + os.stat(fpath).st_mtime, datetime.timezone.utc ) return timestamp(datetime_obj) @@ -646,15 +648,15 @@ def _create_structure(self): # Ensure that the specified path does not exist and create it. if os.path.exists(self._abspath): - raise(StorageBrokerOSError( + raise StorageBrokerOSError( "Path already exists: {}".format(self._abspath) - )) + ) # Make sure the parent directory exists. parent, _ = os.path.split(self._abspath) if not os.path.isdir(parent): - raise(StorageBrokerOSError( - "No such directory: {}".format(parent))) + raise StorageBrokerOSError( + "No such directory: {}".format(parent)) os.mkdir(self._abspath) @@ -761,7 +763,7 @@ def pre_freeze_hook(self): for d in os.listdir(self._abspath): if d not in allowed: msg = "Rogue content in base of dataset: {}".format(d) - raise(DiskStorageBrokerValidationWarning(msg)) + raise DiskStorageBrokerValidationWarning(msg) def post_freeze_hook(self): """Post :meth:`dtoolcore.ProtoDataSet.freeze` cleanup actions. diff --git a/dtoolcore/utils.py b/dtoolcore/utils.py index b8008f4..10c116b 100644 --- a/dtoolcore/utils.py +++ b/dtoolcore/utils.py @@ -1,22 +1,19 @@ """Utility functions for dtoolcore.""" -import os +import base64 +import binascii +import datetime import errno import getpass import hashlib import json +import logging +import os import platform -import binascii -import base64 -import datetime import re import socket -import logging - -try: - from urlparse import urlparse, urlunparse -except ImportError: - from urllib.parse import urlparse, urlunparse +import stat +from urllib.parse import urlparse, urlunparse logger = logging.getLogger(__name__) @@ -161,7 +158,7 @@ def write_config_value_to_file(key, value, config_path=None): json.dump(config, fh, sort_keys=True, indent=2) # Set 600 permissions on the config file. - os.chmod(config_path, 33216) + os.chmod(config_path, stat.S_IRUSR | stat.S_IWUSR) return get_config_value_from_file(key, config_path) @@ -247,8 +244,12 @@ def timestamp(datetime_obj): """Return Unix timestamp as float. The number of seconds that have elapsed since January 1, 1970. + + Naive datetimes are assumed to be in UTC. """ - start_of_time = datetime.datetime(1970, 1, 1) + start_of_time = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + if datetime_obj.tzinfo is None: + datetime_obj = datetime_obj.replace(tzinfo=datetime.timezone.utc) diff = datetime_obj - start_of_time return diff.total_seconds() diff --git a/pyproject.toml b/pyproject.toml index b0281df..271039b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,9 @@ test = [ "pytest-cov", "pytz" ] +lint = [ + "ruff" +] docs = [ "sphinx", "sphinx_rtd_theme" @@ -45,5 +48,10 @@ write_to = "dtoolcore/version.py" testpaths = ["tests"] addopts = "--cov=dtoolcore --cov-report=term-missing" -[tool.flake8] -exclude = ["env*", ".tox", ".git", "*.egg", "build", "docs", "venv"] +[tool.ruff] +line-length = 79 +extend-exclude = ["dtoolcore/version.py", "docs", "venv*"] + +[tool.ruff.lint] +# pycodestyle (E, W), pyflakes (F), isort (I) +select = ["E", "W", "F", "I"] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index d6e1198..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e . diff --git a/tests/__init__.py b/tests/__init__.py index e2b3af8..71c0753 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -9,9 +9,9 @@ from dtoolcore.utils import ( IS_WINDOWS, - windows_to_unix_path, - unix_to_windows_path, generous_parse_uri, + unix_to_windows_path, + windows_to_unix_path, ) _HERE = os.path.dirname(__file__) diff --git a/tests/test_DiskStorageBroker.py b/tests/test_DiskStorageBroker.py index 76e3fa1..701d8b9 100644 --- a/tests/test_DiskStorageBroker.py +++ b/tests/test_DiskStorageBroker.py @@ -1,15 +1,17 @@ """Tests for disk storage broker.""" -import os -import pytz import datetime +import os import time import pytest +import pytz -from . import tmp_dir_fixture # NOQA -from . import tmp_uri_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + tmp_dir_fixture, # NOQA + tmp_uri_fixture, # NOQA +) def test_initialise(): @@ -22,8 +24,7 @@ def test_initialise(): def test_create_structure(tmp_dir_fixture): # NOQA - from dtoolcore.storagebroker import DiskStorageBroker - from dtoolcore.storagebroker import StorageBrokerOSError + from dtoolcore.storagebroker import DiskStorageBroker, StorageBrokerOSError storagebroker = DiskStorageBroker(tmp_dir_fixture) diff --git a/tests/test_abspath_uri.py b/tests/test_abspath_uri.py index 3c3adc0..6423851 100644 --- a/tests/test_abspath_uri.py +++ b/tests/test_abspath_uri.py @@ -3,9 +3,11 @@ import pytest -from . import TEST_SAMPLE_DATA -from . import tmp_dir_fixture # NOQA -from . import chdir_fixture # NOQA +from . import ( + TEST_SAMPLE_DATA, + chdir_fixture, # NOQA + tmp_dir_fixture, # NOQA +) def test_abspath_uri(): diff --git a/tests/test_annotation_functional.py b/tests/test_annotation_functional.py index 86f1d73..085c5e2 100644 --- a/tests/test_annotation_functional.py +++ b/tests/test_annotation_functional.py @@ -4,21 +4,22 @@ import pytest -from . import tmp_dir_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + tmp_dir_fixture, # NOQA +) def test_annotation_functional(tmp_dir_fixture): # NOQA from dtoolcore import ( DataSet, - ProtoDataSet, - DtoolCoreKeyError, DtoolCoreInvalidNameError, - generate_admin_metadata, + DtoolCoreKeyError, + ProtoDataSet, copy, + generate_admin_metadata, ) - from dtoolcore.storagebroker import DiskStorageBroker name = "my_dataset" diff --git a/tests/test_audit_fixes.py b/tests/test_audit_fixes.py index 253afb0..3a76eca 100644 --- a/tests/test_audit_fixes.py +++ b/tests/test_audit_fixes.py @@ -11,14 +11,14 @@ import pytest -from . import tmp_dir_fixture # NOQA - from dtoolcore.utils import ( IS_WINDOWS, generous_parse_uri, windows_to_unix_path, ) +from . import tmp_dir_fixture # NOQA + def _sanitise_base_uri(tmp_dir): base_uri = tmp_dir @@ -79,8 +79,6 @@ def test_generate_proto_dataset_accepts_valid_name(tmp_dir_fixture): # NOQA def test_copy_resume_missing_frozen_at_gives_helpful_error(tmp_dir_fixture): # NOQA import dtoolcore - base_uri = _sanitise_base_uri(tmp_dir_fixture) - src_dir = os.path.join(tmp_dir_fixture, "src") dest_dir = os.path.join(tmp_dir_fixture, "dest") os.mkdir(src_dir) diff --git a/tests/test_compare_module.py b/tests/test_compare_module.py index 9dbd171..595fb35 100644 --- a/tests/test_compare_module.py +++ b/tests/test_compare_module.py @@ -2,8 +2,10 @@ import os -from . import uri_to_path -from . import tmp_uri_fixture # NOQA +from . import ( + tmp_uri_fixture, # NOQA + uri_to_path, +) def create_test_files(uri): @@ -23,8 +25,8 @@ def test_diff_identifiers(tmp_uri_fixture): # NOQA generate_admin_metadata, generate_proto_dataset, ) - from dtoolcore.utils import generate_identifier from dtoolcore.compare import diff_identifiers + from dtoolcore.utils import generate_identifier fpaths = create_test_files(tmp_uri_fixture) @@ -63,8 +65,8 @@ def test_diff_sizes(tmp_uri_fixture): # NOQA generate_admin_metadata, generate_proto_dataset, ) - from dtoolcore.utils import generate_identifier from dtoolcore.compare import diff_sizes + from dtoolcore.utils import generate_identifier fpaths = create_test_files(tmp_uri_fixture) @@ -102,9 +104,9 @@ def test_diff_content(tmp_uri_fixture): # NOQA generate_admin_metadata, generate_proto_dataset, ) - from dtoolcore.utils import generate_identifier from dtoolcore.compare import diff_content from dtoolcore.storagebroker import DiskStorageBroker + from dtoolcore.utils import generate_identifier fpaths = create_test_files(tmp_uri_fixture) diff --git a/tests/test_copy_dataset.py b/tests/test_copy_dataset.py index d5d13d1..5982d3c 100644 --- a/tests/test_copy_dataset.py +++ b/tests/test_copy_dataset.py @@ -4,9 +4,11 @@ import pytest -from . import uri_to_path -from . import tmp_uri_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + tmp_uri_fixture, # NOQA + uri_to_path, +) def test_copy(tmp_uri_fixture): # NOQA @@ -76,7 +78,8 @@ def test_copy(tmp_uri_fixture): # NOQA assert src_ds.list_annotation_names() == dest_ds.list_annotation_names() for annotation_name in dest_ds.list_annotation_names(): - assert src_ds.get_annotation(annotation_name) == dest_ds.get_annotation(annotation_name) + assert src_ds.get_annotation(annotation_name) \ + == dest_ds.get_annotation(annotation_name) def test_copy_resume(tmp_uri_fixture): # NOQA diff --git a/tests/test_create_dataset_helpers_functional.py b/tests/test_create_dataset_helpers_functional.py index c25f792..e252a2e 100644 --- a/tests/test_create_dataset_helpers_functional.py +++ b/tests/test_create_dataset_helpers_functional.py @@ -4,15 +4,17 @@ import pytest -from . import tmp_dir_fixture # NOQA -from . import TEST_SAMPLE_DATA - from dtoolcore.utils import ( IS_WINDOWS, generous_parse_uri, windows_to_unix_path, ) +from . import ( + TEST_SAMPLE_DATA, + tmp_dir_fixture, # NOQA +) + def _sanitise_base_uri(tmp_dir): base_uri = tmp_dir diff --git a/tests/test_creation_and_reading_functional.py b/tests/test_creation_and_reading_functional.py index 2c7b6bc..3f059a4 100644 --- a/tests/test_creation_and_reading_functional.py +++ b/tests/test_creation_and_reading_functional.py @@ -1,23 +1,28 @@ """Functional tests for creation and reading of a disk based DataSet.""" -import os import datetime +import os import time -import pytz import pytest +import pytz -from . import tmp_dir_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + tmp_dir_fixture, # NOQA +) def test_basic_workflow(tmp_dir_fixture): # NOQA - from dtoolcore import ProtoDataSet, generate_admin_metadata - from dtoolcore import DataSet - from dtoolcore import __version__ - from dtoolcore.utils import generate_identifier + from dtoolcore import ( + DataSet, + ProtoDataSet, + __version__, + generate_admin_metadata, + ) from dtoolcore.storagebroker import DiskStorageBroker + from dtoolcore.utils import generate_identifier name = "my_dataset" admin_metadata = generate_admin_metadata(name) @@ -59,13 +64,13 @@ def test_basic_workflow(tmp_dir_fixture): # NOQA def test_proto_dataset_freeze_functional(tmp_dir_fixture): # NOQA from dtoolcore import ( - generate_admin_metadata, DataSet, + DtoolCoreTypeError, ProtoDataSet, - DtoolCoreTypeError + generate_admin_metadata, ) - from dtoolcore.utils import generate_identifier from dtoolcore.storagebroker import DiskStorageBroker + from dtoolcore.utils import generate_identifier name = "func_test_dataset_freeze" admin_metadata = generate_admin_metadata(name) @@ -241,10 +246,9 @@ def test_creation_and_reading(tmp_dir_fixture): # NOQA def test_basic_workflow_with_nested_handle(tmp_dir_fixture): # NOQA - from dtoolcore import ProtoDataSet, generate_admin_metadata - from dtoolcore import DataSet - from dtoolcore.utils import generate_identifier + from dtoolcore import DataSet, ProtoDataSet, generate_admin_metadata from dtoolcore.storagebroker import DiskStorageBroker + from dtoolcore.utils import generate_identifier name = "my_dataset" admin_metadata = generate_admin_metadata(name) diff --git a/tests/test_dtool_list_overlays_when_dir_missing.py b/tests/test_dtool_list_overlays_when_dir_missing.py index 02e071d..97dc3ac 100644 --- a/tests/test_dtool_list_overlays_when_dir_missing.py +++ b/tests/test_dtool_list_overlays_when_dir_missing.py @@ -1,7 +1,9 @@ import os -from . import chdir_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + chdir_fixture, # NOQA +) def test_list_overlays_when_dir_missing(chdir_fixture): # NOQA @@ -14,8 +16,7 @@ def test_list_overlays_when_dir_missing(chdir_fixture): # NOQA https://github.com/jic-dtool/dtoolcore/issues/3 """ - from dtoolcore import ProtoDataSet, generate_admin_metadata - from dtoolcore import DataSet + from dtoolcore import DataSet, ProtoDataSet, generate_admin_metadata from dtoolcore.storagebroker import DiskStorageBroker name = "my_dataset" diff --git a/tests/test_filehasher_api.py b/tests/test_filehasher_api.py index 8264d26..e981338 100644 --- a/tests/test_filehasher_api.py +++ b/tests/test_filehasher_api.py @@ -22,8 +22,9 @@ def test_md5sum_hexdigest(): def test_md5sum_digest(): - from dtoolcore.filehasher import md5sum_digest from base64 import b64encode + + from dtoolcore.filehasher import md5sum_digest expected = "3HMZLS+B1wCc5aHue61XVQ==" test_file = os.path.join(TEST_SAMPLE_DATA, 'tiny.png') actual = b64encode(md5sum_digest(test_file)).decode("utf-8") diff --git a/tests/test_freeze_with_manifest.py b/tests/test_freeze_with_manifest.py index db35550..948ffa2 100644 --- a/tests/test_freeze_with_manifest.py +++ b/tests/test_freeze_with_manifest.py @@ -1,19 +1,18 @@ """Test the freeze_with_manifest method of ProtoDataSet.""" import os -import uuid as uuid_module import pytest -from . import tmp_dir_fixture # NOQA - from dtoolcore.utils import ( IS_WINDOWS, + generate_identifier, generous_parse_uri, windows_to_unix_path, - generate_identifier, ) +from . import tmp_dir_fixture # NOQA + def _sanitise_base_uri(tmp_dir): base_uri = tmp_dir @@ -61,9 +60,9 @@ def test_freeze_with_manifest_basic(tmp_dir_fixture): # NOQA def test_freeze_with_manifest_with_items(tmp_dir_fixture): # NOQA """Test freezing with manifest containing items.""" - import dtoolcore import tempfile - import os + + import dtoolcore base_uri = _sanitise_base_uri(tmp_dir_fixture) name = "test-items" @@ -100,7 +99,9 @@ def test_freeze_with_manifest_with_items(tmp_dir_fixture): # NOQA # Add items to storage temp_files = [] for relpath in ["data/file1.txt", "data/file2.csv"]: - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f: + with tempfile.NamedTemporaryFile( + mode='w', delete=False, suffix='.txt' + ) as f: f.write(f"content for {relpath}") temp_files.append((f.name, relpath)) @@ -127,9 +128,10 @@ def test_freeze_with_manifest_with_items(tmp_dir_fixture): # NOQA def test_freeze_with_manifest_auto_frozen_at(tmp_dir_fixture): # NOQA """Test that frozen_at is auto-generated if not provided.""" - import dtoolcore import time + import dtoolcore + base_uri = _sanitise_base_uri(tmp_dir_fixture) name = "test-auto-frozen-at" @@ -250,9 +252,9 @@ def test_freeze_with_manifest_with_annotations(tmp_dir_fixture): # NOQA def test_freeze_with_manifest_full(tmp_dir_fixture): # NOQA """Test freezing with all features combined.""" - import dtoolcore import tempfile - import os + + import dtoolcore base_uri = _sanitise_base_uri(tmp_dir_fixture) name = "full-test-dataset" @@ -293,7 +295,9 @@ def test_freeze_with_manifest_full(tmp_dir_fixture): # NOQA proto_dataset.put_annotation(ann_name, ann_value) # Add item to storage - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as f: + with tempfile.NamedTemporaryFile( + mode='w', delete=False, suffix='.json' + ) as f: f.write('{"results": "test data"}') temp_path = f.name @@ -393,7 +397,6 @@ def test_dataset_type_after_freeze(tmp_dir_fixture): # NOQA def test_freeze_with_manifest_missing_readme(tmp_dir_fixture): # NOQA """Test that freezing fails if README is missing.""" import dtoolcore - import os base_uri = _sanitise_base_uri(tmp_dir_fixture) name = "test-missing-readme" @@ -468,9 +471,9 @@ def test_freeze_with_manifest_missing_items(tmp_dir_fixture): # NOQA def test_freeze_with_manifest_partial_items(tmp_dir_fixture): # NOQA """Test that freezing fails if some manifest items are missing.""" - import dtoolcore import tempfile - import os + + import dtoolcore base_uri = _sanitise_base_uri(tmp_dir_fixture) name = "test-partial-items" @@ -504,7 +507,9 @@ def test_freeze_with_manifest_partial_items(tmp_dir_fixture): # NOQA ) # Create a temporary file to add to the dataset - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f: + with tempfile.NamedTemporaryFile( + mode='w', delete=False, suffix='.txt' + ) as f: f.write("test content") temp_path = f.name @@ -525,9 +530,9 @@ def test_freeze_with_manifest_partial_items(tmp_dir_fixture): # NOQA def test_freeze_with_manifest_items_exist(tmp_dir_fixture): # NOQA """Test that freezing succeeds when all items exist.""" - import dtoolcore import tempfile - import os + + import dtoolcore base_uri = _sanitise_base_uri(tmp_dir_fixture) name = "test-items-exist" @@ -563,7 +568,9 @@ def test_freeze_with_manifest_items_exist(tmp_dir_fixture): # NOQA # Create temporary files and add them to the dataset temp_files = [] for relpath in ["data/file1.txt", "data/file2.txt"]: - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as f: + with tempfile.NamedTemporaryFile( + mode='w', delete=False, suffix='.txt' + ) as f: f.write(f"content for {relpath}") temp_files.append((f.name, relpath)) diff --git a/tests/test_frozen_at_when_copying_dataset.py b/tests/test_frozen_at_when_copying_dataset.py index caf453c..fbda614 100644 --- a/tests/test_frozen_at_when_copying_dataset.py +++ b/tests/test_frozen_at_when_copying_dataset.py @@ -5,9 +5,10 @@ import os import time +import dtoolcore as dc + from . import tmp_dir_fixture # NOQA -import dtoolcore as dc def test_frozen_at_value_when_copying_dataset(tmp_dir_fixture): # NOQA diff --git a/tests/test_generate_admin_metadata.py b/tests/test_generate_admin_metadata.py index fe6c26a..f6d3a2b 100644 --- a/tests/test_generate_admin_metadata.py +++ b/tests/test_generate_admin_metadata.py @@ -1,9 +1,9 @@ """Test the generate_admin_metadata helper function.""" import datetime -import pytz import pytest +import pytz def test_generate_admin_metadata(): @@ -28,8 +28,7 @@ def test_generate_admin_metadata(): def test_generate_admin_metadata_raises_invalid_name_error(): - from dtoolcore import generate_admin_metadata - from dtoolcore import DtoolCoreInvalidNameError + from dtoolcore import DtoolCoreInvalidNameError, generate_admin_metadata with pytest.raises(DtoolCoreInvalidNameError): generate_admin_metadata("/root/bad-name") diff --git a/tests/test_iter_datasets_in_base_uri.py b/tests/test_iter_datasets_in_base_uri.py index 95912bb..c2ea698 100644 --- a/tests/test_iter_datasets_in_base_uri.py +++ b/tests/test_iter_datasets_in_base_uri.py @@ -6,9 +6,9 @@ def test_iter_datasets_in_base_uri(tmp_uri_fixture): # NOQA from dtoolcore import ( - create_proto_dataset, DataSet, ProtoDataSet, + create_proto_dataset, iter_datasets_in_base_uri, iter_proto_datasets_in_base_uri, ) diff --git a/tests/test_overlays_functional.py b/tests/test_overlays_functional.py index efa10ee..56481ab 100644 --- a/tests/test_overlays_functional.py +++ b/tests/test_overlays_functional.py @@ -4,25 +4,26 @@ import pytest -from . import tmp_dir_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + tmp_dir_fixture, # NOQA +) def test_overlays_functional(tmp_dir_fixture): # NOQA from dtoolcore import ( DataSet, - ProtoDataSet, + DtoolCoreInvalidNameError, DtoolCoreKeyError, DtoolCoreTypeError, DtoolCoreValueError, - DtoolCoreInvalidNameError, - generate_admin_metadata, + ProtoDataSet, copy, + generate_admin_metadata, ) - - from dtoolcore.utils import generate_identifier from dtoolcore.storagebroker import DiskStorageBroker + from dtoolcore.utils import generate_identifier name = "my_dataset" admin_metadata = generate_admin_metadata(name) diff --git a/tests/test_self_description.py b/tests/test_self_description.py index b4440e1..0ea705b 100644 --- a/tests/test_self_description.py +++ b/tests/test_self_description.py @@ -1,12 +1,14 @@ """Test the DiskStorageBroker self description metadata.""" -import os import json +import os from dtoolcore import __version__ -from . import uri_to_path -from . import tmp_uri_fixture # NOQA +from . import ( + tmp_uri_fixture, # NOQA + uri_to_path, +) def test_writing_of_dtool_structure_file(tmp_uri_fixture): # NOQA diff --git a/tests/test_tags_functional.py b/tests/test_tags_functional.py index 74bdc1e..7be2510 100644 --- a/tests/test_tags_functional.py +++ b/tests/test_tags_functional.py @@ -7,7 +7,7 @@ def test_tags_functional(tmp_dir_fixture): # NOQA - from dtoolcore import DataSetCreator, DataSet + from dtoolcore import DataSet, DataSetCreator with DataSetCreator(name="empty-test-ds", base_uri=tmp_dir_fixture) as c: # Test put on proto dataset. diff --git a/tests/test_timestamp.py b/tests/test_timestamp.py index c3e9417..4ef24de 100644 --- a/tests/test_timestamp.py +++ b/tests/test_timestamp.py @@ -1,6 +1,6 @@ """Test timestamp functionality.""" -from datetime import datetime +from datetime import datetime, timezone def test_timestamp_returns_float(): @@ -33,7 +33,7 @@ def test_subsection_precision(): import dtoolcore.utils time_as_float = 946684800.513 - into_new_millenium = datetime.utcfromtimestamp(time_as_float) + into_new_millenium = datetime.fromtimestamp(time_as_float, timezone.utc) tolerance = 0.000001 actual = dtoolcore.utils.timestamp(into_new_millenium) diff --git a/tests/test_uri_property_when_using_relpath.py b/tests/test_uri_property_when_using_relpath.py index d59afdd..7079097 100644 --- a/tests/test_uri_property_when_using_relpath.py +++ b/tests/test_uri_property_when_using_relpath.py @@ -1,19 +1,16 @@ import os -from . import chdir_fixture # NOQA -from . import TEST_SAMPLE_DATA +from . import ( + TEST_SAMPLE_DATA, + chdir_fixture, # NOQA +) def test_uri_property_when_using_relpath(chdir_fixture): # NOQA - from dtoolcore import ProtoDataSet, generate_admin_metadata - from dtoolcore import DataSet + from dtoolcore import DataSet, ProtoDataSet, generate_admin_metadata from dtoolcore.storagebroker import DiskStorageBroker - from dtoolcore.utils import ( - IS_WINDOWS, - windows_to_unix_path, - urlparse - ) + from dtoolcore.utils import IS_WINDOWS, urlparse, windows_to_unix_path name = "my_dataset" admin_metadata = generate_admin_metadata(name) diff --git a/tests/test_uri_sanitisation.py b/tests/test_uri_sanitisation.py index ab908a1..48ddb3f 100644 --- a/tests/test_uri_sanitisation.py +++ b/tests/test_uri_sanitisation.py @@ -43,7 +43,7 @@ def test_generous_parse_uri(): def test_sanitise_uri(): - from dtoolcore.utils import sanitise_uri, IS_WINDOWS, windows_to_unix_path + from dtoolcore.utils import IS_WINDOWS, sanitise_uri, windows_to_unix_path relpath = "./my_data" diff --git a/tests/test_utils_module.py b/tests/test_utils_module.py index d4619a4..0625554 100644 --- a/tests/test_utils_module.py +++ b/tests/test_utils_module.py @@ -1,16 +1,15 @@ """Test the dtoolcore.utils module.""" +import json import os +import stat import sys -import json +from unittest.mock import MagicMock -try: - from unittest.mock import MagicMock -except ImportError: - from mock import MagicMock - -from . import tmp_dir_fixture # NOQA -from . import tmp_env_var +from . import ( + tmp_dir_fixture, # NOQA + tmp_env_var, +) def test_sha1_hexdigest(): @@ -29,32 +28,36 @@ def test_generate_identifier(): def test_cross_platform_getuser_windows_and_no_username_env_var(): - from dtoolcore.utils import cross_platform_getuser import getpass + + from dtoolcore.utils import cross_platform_getuser getpass.getuser = MagicMock(return_value="user1") assert cross_platform_getuser(True, True) == "unknown" getpass.getuser.assert_not_called() def test_cross_platform_getuser_windows_and_username_env_var(): - from dtoolcore.utils import cross_platform_getuser import getpass + + from dtoolcore.utils import cross_platform_getuser getpass.getuser = MagicMock(return_value="user1") assert cross_platform_getuser(True, False) == "user1" getpass.getuser.assert_called_once() def test_cross_platform_getuser_not_windows_and_username_env_var(): - from dtoolcore.utils import cross_platform_getuser import getpass + + from dtoolcore.utils import cross_platform_getuser getpass.getuser = MagicMock(return_value="user1") assert cross_platform_getuser(False, False) == "user1" getpass.getuser.assert_called_once() def test_cross_platform_getuser_not_windows_and_no_username_env_var(): - from dtoolcore.utils import cross_platform_getuser import getpass + + from dtoolcore.utils import cross_platform_getuser getpass.getuser = MagicMock(return_value="user1") assert cross_platform_getuser(False, True) == "user1" getpass.getuser.assert_called_once() @@ -108,7 +111,7 @@ def test_write_config_value_to_file(tmp_dir_fixture): # NOQA # Ensure that the file has 600 permissions. if sys.platform != "win32": # Don't do this check on windows - assert os.stat(config_path).st_mode == 33216 + assert stat.S_IMODE(os.stat(config_path).st_mode) == 0o600 def test_get_config_value(): @@ -206,7 +209,7 @@ def test_name_is_valid(): assert not name_is_valid("x" * 81) assert not name_is_valid("/root/this-is-a-bad-name") - assert not name_is_valid("th\is-is-a-bad-name") # NOQA + assert not name_is_valid(r"th\is-is-a-bad-name") # NOQA assert not name_is_valid("th\\is-is-a-bad-name") assert not name_is_valid("th\nis-is-a-bad-name") assert not name_is_valid("{this-is-a-bad-name}") diff --git a/tox.ini b/tox.ini index d6a253a..034406b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,10 @@ [tox] -envlist=py27,py3,flake8 +envlist = py39,py310,py311,py312,py313,py314,lint [testenv] -deps=pytest - pytest-cov - mock - pytest-mock - coverage - pytz - -r{toxinidir}/requirements.txt -commands=py.test +extras = test +commands = pytest {posargs} -[testenv:flake8] -deps=flake8 -commands=flake8 +[testenv:lint] +extras = lint +commands = ruff check .