diff --git a/src/deploy_tools/compare.py b/src/deploy_tools/compare.py index 61a0ce96..5123a066 100644 --- a/src/deploy_tools/compare.py +++ b/src/deploy_tools/compare.py @@ -87,10 +87,15 @@ def _reconstruct_deployment_config_from_modules(layout: Layout) -> Deployment: Note that the default versions will be different to those in initial configuration. """ releases = _collect_releases(layout) - default_versions = _collect_default_modulefile_versions(layout, list(releases)) - settings = DeploymentSettings(default_versions=default_versions) + deployment = Deployment(settings=DeploymentSettings(), releases=releases) - return Deployment(settings=settings, releases=releases) + # Only non-deprecated (live) modules are associated with a .version file + live_names = list(deployment.get_final_deployed_modules()) + deployment.settings.default_versions = _collect_default_modulefile_versions( + layout, live_names + ) + + return deployment def _collect_releases(layout: Layout) -> ReleasesByNameAndVersion: @@ -152,7 +157,12 @@ def _collect_default_modulefile_versions( default_versions: dict[str, str] = {} for name in names: - default_version = get_default_modulefile_version(name, layout) + try: + default_version = get_default_modulefile_version(name, layout) + except FileNotFoundError as exc: + raise ComparisonError( + f"Live module '{name}' has no default version file" + ) from exc if default_version is not None: default_versions[name] = default_version @@ -194,4 +204,6 @@ def _get_dict_diff(d1: dict[str, Any], d2: dict[str, Any]) -> str: def _yaml_dumps(obj: dict[str, Any], indent: int | None = None) -> str: ta = TypeAdapter(dict[str, Any]) - return yaml.safe_dump(ta.dump_python(obj), indent=indent) + # Dump in JSON mode so pydantic special types (e.g. AnyUrl) are coerced to YAML-safe + # primitives + return yaml.safe_dump(ta.dump_python(obj, mode="json"), indent=indent) diff --git a/src/deploy_tools/deploy.py b/src/deploy_tools/deploy.py index 939c13cd..9ea3c58b 100644 --- a/src/deploy_tools/deploy.py +++ b/src/deploy_tools/deploy.py @@ -1,5 +1,4 @@ import logging -import os import shutil from pathlib import Path @@ -69,7 +68,7 @@ def _deploy_new_releases(to_add: list[Release], layout: Layout) -> None: mf_link = layout.get_modulefile_link(name, version, from_deprecated=deprecated) mf_link.parent.mkdir(parents=True, exist_ok=True) - os.symlink(modulefile, mf_link) + mf_link.symlink_to(modulefile) def _deploy_releases( @@ -155,6 +154,10 @@ def _delete_modulefiles_name_folder( ) -> None: modulefiles_name_path = layout.get_modulefiles_root(from_deprecated) / name + # Handle several versions of the same module being processed together + if not modulefiles_name_path.exists(): + return + # Ignore the default version file when checking for existing modulefile links if next(modulefiles_name_path.glob(MODULE_VERSIONS_GLOB), None) is None: shutil.rmtree(modulefiles_name_path) diff --git a/src/deploy_tools/layout.py b/src/deploy_tools/layout.py index 3d8b98d7..e57a0300 100644 --- a/src/deploy_tools/layout.py +++ b/src/deploy_tools/layout.py @@ -62,7 +62,6 @@ class Layout: DEFAULT_VERSION_FILENAME = ".version" DEPLOYMENT_SNAPSHOT_FILENAME = "deployment.yaml" - PREVIOUS_DEPLOYMENT_SNAPSHOT_FILENAME = "previous-deployment.yaml" def __init__(self, deployment_root: Path, build_root: Path | None = None) -> None: self._root = deployment_root diff --git a/src/deploy_tools/models/save_and_load.py b/src/deploy_tools/models/save_and_load.py index 818cd969..6aabf80f 100644 --- a/src/deploy_tools/models/save_and_load.py +++ b/src/deploy_tools/models/save_and_load.py @@ -5,6 +5,7 @@ import yaml from pydantic import BaseModel +from pydantic import ValidationError as PydanticValidationError from ..errors import DeployToolsError from .deployment import ( @@ -67,8 +68,13 @@ def _load_release(path: Path) -> Release: if path.is_dir() or not path.suffix == YAML_FILE_SUFFIX: raise LoadError(f"Unexpected file in configuration directory:\n{path}") - with open(path) as f: - return load_from_yaml(Release, f) + try: + with open(path) as f: + return load_from_yaml(Release, f) + except (OSError, yaml.YAMLError, PydanticValidationError, TypeError) as exc: + # Include exc: under the top-level handler the traceback (and chained cause) is + # suppressed, so the failing field is only visible if surfaced in the message. + raise LoadError(f"Module configuration is invalid:\n{path}\n{exc}") from exc def _check_filepath_matches(version_path: Path, module: Module) -> None: @@ -76,9 +82,6 @@ def _check_filepath_matches(version_path: Path, module: Module) -> None: It should be the Module's name and version as ///.yaml """ - if version_path.is_dir() and version_path.suffix == YAML_FILE_SUFFIX: - raise LoadError(f"Module directory has incorrect suffix:\n{version_path}") - if not module.name == version_path.parent.name: raise LoadError( f"Module name {module.name} does not match path:\n{version_path}" diff --git a/src/deploy_tools/modulefile.py b/src/deploy_tools/modulefile.py index 5597de89..c8842408 100644 --- a/src/deploy_tools/modulefile.py +++ b/src/deploy_tools/modulefile.py @@ -60,6 +60,8 @@ def apply_default_versions( overwrite=True, ) else: + # Defensive: remove a stale default file for a deployed module that has no + # entry in the default map. This ought to be unreachable via the CLI default_version_file.unlink(missing_ok=True) diff --git a/src/deploy_tools/snapshot.py b/src/deploy_tools/snapshot.py index ad2c2540..134bced1 100644 --- a/src/deploy_tools/snapshot.py +++ b/src/deploy_tools/snapshot.py @@ -1,7 +1,9 @@ import io import logging -from git import Repo +import yaml +from git import BadName, Repo +from pydantic import ValidationError as PydanticValidationError from .errors import DeployToolsError from .layout import Layout @@ -54,14 +56,25 @@ def load_snapshot(layout: Layout, from_scratch: bool = False) -> Deployment: ) logger.debug("Loading snapshot: %s", layout.deployment_snapshot_path) - with open(layout.deployment_snapshot_path) as f: - return load_from_yaml(Deployment, f) + try: + with open(layout.deployment_snapshot_path) as f: + return load_from_yaml(Deployment, f) + except (OSError, yaml.YAMLError, PydanticValidationError, TypeError) as exc: + raise SnapshotError( + f"Deployment snapshot could not be read:\n{layout.deployment_snapshot_path}" + ) from exc def load_snapshot_from_ref(layout: Layout, ref: str) -> Deployment: """Load the deployment snapshot from the given git ref of the deployment area.""" logger.debug("Loading snapshot from ref: %s", ref) - repo = Repo(layout.deployment_root) - ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] - with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore - return load_from_yaml(Deployment, snapshot_f) + with Repo(layout.deployment_root) as repo: + try: + ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] + except (BadName, KeyError) as exc: + raise SnapshotError( + f"Deployment snapshot not found at git ref:\n{ref}" + ) from exc + + with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore + return load_from_yaml(Deployment, snapshot_f) diff --git a/src/deploy_tools/sync.py b/src/deploy_tools/sync.py index fb7d105a..b2dd2dd7 100644 --- a/src/deploy_tools/sync.py +++ b/src/deploy_tools/sync.py @@ -1,11 +1,12 @@ import logging from pathlib import Path -from git import Repo +from git import InvalidGitRepositoryError, Repo from .build import build, clean_build_area from .deploy import deploy_changes -from .layout import Layout +from .errors import DeployToolsError +from .layout import Layout, ModuleBuildLayout from .models.save_and_load import load_deployment from .snapshot import create_snapshot, load_snapshot from .templater import Templater, TemplateType @@ -15,7 +16,17 @@ logger = logging.getLogger(__name__) -IGNORE_DIRS = ["/build", "/modules/*/*/sif_files"] + +class SyncError(DeployToolsError): + """Raised when the deployment area is not in a state the sync process can use.""" + + +# Files not tracked in the deployment area's git repo (kept only as a change reference): +# the transient build area and the large Apptainer .sif images. +IGNORE_DIRS = [ + f"/{Layout.DEFAULT_BUILD_ROOT_NAME}", + f"/{Layout.MODULES_ROOT_NAME}/*/*/{ModuleBuildLayout.SIF_FILES_FOLDER}", +] def synchronise( @@ -45,18 +56,25 @@ def synchronise( logger.info("Creating git repository in deployment area") repo = _initialise_git_repo(layout.deployment_root, IGNORE_DIRS) else: - repo = Repo(layout.deployment_root) - - logger.info("Creating snapshot") - create_snapshot(deployment, layout) - - logger.info("Deploying changes") - deploy_changes(deployment_changes, layout) - - logger.info("Committing changes to git (for reference)") - repo.git.add("--all") - commit = repo.index.commit("Performed sync process") - logger.info("Commit SHA: %s", commit.hexsha) + try: + repo = Repo(layout.deployment_root) + except InvalidGitRepositoryError as exc: + raise SyncError( + f"Deployment area has a snapshot but is not a git repository " + f"(git metadata missing or corrupt):\n{layout.deployment_root}" + ) from exc + + with repo: + logger.info("Creating snapshot") + create_snapshot(deployment, layout) + + logger.info("Deploying changes") + deploy_changes(deployment_changes, layout) + + logger.info("Committing changes to git (for reference)") + repo.git.add("--all") + commit = repo.index.commit("Performed sync process") + logger.info("Commit SHA: %s", commit.hexsha) logger.info("Sync process finished") diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index 84bdd533..7883a72e 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -157,11 +157,11 @@ def _get_release_changes( return release_changes -def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None: +def _validate_added_modules(releases: list[Release], allow_all: bool) -> None: for release in releases: module = release.module if release.deprecated: - if not from_scratch: + if not allow_all: raise ValidationError( f"Module {module.name}/{module.version} cannot have deprecated " f"status on initial creation." diff --git a/tests/configs/README.md b/tests/configs/README.md new file mode 100644 index 00000000..eb592576 --- /dev/null +++ b/tests/configs/README.md @@ -0,0 +1,16 @@ +# Test configurations + +Sample `deploy-tools` configuration folders used by the test suite, grouped by purpose: + +- **`golden-master/`** — the `01-initial` … `06-removed` lifecycle sequence. These are + **cumulative and ordered**: each stage is synced on top of the previous one to model a + real deployment lifecycle (only `01-initial` uses `--from-scratch`). Driven by + `test_golden_master.py` and regenerated by `generate_samples.sh`. Do not reorder or make + independent. +- **`valid/`** — standalone, self-contained configs that should deploy/validate cleanly, + each serving one focused test (e.g. `minimal`, `empty`, `multi-version-active`). +- **`invalid/`** — configs that a command must reject; each maps to a test asserting the + specific error message. + +When adding a fixture, prefer reusing an existing one; otherwise add a single-purpose +folder under `valid/` or `invalid/`. Leave `golden-master/` to the lifecycle sequence. diff --git a/tests/configs/invalid/default-version-not-deployed/example-module-shell/1.0.yaml b/tests/configs/invalid/default-version-not-deployed/example-module-shell/1.0.yaml new file mode 100644 index 00000000..deda7448 --- /dev/null +++ b/tests/configs/invalid/default-version-not-deployed/example-module-shell/1.0.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Minimal shell-only module used by the validate tests + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/invalid/default-version-not-deployed/settings.yaml b/tests/configs/invalid/default-version-not-deployed/settings.yaml new file mode 100644 index 00000000..33cecd08 --- /dev/null +++ b/tests/configs/invalid/default-version-not-deployed/settings.yaml @@ -0,0 +1,4 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: + example-module-shell: "2.0" diff --git a/tests/configs/invalid/deprecated-on-creation/example-module-shell/1.0.yaml b/tests/configs/invalid/deprecated-on-creation/example-module-shell/1.0.yaml new file mode 100644 index 00000000..deda7448 --- /dev/null +++ b/tests/configs/invalid/deprecated-on-creation/example-module-shell/1.0.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Minimal shell-only module used by the validate tests + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/invalid/deprecated-on-creation/example-module-shell/2.0.yaml b/tests/configs/invalid/deprecated-on-creation/example-module-shell/2.0.yaml new file mode 100644 index 00000000..ec135cff --- /dev/null +++ b/tests/configs/invalid/deprecated-on-creation/example-module-shell/2.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "2.0" + description: Minimal shell-only module used by the validate tests + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello + +deprecated: true diff --git a/tests/configs/invalid/deprecated-on-creation/settings.yaml b/tests/configs/invalid/deprecated-on-creation/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/deprecated-on-creation/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/invalid-module-name/example-module-shell/1.0.yaml b/tests/configs/invalid/invalid-module-name/example-module-shell/1.0.yaml new file mode 100644 index 00000000..b4ea58ac --- /dev/null +++ b/tests/configs/invalid/invalid-module-name/example-module-shell/1.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +# The module name contains a dot, which MODULE_NAME_REGEX forbids; loading must fail +# with a clear LoadError that names the offending field. +module: + name: example.module.shell + version: "1.0" + description: Module with a name that violates the name constraint + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/invalid/invalid-module-name/settings.yaml b/tests/configs/invalid/invalid-module-name/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/invalid-module-name/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/invalid-shell-script/example-module-shell/1.0.yaml b/tests/configs/invalid/invalid-shell-script/example-module-shell/1.0.yaml new file mode 100644 index 00000000..8b749cd2 --- /dev/null +++ b/tests/configs/invalid/invalid-shell-script/example-module-shell/1.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Module whose shell script is not valid bash + + applications: + - app_type: shell + + name: test-shell-echo + # 'fi' with no matching 'if' is invalid bash, so `validate --test-build` must + # reject the generated entrypoint when it runs `bash -n` over it. + script: + - fi diff --git a/tests/configs/invalid/invalid-shell-script/settings.yaml b/tests/configs/invalid/invalid-shell-script/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/invalid-shell-script/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/modified-without-allow-updates/example-module-shell/1.0.yaml b/tests/configs/invalid/modified-without-allow-updates/example-module-shell/1.0.yaml new file mode 100644 index 00000000..7f3194e6 --- /dev/null +++ b/tests/configs/invalid/modified-without-allow-updates/example-module-shell/1.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Minimal shell-only module used by the validate tests + + applications: + - app_type: shell + + name: test-shell-echo + # Differs from the valid/minimal baseline (echo hello) so that deploying this over + # it is an in-place modification, rejected without allow_updates / a version bump. + script: + - echo modified diff --git a/tests/configs/invalid/modified-without-allow-updates/settings.yaml b/tests/configs/invalid/modified-without-allow-updates/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/modified-without-allow-updates/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/name-mismatch/settings.yaml b/tests/configs/invalid/name-mismatch/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/name-mismatch/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/name-mismatch/wrong-folder-name/1.0.yaml b/tests/configs/invalid/name-mismatch/wrong-folder-name/1.0.yaml new file mode 100644 index 00000000..c9cab129 --- /dev/null +++ b/tests/configs/invalid/name-mismatch/wrong-folder-name/1.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +# The module name does not match its parent folder (wrong-folder-name), which the +# loader requires to match. +module: + name: example-module-shell + version: "1.0" + description: Module whose name does not match its containing folder + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/invalid/no-eligible-default/example-module-shell/1.0.yaml b/tests/configs/invalid/no-eligible-default/example-module-shell/1.0.yaml new file mode 100644 index 00000000..77d40f52 --- /dev/null +++ b/tests/configs/invalid/no-eligible-default/example-module-shell/1.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Shell module excluded from defaults with no eligible fallback + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello + + exclude_from_defaults: true diff --git a/tests/configs/invalid/no-eligible-default/settings.yaml b/tests/configs/invalid/no-eligible-default/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/no-eligible-default/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/removed-without-deprecation/settings.yaml b/tests/configs/invalid/removed-without-deprecation/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/removed-without-deprecation/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/stray-file/example-module-shell/1.0.yaml b/tests/configs/invalid/stray-file/example-module-shell/1.0.yaml new file mode 100644 index 00000000..16c35a7b --- /dev/null +++ b/tests/configs/invalid/stray-file/example-module-shell/1.0.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Minimal shell-only module sat alongside a stray non-yaml file + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/invalid/stray-file/example-module-shell/README.txt b/tests/configs/invalid/stray-file/example-module-shell/README.txt new file mode 100644 index 00000000..40412955 --- /dev/null +++ b/tests/configs/invalid/stray-file/example-module-shell/README.txt @@ -0,0 +1,2 @@ +A stray non-YAML file in a module directory. The config loader treats every entry +under a module folder as a release file, so this must be rejected with a LoadError. diff --git a/tests/configs/invalid/stray-file/settings.yaml b/tests/configs/invalid/stray-file/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/stray-file/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/unknown-dependency/example-module-base/1.0.yaml b/tests/configs/invalid/unknown-dependency/example-module-base/1.0.yaml new file mode 100644 index 00000000..7e8c3115 --- /dev/null +++ b/tests/configs/invalid/unknown-dependency/example-module-base/1.0.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-base + version: "1.0" + description: Base module that the dependent module references + + applications: [] diff --git a/tests/configs/invalid/unknown-dependency/example-module-dependent/1.0.yaml b/tests/configs/invalid/unknown-dependency/example-module-dependent/1.0.yaml new file mode 100644 index 00000000..dc38a528 --- /dev/null +++ b/tests/configs/invalid/unknown-dependency/example-module-dependent/1.0.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-dependent + version: "1.0" + description: References a version of example-module-base that does not exist + + dependencies: + - name: example-module-base + version: "9.9" + + applications: [] diff --git a/tests/configs/invalid/unknown-dependency/settings.yaml b/tests/configs/invalid/unknown-dependency/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/unknown-dependency/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/invalid/version-mismatch/example-module-shell/2.0.yaml b/tests/configs/invalid/version-mismatch/example-module-shell/2.0.yaml new file mode 100644 index 00000000..8fd14619 --- /dev/null +++ b/tests/configs/invalid/version-mismatch/example-module-shell/2.0.yaml @@ -0,0 +1,15 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +# The module version does not match its filename (2.0.yaml), which the loader requires +# to match. +module: + name: example-module-shell + version: "1.0" + description: Module whose version does not match its filename + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/invalid/version-mismatch/settings.yaml b/tests/configs/invalid/version-mismatch/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/invalid/version-mismatch/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/valid/apptainer-pull/example-apptainer/1.0.yaml b/tests/configs/valid/apptainer-pull/example-apptainer/1.0.yaml new file mode 100644 index 00000000..58d70c18 --- /dev/null +++ b/tests/configs/valid/apptainer-pull/example-apptainer/1.0.yaml @@ -0,0 +1,19 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-apptainer + version: "1.0" + description: Minimal single-image module exercising a real apptainer pull + + applications: + - app_type: apptainer + + container: + path: docker://ghcr.io/apptainer/lolcow + version: latest + + entrypoints: + - name: cowsay-hello + command: cowsay + options: + command_args: Hello diff --git a/tests/configs/valid/apptainer-pull/settings.yaml b/tests/configs/valid/apptainer-pull/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/valid/apptainer-pull/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/valid/empty/settings.yaml b/tests/configs/valid/empty/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/valid/empty/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/valid/minimal/example-module-shell/1.0.yaml b/tests/configs/valid/minimal/example-module-shell/1.0.yaml new file mode 100644 index 00000000..deda7448 --- /dev/null +++ b/tests/configs/valid/minimal/example-module-shell/1.0.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-shell + version: "1.0" + description: Minimal shell-only module used by the validate tests + + applications: + - app_type: shell + + name: test-shell-echo + script: + - echo hello diff --git a/tests/configs/valid/minimal/settings.yaml b/tests/configs/valid/minimal/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/valid/minimal/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/valid/multi-version-active/example-module-multi/1.0.yaml b/tests/configs/valid/multi-version-active/example-module-multi/1.0.yaml new file mode 100644 index 00000000..d006a584 --- /dev/null +++ b/tests/configs/valid/multi-version-active/example-module-multi/1.0.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-multi + version: "1.0" + description: Module with two versions, used to exercise name-folder cleanup + + applications: [] diff --git a/tests/configs/valid/multi-version-active/example-module-multi/2.0.yaml b/tests/configs/valid/multi-version-active/example-module-multi/2.0.yaml new file mode 100644 index 00000000..c6cd894b --- /dev/null +++ b/tests/configs/valid/multi-version-active/example-module-multi/2.0.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-multi + version: "2.0" + description: Module with two versions, used to exercise name-folder cleanup + + applications: [] diff --git a/tests/configs/valid/multi-version-active/settings.yaml b/tests/configs/valid/multi-version-active/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/valid/multi-version-active/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/valid/multi-version-deprecated/example-module-multi/1.0.yaml b/tests/configs/valid/multi-version-deprecated/example-module-multi/1.0.yaml new file mode 100644 index 00000000..4452fbbc --- /dev/null +++ b/tests/configs/valid/multi-version-deprecated/example-module-multi/1.0.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-multi + version: "1.0" + description: Module with two versions, used to exercise name-folder cleanup + + applications: [] + +# Deprecating both versions together moves their modulefile links into the deprecated +# area and clears the now-spent live modulefiles name folder. +deprecated: true diff --git a/tests/configs/valid/multi-version-deprecated/example-module-multi/2.0.yaml b/tests/configs/valid/multi-version-deprecated/example-module-multi/2.0.yaml new file mode 100644 index 00000000..de2c0844 --- /dev/null +++ b/tests/configs/valid/multi-version-deprecated/example-module-multi/2.0.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-multi + version: "2.0" + description: Module with two versions, used to exercise name-folder cleanup + + applications: [] + +# Deprecating both versions together moves their modulefile links into the deprecated +# area and clears the now-spent live modulefiles name folder. +deprecated: true diff --git a/tests/configs/valid/multi-version-deprecated/settings.yaml b/tests/configs/valid/multi-version-deprecated/settings.yaml new file mode 100644 index 00000000..70f60017 --- /dev/null +++ b/tests/configs/valid/multi-version-deprecated/settings.yaml @@ -0,0 +1,3 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +default_versions: {} diff --git a/tests/configs/valid/multi-version-explicit-default/example-module-multi/1.0.yaml b/tests/configs/valid/multi-version-explicit-default/example-module-multi/1.0.yaml new file mode 100644 index 00000000..675a9eec --- /dev/null +++ b/tests/configs/valid/multi-version-explicit-default/example-module-multi/1.0.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-multi + version: "1.0" + description: Module with two versions, used to exercise explicit default selection + + applications: [] diff --git a/tests/configs/valid/multi-version-explicit-default/example-module-multi/2.0.yaml b/tests/configs/valid/multi-version-explicit-default/example-module-multi/2.0.yaml new file mode 100644 index 00000000..1335942e --- /dev/null +++ b/tests/configs/valid/multi-version-explicit-default/example-module-multi/2.0.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: example-module-multi + version: "2.0" + description: Module with two versions, used to exercise explicit default selection + + applications: [] diff --git a/tests/configs/valid/multi-version-explicit-default/settings.yaml b/tests/configs/valid/multi-version-explicit-default/settings.yaml new file mode 100644 index 00000000..f3215dbe --- /dev/null +++ b/tests/configs/valid/multi-version-explicit-default/settings.yaml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json + +# Pin the default to 1.0 even though 2.0 is present and would be auto-selected. +default_versions: + example-module-multi: "1.0" diff --git a/tests/test_apptainer_pull.py b/tests/test_apptainer_pull.py new file mode 100644 index 00000000..406d24ac --- /dev/null +++ b/tests/test_apptainer_pull.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from conftest import run_cli + + +def test_sync_builds_real_sif_file(configs: Path, tmp_path: Path) -> None: + # The golden-master test stubs the apptainer pull, so this is the only test that + # exercises a real one. It needs apptainer installed and network access to ghcr.io; + # a missing binary fails the test (via run_cli) rather than skipping it, by design. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "apptainer-pull") + + sif_files = list(tmp_path.glob("**/*.sif")) + assert sif_files, "sync did not produce a .sif file" + assert all(f.stat().st_size > 0 for f in sif_files), "produced .sif file is empty" diff --git a/tests/test_binary.py b/tests/test_binary.py new file mode 100644 index 00000000..1dd0ca19 --- /dev/null +++ b/tests/test_binary.py @@ -0,0 +1,100 @@ +import hashlib +from pathlib import Path + +import pytest +import yaml + +from conftest import run_cli +from deploy_tools.app_builder import AppBuilderError + +# A small payload served as a local file:// "download" source, so the binary builder's +# real urlretrieve and hashing run with no network access. +BINARY_CONTENT = b"#!/bin/sh\necho 'hello from a binary module'\n" + +DIGESTS = { + "sha256": hashlib.sha256(BINARY_CONTENT).hexdigest(), + "sha512": hashlib.sha512(BINARY_CONTENT).hexdigest(), + "md5": hashlib.md5(BINARY_CONTENT).hexdigest(), +} + + +def _write_binary_deployment( + tmp_path: Path, hash_type: str, hash_value: str +) -> tuple[Path, Path]: + """Build a deployment root and a single binary-module config under ``tmp_path``. + + Binary modules download from a URL and verify a hash, so the config must reference a + real file whose digest is known at test time. A committed config cannot carry such a + machine-specific ``file://`` path, so it is generated per test. + + Args: + tmp_path: The per-test scratch directory. + hash_type: The ``hash_type`` to record in the config. + hash_value: The hash to record in the config. + + Returns: + The ``(deployment_root, config_folder)`` pair to pass to ``sync``. + """ + source = tmp_path / "downloads" / "example-tool" + source.parent.mkdir() + source.write_bytes(BINARY_CONTENT) + + config_folder = tmp_path / "config" + module_dir = config_folder / "example-binary" + module_dir.mkdir(parents=True) + (config_folder / "settings.yaml").write_text("default_versions: {}\n") + + release = { + "module": { + "name": "example-binary", + "version": "1.0", + "description": "Single binary module for builder coverage", + "applications": [ + { + "app_type": "binary", + "name": "example-tool", + "url": source.as_uri(), + "hash": hash_value, + "hash_type": hash_type, + } + ], + } + } + (module_dir / "1.0.yaml").write_text(yaml.safe_dump(release)) + + deployment_root = tmp_path / "deployment" + deployment_root.mkdir() + return deployment_root, config_folder + + +@pytest.mark.parametrize("hash_type", ["sha256", "sha512", "md5"]) +def test_sync_builds_binary_with_hash_check(tmp_path: Path, hash_type: str) -> None: + # Each supported algorithm should validate the download and deploy an executable + # binary entrypoint with the original contents. + deployment_root, config_folder = _write_binary_deployment( + tmp_path, hash_type, DIGESTS[hash_type] + ) + run_cli("sync", "--from-scratch", deployment_root, config_folder) + + binary = deployment_root / "modules/example-binary/1.0/entrypoints/example-tool" + assert binary.read_bytes() == BINARY_CONTENT + assert binary.stat().st_mode & 0o111, "deployed binary is not executable" + + +def test_sync_builds_binary_without_hash_check(tmp_path: Path) -> None: + # hash_type "none" skips verification; the binary is still downloaded and deployed. + deployment_root, config_folder = _write_binary_deployment(tmp_path, "none", "") + run_cli("sync", "--from-scratch", deployment_root, config_folder) + + binary = deployment_root / "modules/example-binary/1.0/entrypoints/example-tool" + assert binary.read_bytes() == BINARY_CONTENT + + +def test_sync_rejects_binary_with_wrong_hash(tmp_path: Path) -> None: + # A digest that does not match the download must fail the build rather than deploy + # an unverified binary. + deployment_root, config_folder = _write_binary_deployment( + tmp_path, "sha256", "0" * 64 + ) + with pytest.raises(AppBuilderError, match="hash check failed"): + run_cli("sync", "--from-scratch", deployment_root, config_folder) diff --git a/tests/test_cli.py b/tests/test_cli.py index cdbd77b1..be6ca5e6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,6 @@ import subprocess import sys +from pathlib import Path from deploy_tools import __version__ @@ -7,3 +8,16 @@ def test_cli_version() -> None: cmd = [sys.executable, "-m", "deploy_tools", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ + + +def test_cli_reports_domain_error_without_traceback(tmp_path: Path) -> None: + # compare on an existing area with no snapshot raises SnapshotError, a + # DeployToolsError. Run the real entry point and confirm the message is presented as + # a clean 'Error: ...' on stderr with exit 1, and no traceback is dumped. + cmd = [sys.executable, "-m", "deploy_tools", "compare", str(tmp_path)] + result = subprocess.run(cmd, capture_output=True, text=True) + + assert result.returncode == 1 + assert "Error:" in result.stderr + assert "snapshot not found" in result.stderr + assert "Traceback" not in result.stderr diff --git a/tests/test_compare.py b/tests/test_compare.py new file mode 100644 index 00000000..490c7b8d --- /dev/null +++ b/tests/test_compare.py @@ -0,0 +1,178 @@ +from pathlib import Path +from shutil import rmtree + +import pytest + +from conftest import run_cli +from deploy_tools.compare import ComparisonError, compare_to_snapshot +from deploy_tools.layout import Layout +from deploy_tools.snapshot import SnapshotError + +# The minimal config deploys a single shell-only module. Tests that start from a clean +# deployment and then corrupt it in one specific way share these coordinates. +MODULE_NAME = "example-module-shell" +MODULE_VERSION = "1.0" + + +def _sync_minimal(area: Path, configs: Path) -> Layout: + """Deploy the minimal shell-only config into ``area`` and return its ``Layout``. + + Several tests below need a single, cleanly-deployed module that they then corrupt in + one specific way, so this captures the shared from-scratch sync. + + Args: + area: Empty deployment area to deploy into. + configs: The ``configs`` fixture: the directory holding sample configurations. + + Returns: + A ``Layout`` describing the freshly-deployed ``area``. + """ + run_cli("sync", "--from-scratch", area, configs / "valid" / "minimal") + return Layout(area) + + +def test_compare_accepts_clean_deployment(tmp_path: Path, configs: Path) -> None: + # A deployment area is self-consistent immediately after a sync, so compare should + # succeed and print nothing. + _sync_minimal(tmp_path, configs) + assert run_cli("compare", tmp_path) == "" + + +def test_compare_accepts_deprecated_modules(tmp_path: Path, configs: Path) -> None: + # Ensure compare runs successfully with deprecated modulefile links. + run_cli( + "sync", "--from-scratch", tmp_path, configs / "golden-master" / "04-deprecated" + ) + assert run_cli("compare", tmp_path) == "" + + +def test_compare_from_scratch_accepts_empty_area(tmp_path: Path) -> None: + # --from-scratch only checks that the area is an existing, empty directory. + assert run_cli("compare", "--from-scratch", tmp_path) == "" + + +def test_compare_from_scratch_rejects_non_empty_area(tmp_path: Path) -> None: + # Any pre-existing content means the area is not ready for a from-scratch deploy. + (tmp_path / "stray-file").touch() + with pytest.raises(ComparisonError, match="not empty"): + run_cli("compare", "--from-scratch", tmp_path) + + +def test_compare_rejects_missing_snapshot(tmp_path: Path) -> None: + # An existing area with no deployment.yaml has no snapshot to compare against. + with pytest.raises(SnapshotError, match="snapshot not found"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_module_without_modulefile( + tmp_path: Path, configs: Path +) -> None: + # Removing a module's modulefile link leaves a built module that is not exposed. + layout = _sync_minimal(tmp_path, configs) + layout.get_modulefile_link(MODULE_NAME, MODULE_VERSION).unlink() + with pytest.raises(ComparisonError, match="No modulefile found"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_modulefile_without_module( + tmp_path: Path, configs: Path +) -> None: + # Removing the built module leaves a dangling modulefile link + layout = _sync_minimal(tmp_path, configs) + rmtree(layout.get_module_folder(MODULE_NAME, MODULE_VERSION)) + with pytest.raises(ComparisonError, match="without corresponding built module"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_duplicate_modulefiles(tmp_path: Path, configs: Path) -> None: + # The same modulefile must not appear in both the live and deprecated areas. + layout = _sync_minimal(tmp_path, configs) + deprecated_link = layout.get_modulefile_link( + MODULE_NAME, MODULE_VERSION, from_deprecated=True + ) + deprecated_link.parent.mkdir(parents=True, exist_ok=True) + deprecated_link.touch() + with pytest.raises(ComparisonError, match="Duplicate modulefiles"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_release_mismatch(tmp_path: Path, configs: Path) -> None: + # Tamper with the snapshot's record of the module so it no longer matches the module + # configuration reconstructed from the deployment area. + layout = _sync_minimal(tmp_path, configs) + snapshot = layout.deployment_snapshot_path + contents = snapshot.read_text() + assert "Minimal shell-only module" in contents + snapshot.write_text( + contents.replace("Minimal shell-only module", "Tampered module") + ) + with pytest.raises(ComparisonError, match="release configuration do not match"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_default_version_mismatch( + tmp_path: Path, configs: Path +) -> None: + # Point the module's .version file at a different version than the snapshot expects. + layout = _sync_minimal(tmp_path, configs) + version_file = layout.get_default_version_file(MODULE_NAME) + version_file.write_text("#%Module1.0\nset ModulesVersion 9.9\n") + with pytest.raises(ComparisonError, match="default versions do not match"): + run_cli("compare", tmp_path) + + +def test_compare_use_ref_detects_drift(tmp_path: Path, configs: Path) -> None: + # sync commits a snapshot to the deployment area's git repo on every run. After two + # syncs the area matches its own (HEAD) snapshot, but not the previous commit's + # snapshot, which predates the module added by the second sync. + run_cli( + "sync", "--from-scratch", tmp_path, configs / "golden-master" / "01-initial" + ) + run_cli("sync", tmp_path, configs / "golden-master" / "02-added") + + assert run_cli("compare", tmp_path) == "" + assert run_cli("compare", "--use-ref", "HEAD", tmp_path) == "" + + with pytest.raises(ComparisonError, match="release configuration do not match"): + run_cli("compare", "--use-ref", "HEAD~1", tmp_path) + + +def test_compare_from_scratch_rejects_missing_root(tmp_path: Path) -> None: + # The CLI's argument validation rejects a non-existent path before the command runs, + # so exercise this guard by calling the function directly. + with pytest.raises(ComparisonError, match="does not exist"): + compare_to_snapshot(tmp_path / "does-not-exist", from_scratch=True) + + +def test_compare_rejects_missing_root(tmp_path: Path) -> None: + # As above, but for the snapshot-loading path used by a non from-scratch compare. + with pytest.raises(SnapshotError, match="does not exist"): + compare_to_snapshot(tmp_path / "does-not-exist") + + +def test_compare_rejects_missing_default_version_file( + tmp_path: Path, configs: Path +) -> None: + # A live module records its default version in a .version file; a missing one is + # corruption that compare must report cleanly, not as a raw FileNotFoundError. + layout = _sync_minimal(tmp_path, configs) + layout.get_default_version_file(MODULE_NAME).unlink() + with pytest.raises(ComparisonError, match="default version file"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_unreadable_snapshot(tmp_path: Path, configs: Path) -> None: + # A corrupt deployment.yaml (e.g. truncated by a failed write) must surface as a + # SnapshotError rather than a raw parser error. + layout = _sync_minimal(tmp_path, configs) + layout.deployment_snapshot_path.write_text("") + with pytest.raises(SnapshotError, match="could not be read"): + run_cli("compare", tmp_path) + + +def test_compare_rejects_unknown_git_ref(tmp_path: Path, configs: Path) -> None: + # An unresolvable --use-ref should fail with a clear SnapshotError, not a raw + # GitPython BadName error. + _sync_minimal(tmp_path, configs) + with pytest.raises(SnapshotError, match="git ref"): + run_cli("compare", "--use-ref", "no-such-ref", tmp_path) diff --git a/tests/test_default_versions.py b/tests/test_default_versions.py new file mode 100644 index 00000000..2025ce6e --- /dev/null +++ b/tests/test_default_versions.py @@ -0,0 +1,126 @@ +"""Unit tests for default-version selection. + +These call the function directly rather than through the CLI. Here that is to isolate +pure, non-trivial logic (e.g. natsort ordering of non-SemVer versions): a targeted unit +test states the outcome precisely and pins any regression to a line, rather than +surfacing it as an opaque golden-master diff. +""" + +from collections import defaultdict + +import pytest + +from deploy_tools.models.deployment import ( + Deployment, + DeploymentSettings, + ReleasesByNameAndVersion, +) +from deploy_tools.models.module import Module, Release +from deploy_tools.validate import ValidationError, validate_default_versions + + +def _release( + name: str, version: str, *, deprecated: bool = False, excluded: bool = False +) -> Release: + """Build a minimal Release with no applications, for selection tests.""" + return Release( + module=Module( + name=name, + version=version, + applications=[], + exclude_from_defaults=excluded, + ), + deprecated=deprecated, + ) + + +def _deployment( + *releases: Release, default_versions: dict[str, str] | None = None +) -> Deployment: + """Assemble a Deployment from loose Releases and optional explicit defaults.""" + by_name: ReleasesByNameAndVersion = defaultdict(dict) + for release in releases: + by_name[release.module.name][release.module.version] = release + + return Deployment( + settings=DeploymentSettings(default_versions=default_versions or {}), + releases=by_name, + ) + + +def test_auto_selects_latest_version_by_natsort() -> None: + # natsort, not string ordering: "10.0" must beat "2.0" (which sorts last lexically). + deployment = _deployment( + _release("mod", "1.0"), + _release("mod", "2.0"), + _release("mod", "10.0"), + ) + assert validate_default_versions(deployment) == {"mod": "10.0"} + + +def test_prerelease_sorts_before_final_release() -> None: + # The documented non-SemVer case: "1.2rc1" must come before "1.2", so the final + # release is chosen as default over its own release candidate. + deployment = _deployment( + _release("mod", "1.2rc1"), + _release("mod", "1.2"), + ) + assert validate_default_versions(deployment) == {"mod": "1.2"} + + +def test_excluded_versions_are_not_auto_selected() -> None: + # The highest version opts out of defaults, so the next-highest is chosen instead. + deployment = _deployment( + _release("mod", "1.0"), + _release("mod", "2.0", excluded=True), + ) + assert validate_default_versions(deployment) == {"mod": "1.0"} + + +def test_deprecated_versions_are_not_auto_selected() -> None: + # Deprecated releases are excluded from the deployed set, so a higher deprecated + # version must not become the default over a lower active one. + deployment = _deployment( + _release("mod", "1.0"), + _release("mod", "2.0", deprecated=True), + ) + assert validate_default_versions(deployment) == {"mod": "1.0"} + + +def test_explicit_default_is_preserved_over_auto_selection() -> None: + # An explicit default wins even when a higher version exists. + deployment = _deployment( + _release("mod", "1.0"), + _release("mod", "2.0"), + default_versions={"mod": "1.0"}, + ) + assert validate_default_versions(deployment) == {"mod": "1.0"} + + +def test_explicit_default_for_nonexistent_version_raises() -> None: + # Pointing the default at a version that will not be deployed is rejected. + deployment = _deployment( + _release("mod", "1.0"), + default_versions={"mod": "9.9"}, + ) + with pytest.raises(ValidationError, match="Unable to configure"): + validate_default_versions(deployment) + + +def test_explicit_default_for_deprecated_version_raises() -> None: + # A deprecated version is not in the deployed set, so it cannot be an explicit + # default either. + deployment = _deployment( + _release("mod", "1.0", deprecated=True), + default_versions={"mod": "1.0"}, + ) + with pytest.raises(ValidationError, match="Unable to configure"): + validate_default_versions(deployment) + + +def test_all_versions_excluded_raises() -> None: + # With every version opting out and no explicit default, there is no eligible + # version to make default. + deployment = _deployment(_release("mod", "1.0", excluded=True)) + with pytest.raises(ValidationError, match="exclude_from_defaults"): + validate_default_versions(deployment) diff --git a/tests/test_external_tools.py b/tests/test_external_tools.py new file mode 100644 index 00000000..89d079b2 --- /dev/null +++ b/tests/test_external_tools.py @@ -0,0 +1,17 @@ +import pytest + +from deploy_tools.external_tools import ExternalToolError, run_command + + +def test_run_command_reports_missing_tool() -> None: + # A required external tool that is not installed must surface as a clean + # ExternalToolError naming the tool, not a raw FileNotFoundError traceback. + with pytest.raises(ExternalToolError, match="deploy-tools-no-such-binary"): + run_command(["deploy-tools-no-such-binary"]) + + +def test_run_command_reports_failed_tool() -> None: + # With check=True, a non-zero exit is reported as an ExternalToolError rather than a + # raw CalledProcessError. + with pytest.raises(ExternalToolError, match="exit status"): + run_command(["bash", "-c", "exit 3"], check=True) diff --git a/tests/test_load.py b/tests/test_load.py new file mode 100644 index 00000000..c8f2d0bf --- /dev/null +++ b/tests/test_load.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import pytest + +from conftest import run_cli +from deploy_tools.models.save_and_load import LoadError + +# Configs whose on-disk layout is malformed: loading must fail with a clear LoadError +# before any validation logic runs. Each maps a folder under configs/invalid to a +# substring expected in the error. +MALFORMED_CONFIGS = [ + ("stray-file", "Unexpected file in configuration directory"), + ("name-mismatch", "Module name .* does not match path"), + ("version-mismatch", "Module version .* does not match path"), +] + + +@pytest.mark.parametrize("config_name, message", MALFORMED_CONFIGS) +def test_load_rejects_malformed_config_layout( + tmp_path: Path, configs: Path, config_name: str, message: str +) -> None: + with pytest.raises(LoadError, match=message): + run_cli( + "validate", "--from-scratch", tmp_path, configs / "invalid" / config_name + ) + + +def test_load_surfaces_invalid_field_in_error(tmp_path: Path, configs: Path) -> None: + # A config that fails model validation must raise a clean LoadError that names the + # offending field; the top-level handler hides the traceback, so the field detail is + # only visible if carried in the message. + with pytest.raises(LoadError) as exc_info: + run_cli( + "validate", + "--from-scratch", + tmp_path, + configs / "invalid" / "invalid-module-name", + ) + message = str(exc_info.value) + assert "Module configuration is invalid" in message + assert "module.name" in message diff --git a/tests/test_module_options.py b/tests/test_module_options.py new file mode 100644 index 00000000..477f4d6b --- /dev/null +++ b/tests/test_module_options.py @@ -0,0 +1,185 @@ +"""Tests for Module/app options that drive otherwise-untested template branches. + +These CLI-driven ``sync`` tests build a config from typed models, set each option to a +representative value, and assert the rendered output. +""" + +from pathlib import Path + +from conftest import run_cli +from deploy_tools.models.apptainer_app import ( + ApptainerApp, + ContainerImage, + Entrypoint, + EntrypointOptions, +) +from deploy_tools.models.deployment import DeploymentSettings +from deploy_tools.models.module import EnvVar, Module, ModuleDependency, Release +from deploy_tools.models.save_and_load import ( + DEPLOYMENT_SETTINGS, + YAML_FILE_SUFFIX, + save_as_yaml, +) +from deploy_tools.models.shell_app import ShellApp + + +def _sync(tmp_path: Path, *releases: Release) -> Path: + """Serialise a config of the given releases under ``tmp_path`` and ``sync`` it. + + Args: + tmp_path: The per-test scratch directory. + releases: The Releases to deploy from scratch. + + Returns: + The deployment root the modules were synced into. + """ + config_folder = tmp_path / "config" + save_as_yaml(DeploymentSettings(), config_folder / DEPLOYMENT_SETTINGS, True) + for release in releases: + module = release.module + save_as_yaml( + release, + config_folder / module.name / f"{module.version}{YAML_FILE_SUFFIX}", + True, + ) + + deployment_root = tmp_path / "deployment" + deployment_root.mkdir() + run_cli("sync", "--from-scratch", deployment_root, config_folder) + return deployment_root + + +def test_load_and_unload_scripts_render_tcl_blocks(tmp_path: Path) -> None: + release = Release( + module=Module( + name="example-scripted", + version="1.0", + description="Module exercising load/unload script hooks", + load_script=["set example_loaded 1"], + unload_script=["unset example_loaded"], + applications=[ + ShellApp(app_type="shell", name="example-cmd", script=["echo hello"]) + ], + ) + ) + deployment_root = _sync(tmp_path, release) + + modulefile = ( + deployment_root / "modules/example-scripted/1.0/modulefile" + ).read_text() + + # The unload-mode guard is produced only by the unload_script branch, so its + # presence confirms that branch rendered; the script lines confirm both blocks + # carry content. + assert "if { [module-info mode unload] } {" in modulefile + assert "set example_loaded 1" in modulefile + assert "unset example_loaded" in modulefile + + +def test_apptainer_options_merge_global_and_entrypoint( + tmp_path: Path, stub_apptainer_pull: None +) -> None: + # The Apptainer entrypoint combines global_options with per-entrypoint options + release = Release( + module=Module( + name="example-apptainer-opts", + version="1.0", + description="Module exercising apptainer entrypoint option merge", + applications=[ + ApptainerApp( + app_type="apptainer", + container=ContainerImage( + path="docker://ghcr.io/apptainer/lolcow", # type: ignore[arg-type] + version="latest", + ), + global_options=EntrypointOptions( + mounts=["/global/mount"], + host_binaries=["globalbin"], + apptainer_args="--global-arg", + command_args="--global-cmd", + ), + entrypoints=[ + Entrypoint( + name="example-cmd", + command="realcmd", + options=EntrypointOptions( + mounts=["/local/mount"], + host_binaries=["localbin"], + apptainer_args="--local-arg", + command_args="--local-cmd", + ), + ) + ], + ) + ], + ) + ) + deployment_root = _sync(tmp_path, release) + + entrypoint = ( + deployment_root / "modules/example-apptainer-opts/1.0/entrypoints/example-cmd" + ).read_text() + + # Global and per-entrypoint values are concatenated, global first: comma-joined for + # mounts, space-joined for host_binaries and the arg strings. + assert 'mounts="/global/mount,/local/mount"' in entrypoint + assert 'apptainer_args="--global-arg --local-arg"' in entrypoint + assert 'command_args="--global-cmd --local-cmd"' in entrypoint + assert "for i in globalbin localbin; do" in entrypoint + assert 'command="realcmd"' in entrypoint + + +def test_dependencies_render_module_load_lines(tmp_path: Path) -> None: + # The modulefile dependency block branches on whether a version is given: a managed + # dependency pins a version, while a version-less dependency (valid only for modules + # not managed by deploy-tools) renders the bare module name. + dependency = Release( + module=Module( + name="example-dependency", + version="1.0", + description="Managed module used as a versioned dependency target", + applications=[], + ) + ) + dependent = Release( + module=Module( + name="example-dependent", + version="1.0", + description="Module with a managed and an external dependency", + dependencies=[ + ModuleDependency(name="example-dependency", version="1.0"), + ModuleDependency(name="external-module"), + ], + applications=[], + ) + ) + deployment_root = _sync(tmp_path, dependency, dependent) + + modulefile = ( + deployment_root / "modules/example-dependent/1.0/modulefile" + ).read_text() + + # Versioned dependency renders with its version. The version-less external one + # renders the bare module name. + assert "module load example-dependency/1.0" in modulefile + assert "module load external-module\n" in modulefile + + +def test_env_vars_render_setenv(tmp_path: Path) -> None: + # env_vars render as setenv lines in the modulefile; assert one is emitted. + release = Release( + module=Module( + name="example-env-vars", + version="1.0", + description="Module exercising environment variable rendering", + env_vars=[EnvVar(name="EXAMPLE_VAR", value="example-value")], + applications=[], + ) + ) + deployment_root = _sync(tmp_path, release) + + modulefile = ( + deployment_root / "modules/example-env-vars/1.0/modulefile" + ).read_text() + + assert 'setenv EXAMPLE_VAR "example-value"' in modulefile diff --git a/tests/test_modulefile.py b/tests/test_modulefile.py new file mode 100644 index 00000000..86ca79a8 --- /dev/null +++ b/tests/test_modulefile.py @@ -0,0 +1,29 @@ +"""Direct-call unit tests for ``modulefile`` helpers.""" + +from pathlib import Path + +from deploy_tools.layout import Layout +from deploy_tools.modulefile import apply_default_versions + + +def test_apply_default_versions_removes_default_for_unlisted_module( + tmp_path: Path, +) -> None: + # apply_default_versions removes the stale default (.version) file for a deployed + # module that is absent from the default map. The CLI never reaches this branch + # because validate_default_versions assigns a default to every deployed module + # (raising otherwise), so call the function directly to pin its cleanup contract. + layout = Layout(tmp_path) + name, version = "example", "1.0" + + # A deployed module: a live modulefile link plus an existing default (.version) + # file. + mf_link = layout.get_modulefile_link(name, version) + mf_link.parent.mkdir(parents=True, exist_ok=True) + mf_link.touch() + default_file = layout.get_default_version_file(name) + default_file.write_text("#%Module1.0\nset ModulesVersion 1.0\n") + + apply_default_versions({}, layout) + + assert not default_file.exists() diff --git a/tests/test_sync.py b/tests/test_sync.py new file mode 100644 index 00000000..9bd2984b --- /dev/null +++ b/tests/test_sync.py @@ -0,0 +1,75 @@ +import shutil +from pathlib import Path + +import pytest + +from conftest import run_cli +from deploy_tools.layout import Layout +from deploy_tools.snapshot import SnapshotError +from deploy_tools.sync import SyncError + +MODULE_NAME = "example-module-multi" + + +def test_deprecate_then_remove_multiple_versions_of_same_module( + tmp_path: Path, configs: Path +) -> None: + # Regression test for issues with removing name folders for the same module name + # twice. + run_cli( + "sync", "--from-scratch", tmp_path, configs / "valid" / "multi-version-active" + ) + + layout = Layout(tmp_path) + live_name_folder = layout.modulefiles_root / MODULE_NAME + deprecated_name_folder = layout.deprecated_modulefiles_root / MODULE_NAME + + run_cli("sync", tmp_path, configs / "valid" / "multi-version-deprecated") + assert not live_name_folder.exists() + assert {p.name for p in deprecated_name_folder.iterdir()} == {"1.0", "2.0"} + + run_cli("sync", tmp_path, configs / "valid" / "empty") + assert not deprecated_name_folder.exists() + assert not (layout.modules_root / MODULE_NAME).exists() + assert run_cli("compare", tmp_path) == "" + + +def test_sync_applies_explicit_default_version_over_auto_selection( + tmp_path: Path, configs: Path +) -> None: + # settings.yaml pins the default to 1.0 even though 2.0 is deployed and would be + # auto-selected. Confirm the explicit pin is set in the on-disk .version file. + run_cli( + "sync", + "--from-scratch", + tmp_path, + configs / "valid" / "multi-version-explicit-default", + ) + + default_version_file = Layout(tmp_path).get_default_version_file(MODULE_NAME) + assert "set ModulesVersion 1.0" in default_version_file.read_text() + + +def test_sync_rejects_deployment_area_without_git_repo( + tmp_path: Path, configs: Path +) -> None: + # An area whose .git has been lost still has a snapshot but no history to commit + # against: a corrupt state surfaced as a clean error, not a GitPython traceback. + run_cli( + "sync", "--from-scratch", tmp_path, configs / "valid" / "multi-version-active" + ) + shutil.rmtree(tmp_path / ".git") + + with pytest.raises(SyncError, match="not a git repository"): + run_cli("sync", tmp_path, configs / "valid" / "multi-version-active") + + +def test_sync_from_scratch_rejects_existing_snapshot( + tmp_path: Path, configs: Path +) -> None: + # --from-scratch must refuse to run when the area already holds a snapshot. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") + with pytest.raises( + SnapshotError, match="must not exist when deploying from scratch" + ): + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") diff --git a/tests/test_validate.py b/tests/test_validate.py new file mode 100644 index 00000000..63df526d --- /dev/null +++ b/tests/test_validate.py @@ -0,0 +1,110 @@ +from pathlib import Path + +import pytest + +from conftest import run_cli +from deploy_tools.validate import ValidationError + +# Configurations that 'validate' should reject when deployed from scratch into an empty +# area. Each maps a config folder under configs/invalid to a substring expected in the +# resulting ValidationError. +INVALID_INITIAL_CONFIGS = [ + ("unknown-dependency", "unknown module dependency"), + ("default-version-not-deployed", "Unable to configure"), + ("no-eligible-default", "exclude_from_defaults"), +] + + +@pytest.mark.parametrize("config_name, message", INVALID_INITIAL_CONFIGS) +def test_validate_rejects_invalid_initial_config( + tmp_path: Path, configs: Path, config_name: str, message: str +) -> None: + with pytest.raises(ValidationError, match=message): + run_cli( + "validate", "--from-scratch", tmp_path, configs / "invalid" / config_name + ) + + +def test_validate_rejects_update_without_allow_updates( + tmp_path: Path, configs: Path +) -> None: + # Deploy a baseline whose module did not opt in to allow_updates, then validate a + # config that changes that module in place: this must be rejected. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") + with pytest.raises(ValidationError, match="modified without updating version"): + run_cli( + "validate", tmp_path, configs / "invalid" / "modified-without-allow-updates" + ) + + +def test_validate_rejects_removal_without_deprecation( + tmp_path: Path, configs: Path +) -> None: + # Deploy a baseline module, then validate a config that drops it without first + # deprecating it (and without --allow-all): this must be rejected. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") + with pytest.raises(ValidationError, match="removed without prior deprecation"): + run_cli( + "validate", tmp_path, configs / "invalid" / "removed-without-deprecation" + ) + + +def test_validate_rejects_added_deprecated_module( + tmp_path: Path, configs: Path +) -> None: + # Deploy a baseline module, then validate a config that introduces a brand-new + # release already in a deprecated state. Deprecating a module on initial creation + # is only allowed with --allow-all, so a normal validate must reject it. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") + with pytest.raises(ValidationError, match="cannot have deprecated status"): + run_cli("validate", tmp_path, configs / "invalid" / "deprecated-on-creation") + + +def test_validate_allows_added_deprecated_module_with_allow_all( + tmp_path: Path, configs: Path +) -> None: + # The --allow-all flag deliberately permits introducing an already-deprecated + # release on initial creation. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") + run_cli( + "validate", + "--allow-all", + tmp_path, + configs / "invalid" / "deprecated-on-creation", + ) + + +def test_validate_test_build(tmp_path: Path, configs: Path) -> None: + # --test-build actually builds the modules into a temporary area and syntax-checks + # the generated shell entrypoints, so exercise it on a valid shell-only config. + run_cli( + "validate", + "--test-build", + "--from-scratch", + tmp_path, + configs / "valid" / "minimal", + ) + + +def test_validate_test_build_rejects_invalid_script( + tmp_path: Path, configs: Path +) -> None: + # --test-build runs `bash -n` over each generated shell entrypoint, so a script that + # is not valid bash must be rejected. + with pytest.raises(ValidationError, match="is invalid with errors"): + run_cli( + "validate", + "--test-build", + "--from-scratch", + tmp_path, + configs / "invalid" / "invalid-shell-script", + ) + + +def test_validate_reports_no_actions_when_unchanged( + tmp_path: Path, configs: Path +) -> None: + # Validating a config that is already fully deployed previews no changes. + run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "minimal") + output = run_cli("validate", tmp_path, configs / "valid" / "minimal") + assert "No release actions required" in output