Add sync tests

src/deploy_tools/models/module.py

@@ -20,16 +20,19 @@
 class ModuleDependency(ParentModel):
     """Specify an Environment Module to include as a dependency.
 
-    If the dependent Environment Module is managed by this same Deployment (i.e. is a
-    Module), you must specify a specific version in order to pass validation.
+    The version is optional. If a version is given and the dependency is a Module
+    managed by this same Deployment, that version must exist among the deployed
+    (non-deprecated) versions to pass validation. If no version is given, the
+    dependency's default version is resolved at load time and is not validated.
     """
 
     name: Annotated[str, Field(description="Name of module dependency")]
     version: Annotated[
         str | None,
         Field(
-            description="Version of dependency. Will use default if none specified, "
-            "but this is only valid for modules not managed using deploy-tools"
+            description="Version of dependency. If omitted, the default version is "
+            "resolved at load time. If given for a deploy-tools-managed module, the "
+            "version must exist in the deployment."
         ),
     ] = None
 

src/deploy_tools/models/schemas/deployment.json

@@ -354,7 +354,7 @@
     },
     "ModuleDependency": {
       "additionalProperties": false,
-      "description": "Specify an Environment Module to include as a dependency.\n\nIf the dependent Environment Module is managed by this same Deployment (i.e. is a\nModule), you must specify a specific version in order to pass validation.",
+      "description": "Specify an Environment Module to include as a dependency.\n\nThe version is optional. If a version is given and the dependency is a Module\nmanaged by this same Deployment, that version must exist among the deployed\n(non-deprecated) versions to pass validation. If no version is given, the\ndependency's default version is resolved at load time and is not validated.",
       "properties": {
         "name": {
           "description": "Name of module dependency",
@@ -371,7 +371,7 @@
             }
           ],
           "default": null,
-          "description": "Version of dependency. Will use default if none specified, but this is only valid for modules not managed using deploy-tools",
+          "description": "Version of dependency. If omitted, the default version is resolved at load time. If given for a deploy-tools-managed module, the version must exist in the deployment.",
           "title": "Version"
         }
       },

src/deploy_tools/models/schemas/module.json

@@ -223,7 +223,7 @@
     },
     "ModuleDependency": {
       "additionalProperties": false,
-      "description": "Specify an Environment Module to include as a dependency.\n\nIf the dependent Environment Module is managed by this same Deployment (i.e. is a\nModule), you must specify a specific version in order to pass validation.",
+      "description": "Specify an Environment Module to include as a dependency.\n\nThe version is optional. If a version is given and the dependency is a Module\nmanaged by this same Deployment, that version must exist among the deployed\n(non-deprecated) versions to pass validation. If no version is given, the\ndependency's default version is resolved at load time and is not validated.",
       "properties": {
         "name": {
           "description": "Name of module dependency",
@@ -240,7 +240,7 @@
             }
           ],
           "default": null,
-          "description": "Version of dependency. Will use default if none specified, but this is only valid for modules not managed using deploy-tools",
+          "description": "Version of dependency. If omitted, the default version is resolved at load time. If given for a deploy-tools-managed module, the version must exist in the deployment.",
           "title": "Version"
         }
       },

src/deploy_tools/models/schemas/release.json

@@ -335,7 +335,7 @@
     },
     "ModuleDependency": {
       "additionalProperties": false,
-      "description": "Specify an Environment Module to include as a dependency.\n\nIf the dependent Environment Module is managed by this same Deployment (i.e. is a\nModule), you must specify a specific version in order to pass validation.",
+      "description": "Specify an Environment Module to include as a dependency.\n\nThe version is optional. If a version is given and the dependency is a Module\nmanaged by this same Deployment, that version must exist among the deployed\n(non-deprecated) versions to pass validation. If no version is given, the\ndependency's default version is resolved at load time and is not validated.",
       "properties": {
         "name": {
           "description": "Name of module dependency",
@@ -352,7 +352,7 @@
             }
           ],
           "default": null,
-          "description": "Version of dependency. Will use default if none specified, but this is only valid for modules not managed using deploy-tools",
+          "description": "Version of dependency. If omitted, the default version is resolved at load time. If given for a deploy-tools-managed module, the version must exist in the deployment.",
           "title": "Version"
         }
       },

src/deploy_tools/validate.py

@@ -88,12 +88,11 @@ def _validate_release_changes(
 
 
 def _validate_module_dependencies(deployment: Deployment) -> None:
-    """Ensure that all module dependencies are set appropriately.
+    """Ensure that all module dependencies reference versions that will exist.
 
-    This checks any module dependency names that come from current configuration to
-    ensure they exist and are not deprecated. Not specifying a particular version is
-    only valid for dependencies that are managed outside of the current deployment
-    configuration.
+    Only dependencies that pin a version are checked: if the dependency is a Module
+    managed by this deployment, the pinned version must be present among the final
+    deployed (non-deprecated) versions.
     """
     final_deployed_versions = deployment.get_final_deployed_versions()
 

tests/configs/valid/apptainer-pull/example-apptainer/1.0.yaml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-apptainer
+  version: "1.0"
+  description: Minimal single-image module exercising a real apptainer pull
+
+  applications:
+    - app_type: apptainer
+
+      container:
+        path: docker://ghcr.io/apptainer/lolcow
+        version: latest
+
+      entrypoints:
+        - name: cowsay-hello
+          command: cowsay
+          options:
+            command_args: Hello

tests/configs/valid/apptainer-pull/settings.yaml

@@ -0,0 +1,3 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json
+
+default_versions: {}

tests/configs/valid/empty/settings.yaml

@@ -0,0 +1,3 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json
+
+default_versions: {}

tests/configs/valid/multi-version-active/example-module-multi/1.0.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-module-multi
+  version: "1.0"
+  description: Module with two versions
+
+  applications: []

tests/configs/valid/multi-version-active/example-module-multi/2.0.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-module-multi
+  version: "2.0"
+  description: Module with two versions
+
+  applications: []

tests/configs/valid/multi-version-active/settings.yaml

@@ -0,0 +1,3 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json
+
+default_versions: {}

tests/configs/valid/multi-version-deprecated/example-module-multi/1.0.yaml

@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-module-multi
+  version: "1.0"
+  description: Module with two versions
+
+  applications: []
+
+# Deprecating both versions together moves their modulefile links into the deprecated
+# area and clears the now-spent live modulefiles name folder.
+deprecated: true

tests/configs/valid/multi-version-deprecated/example-module-multi/2.0.yaml

@@ -0,0 +1,12 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-module-multi
+  version: "2.0"
+  description: Module with two versions
+
+  applications: []
+
+# Deprecating both versions together moves their modulefile links into the deprecated
+# area and clears the now-spent live modulefiles name folder.
+deprecated: true

tests/configs/valid/multi-version-deprecated/settings.yaml

@@ -0,0 +1,3 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json
+
+default_versions: {}

tests/configs/valid/multi-version-explicit-default/example-module-multi/1.0.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-module-multi
+  version: "1.0"
+  description: Module with two versions, used to exercise explicit default selection
+
+  applications: []

tests/configs/valid/multi-version-explicit-default/example-module-multi/2.0.yaml

@@ -0,0 +1,8 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json
+
+module:
+  name: example-module-multi
+  version: "2.0"
+  description: Module with two versions, used to exercise explicit default selection
+
+  applications: []

tests/configs/valid/multi-version-explicit-default/settings.yaml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/deployment-settings.json
+
+# Pin the default to 1.0 even though 2.0 is present and would be auto-selected.
+default_versions:
+  example-module-multi: "1.0"

tests/test_apptainer_pull.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+from conftest import run_cli
+
+
+def test_sync_builds_real_sif_file(configs: Path, tmp_path: Path) -> None:
+    # The golden-master test stubs the apptainer pull, so this is the only test that
+    # exercises a real one. It needs apptainer installed and network access to ghcr.io;
+    # a missing binary fails the test (via run_cli) rather than skipping it, by design.
+    run_cli("sync", "--from-scratch", tmp_path, configs / "valid" / "apptainer-pull")
+
+    sif_files = list(tmp_path.glob("**/*.sif"))
+    assert sif_files, "sync did not produce a .sif file"
+    assert all(f.stat().st_size > 0 for f in sif_files), "produced .sif file is empty"

tests/test_binary.py

@@ -0,0 +1,103 @@
+import hashlib
+from pathlib import Path
+
+import pytest
+import yaml
+
+from conftest import run_cli
+from deploy_tools.app_builder import AppBuilderError
+
+# A small payload served as a local file:// "download" source, so the binary builder's
+# real urlretrieve and hashing run with no network access.
+BINARY_CONTENT = b"#!/bin/sh\necho 'hello from a binary module'\n"
+
+DIGESTS = {
+    "sha256": hashlib.sha256(BINARY_CONTENT).hexdigest(),
+    "sha512": hashlib.sha512(BINARY_CONTENT).hexdigest(),
+    "md5": hashlib.md5(BINARY_CONTENT).hexdigest(),
+}
+
+
+def _write_binary_deployment(
+    tmp_path: Path, hash_type: str, hash_value: str
+) -> tuple[Path, Path]:
+    """Build a deployment root and a single binary-module config under ``tmp_path``.
+
+    Binary modules download from a URL and verify a hash, so the config must reference a
+    real file whose digest is known at test time. A committed config cannot carry such a
+    machine-specific ``file://`` path, so it is generated per test.
+
+    Args:
+        tmp_path: The per-test scratch directory.
+        hash_type: The ``hash_type`` to record in the config.
+        hash_value: The hash to record in the config.
+
+    Returns:
+        The ``(deployment_root, config_folder)`` pair to pass to ``sync``.
+    """
+    source = tmp_path / "downloads" / "example-tool"
+    source.parent.mkdir()
+    source.write_bytes(BINARY_CONTENT)
+
+    config_folder = tmp_path / "config"
+    module_dir = config_folder / "example-binary"
+    module_dir.mkdir(parents=True)
+    (config_folder / "settings.yaml").write_text("default_versions: {}\n")
+
+    release = {
+        "module": {
+            "name": "example-binary",
+            "version": "1.0",
+            "description": "Single binary module for builder coverage",
+            "applications": [
+                {
+                    "app_type": "binary",
+                    "name": "example-tool",
+                    "url": source.as_uri(),
+                    "hash": hash_value,
+                    "hash_type": hash_type,
+                }
+            ],
+        }
+    }
+    (module_dir / "1.0.yaml").write_text(yaml.safe_dump(release))
+
+    deployment_root = tmp_path / "deployment"
+    deployment_root.mkdir()
+    return deployment_root, config_folder
+
+
+@pytest.mark.parametrize("hash_type", ["sha256", "sha512", "md5"])
+def test_sync_builds_binary_with_hash_check(tmp_path: Path, hash_type: str) -> None:
+    # Each supported algorithm should validate the download and deploy an executable
+    # binary entrypoint with the original contents.
+    deployment_root, config_folder = _write_binary_deployment(
+        tmp_path, hash_type, DIGESTS[hash_type]
+    )
+    run_cli("sync", "--from-scratch", deployment_root, config_folder)
+
+    binary = deployment_root / "modules/example-binary/1.0/entrypoints/example-tool"
+    assert binary.read_bytes() == BINARY_CONTENT
+    assert binary.stat().st_mode & 0o111, "deployed binary is not executable"
+
+
+def test_sync_builds_binary_without_hash_check(tmp_path: Path) -> None:
+    # hash_type "none" skips verification; the binary is still downloaded and deployed.
+    deployment_root, config_folder = _write_binary_deployment(tmp_path, "none", "")
+    run_cli("sync", "--from-scratch", deployment_root, config_folder)
+
+    binary = deployment_root / "modules/example-binary/1.0/entrypoints/example-tool"
+    assert binary.read_bytes() == BINARY_CONTENT
+
+
+def test_sync_rejects_binary_with_wrong_hash(tmp_path: Path) -> None:
+    # A digest that does not match the download must fail the build rather than deploy
+    # an unverified binary.
+    deployment_root, config_folder = _write_binary_deployment(
+        tmp_path, "sha256", "0" * 64
+    )
+    with pytest.raises(
+        AppBuilderError,
+        match="Downloaded Binary .*/downloads/example-tool hash check failed",
+    ):
+        run_cli("sync", "--from-scratch", deployment_root, config_folder)

tests/test_external_tools.py

@@ -0,0 +1,19 @@
+import pytest
+
+from deploy_tools.external_tools import ExternalToolError, run_command
+
+
+def test_run_command_reports_missing_tool() -> None:
+    # A required external tool that is not installed must surface as a clean
+    # ExternalToolError naming the tool, not a raw FileNotFoundError traceback.
+    with pytest.raises(ExternalToolError, match="deploy-tools-no-such-binary"):
+        run_command(["deploy-tools-no-such-binary"])
+
+
+def test_run_command_reports_failed_tool() -> None:
+    # With check=True, a non-zero exit is reported as an ExternalToolError rather than a
+    # raw CalledProcessError.
+    with pytest.raises(
+        ExternalToolError, match="External tool 'bash' failed with exit status 3"
+    ):
+        run_command(["bash", "-c", "exit 3"], check=True)