From 41ad43ec586f9fcc6ae7be2ebd47d2fd9ec52817 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Fri, 17 Jul 2026 19:13:33 +0000 Subject: [PATCH 1/9] Add pre-commit config to enforce line-length limit in docs/ --- .markdownlint.yaml | 12 ++++++++++++ .pre-commit-config.yaml | 16 ++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .markdownlint.yaml diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 00000000..33d85239 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,12 @@ +# Documentation line-length enforcement. +# +# An 88-character limit for documentation, matching [tool.ruff] line-length in +# pyproject.toml. Only the line-length rule (MD013) is enabled. +# +# Files generated by the Copier template are excluded via the +# markdownlint-cli2 hook in .pre-commit-config.yaml rather than reformatted. +default: false +MD013: + line_length: 88 + tables: false # Markdown tables cannot be sensibly wrapped + code_blocks: false # fenced code and tree diagrams cannot be wrapped diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a43ed574..6c3561d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,6 +31,22 @@ repos: entry: uv sync files: ^(uv\.lock|pyproject\.toml)$ + - repo: https://github.com/DavidAnson/markdownlint-cli2 + rev: v0.23.1 + hooks: + - id: markdownlint-cli2 + name: enforce 88-char line length in docs + files: ^docs/.*\.md$ + # Files generated by the Copier template are excluded. + exclude: | + (?x)^docs/( + index|genindex|how-to|tutorials|explanations|reference| + how-to/contribute|how-to/run-container|tutorials/installation| + explanations/decisions| + explanations/decisions/0001-record-architecture-decisions| + explanations/decisions/0002-switched-to-python-copier-template + )\.md$ + - repo: https://github.com/gitleaks/gitleaks rev: v8.28.0 hooks: From 8593ae04d4bded750108f4d7d9159b1f783bddf0 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 21 Jul 2026 12:32:15 +0000 Subject: [PATCH 2/9] Add explanatory docs for deploy-tools --- docs/explanations.md | 7 ++- docs/explanations/default-versions.md | 48 ++++++++++++++ docs/explanations/deployment-area.md | 73 ++++++++++++++++++++++ docs/explanations/deprecation-lifecycle.md | 42 +++++++++++++ docs/explanations/snapshots-and-compare.md | 64 +++++++++++++++++++ 5 files changed, 232 insertions(+), 2 deletions(-) create mode 100644 docs/explanations/default-versions.md create mode 100644 docs/explanations/deployment-area.md create mode 100644 docs/explanations/deprecation-lifecycle.md create mode 100644 docs/explanations/snapshots-and-compare.md diff --git a/docs/explanations.md b/docs/explanations.md index 73ab289b..3a758d1a 100644 --- a/docs/explanations.md +++ b/docs/explanations.md @@ -4,7 +4,10 @@ Explanations of how it works and why it works that way. ```{toctree} :maxdepth: 1 -:glob: -explanations/* +explanations/deployment-area +explanations/snapshots-and-compare +explanations/deprecation-lifecycle +explanations/default-versions +explanations/decisions ``` diff --git a/docs/explanations/default-versions.md b/docs/explanations/default-versions.md new file mode 100644 index 00000000..8a1a4c19 --- /dev/null +++ b/docs/explanations/default-versions.md @@ -0,0 +1,48 @@ +# Default version resolution + +When a user runs `module load ` without a version, Environment Modules must pick +one. By default it picks the lexically last version, which is often wrong: `1.10` sorts +before `1.9`, and an unfinished alpha might also win. `deploy-tools` allows you to make +an explicit choice, and otherwise uses the `natsort` library for a more reasonable default. + +## Every Module has a specific default configured + +For every live module name, `deploy-tools` writes a `.version` file next to its modulefile +links (`modulefiles//.version`) with a single `set ModulesVersion ` line. +It is written whether or not the configuration names a default, so Environment Modules never +guesses. + +The version it records is resolved as follows: + +1. **An explicit default has priority.** If `settings.default_versions` names a version, + it is used. It must be a live (non-deprecated) version that exists, or validation fails. +2. **Otherwise the highest eligible version is chosen automatically**, by natural version + sorting (see below), considering only versions *not* marked `exclude_from_defaults`. + +Deprecated versions are never candidates and never get a `.version` file. + +## Natural version sorting + +Automatic selection uses [`natsort`](https://natsort.readthedocs.io) rather than plain +string comparison, so `1.10` sorts after `1.9`. It also uses natsort's key for +pre-release schemes, so pre-release tags sort *before* the release they lead up to: + +```text +1.2alpha1 < 1.2rc1 < 1.2 < 1.2.1 +``` + +So publishing `1.2rc1` does not make it the default while `1.2` exists, and publishing +`1.2` makes it the default over its own release candidates. Be aware that `1.3alpha1` still +sorts after `1.2.1`, however; to keep alphas or similar out of the automatic default, mark +them `exclude_from_defaults: true`. + +## Excluding a version from the automatic default + +Marking a version `exclude_from_defaults: true` keeps it out of *automatic* selection +while still allowing it to be set explicitly. Use this to publish an alpha or beta that +users can opt into with `module load /` without it ever being handed to +someone who just typed `module load `. + +One safety check applies: if *every* version of a name is excluded and no explicit default +is given, validation fails rather than leave the Module with no default. Either give an +explicit default or keep at least one non-excluded version. diff --git a/docs/explanations/deployment-area.md b/docs/explanations/deployment-area.md new file mode 100644 index 00000000..baccfc3f --- /dev/null +++ b/docs/explanations/deployment-area.md @@ -0,0 +1,73 @@ +# The deployment area + +The *deployment area* is the directory tree that `deploy-tools` writes into. It is normally +a shared filesystem location that end users put on their `MODULEPATH`, so its layout is +important to the end user. This page explains what lives where, and why. + +## Layout + +Everything below is created and maintained by `deploy-tools`; nothing in the area is meant +to be edited by hand. + +```text +/ +├── deployment.yaml # snapshot of the deployed configuration +├── modules/ # Modules Area — the built files for every Module +│ └── // +│ ├── modulefile # the Environment Modules (Tcl) file +│ ├── entrypoints/ # the executables added to PATH on module load +│ ├── module.yaml # this Module's configuration snapshot +│ └── sif_files/ # Apptainer images, when the Module uses any +├── modulefiles/ # Modulefiles Folder — placed on MODULEPATH +│ └── / +│ ├── # symlink → ../../modules///modulefile +│ └── .version # records the default version (see below) +├── deprecated/ +│ └── modulefiles/ # Deprecated Folder — optionally on MODULEPATH +│ └── / # symlink for a deprecated Module version +└── build/ # transient build area (default location) +``` + +The split between the **Modules Area** (`modules/`) and the **Modulefiles Folder** +(`modulefiles/`) is the central design choice. The Modules Area holds the real files for +*every* version that has been deployed and not removed, both live and deprecated. The +Modulefiles Folder holds only a thin tree of symlinks, one per version, pointing at the +corresponding `modulefile`. Users put only the *modulefiles* directories on their +`MODULEPATH`, making each Module visible to `module avail` and `module load`. + +Deprecation is therefore cheap and reversible: the built files never move, only the symlink +moves between `modulefiles/` and `deprecated/modulefiles/`. See +[the release lifecycle](deprecation-lifecycle.md) for the full set of transitions. + +## The build area + +Modules are first assembled under the **build area** (`build/` by default), then moved into +the Modules Area. The move is a filesystem `rename`, which is atomic *only when source and +destination are on the same filesystem* — hence the default build area sits inside the +deployment root. Building and deploying separately means a half-built Module is never +visible: either the rename succeeds and the whole version appears, or nothing changes. + +The transient `build/` directory and the large Apptainer `sif_files/` images are +deliberately kept out of the deployment area's git history (see +[snapshots and the compare safety net](snapshots-and-compare.md)). They are reference-only +data used by the `compare` command, not something to revert to. + +## Why modulefiles embed absolute paths + +A generated `modulefile` adds its Module's executables to the user's path with one line: + +```tcl +prepend-path PATH "/modules///entrypoints" +``` + +That path is **absolute**, so a deployment area cannot be relocated after the fact. +This is a deliberate trade-off. + +A path relative to the modulefile does not work. Environment Modules sources the file +*through the symlink* under `modulefiles/` without resolving it first (`[info script]` +/ `ModulesCurrentModulefile` report the link path). A relative path would be interpreted +relative to the *link's* location — which changes when a Module is deprecated and the link +moves to a different location in the tree, silently breaking it. + +This allows us to use two separate `modulefiles/` folders, corresponding to live and +deprecated modules, without needing to edit modulefiles after initial creation. diff --git a/docs/explanations/deprecation-lifecycle.md b/docs/explanations/deprecation-lifecycle.md new file mode 100644 index 00000000..a9c38446 --- /dev/null +++ b/docs/explanations/deprecation-lifecycle.md @@ -0,0 +1,42 @@ +# The release lifecycle + +A *Release* is a Module version plus one boolean: whether it is `deprecated`. The whole +lifecycle of a deployed Module — appearing, being updated, retired, restored, and finally +deleted — is expressed by adding or removing a Release in configuration and toggling its +`deprecated` flag. This page explains these transitions and the guard rails around them. + +## Configuration is declarative + +You never tell `deploy-tools` to "deprecate" or "remove" something directly. You describe +the set of Releases you want, and the tool compares that against the +[snapshot](snapshots-and-compare.md) of the last `sync` to derive the actions needed. Each +Release falls into one of these cases: + +| Transition | Detected when… | Effect on the deployment area | +|------------|----------------|-------------------------------| +| **Add** | the name or version is not in the last snapshot | build the Module, create a live modulefile link | +| **Update** | the version exists but its Module config changed | rebuild in place (only if `allow_updates` is set) | +| **Deprecate** | the version was live in the last snapshot, now `deprecated: true` | move the modulefile link into `deprecated/modulefiles/` | +| **Restore** | was deprecated in the last snapshot, now `deprecated: false` | move the link back into `modulefiles/` | +| **Remove** | present in the last snapshot, absent from configuration | delete the modulefile link and the built files | + +Deprecate and restore only move a symlink; the built files stay in the Modules Area (see +[the deployment area](deployment-area.md)). If your `MODULEPATH` includes +`deprecated/modulefiles`, a deprecated Module still loads as before — deprecation indicates +"you should stop using this", not "this is gone". + +## The guard rails + +Some transitions are blocked by default so mistakes aren't applied silently: + +- **A new version may not be born deprecated.** Adding a Release with `deprecated: true` + is rejected. +- **An existing version may not change without a new version.** If its configuration differs + from the last snapshot, validation fails unless the Module sets `allow_updates: true`. +- **A live version may not be deleted directly.** Deprecate it in one `sync`, then + remove it in a later one. + +`--allow-all` lifts the first and third guards, so you can force those transitions. It does +*not* affect updates: changing a version always requires `allow_updates: true`. +`--from-scratch` implies `--allow-all`. Neither relaxes the integrity checks — the area +must still be healthy. diff --git a/docs/explanations/snapshots-and-compare.md b/docs/explanations/snapshots-and-compare.md new file mode 100644 index 00000000..dab83102 --- /dev/null +++ b/docs/explanations/snapshots-and-compare.md @@ -0,0 +1,64 @@ +# Snapshots and the compare safety net + +`deploy-tools` has no transactional rollback. Instead it relies on a *snapshot* of the +intended configuration plus a `compare` command that detects when reality has drifted from +it. This page explains how the two fit together. + +## What the snapshot is + +The snapshot is a single file, `deployment.yaml`, at the root of the deployment area. +It is a serialised [`Deployment`](../schemas/release.rst) — the complete set of Releases +(live and deprecated) plus the settings, including each Module's default version. It +records what the deployment area is supposed to contain. + +The `sync` command writes it; `sync`, `validate` and `compare` read it back to work out +what has changed since the last `sync`. + +## Why it is written *before* the deploy + +During a `sync`, the snapshot is written **before** the files are moved into place. This +means the snapshot always describes the *target* state. If the deploy step then fails part +way through, the snapshot and the actual files no longer agree, and the next `compare` +reports that discrepancy. Were the snapshot written last, a deploy that died before making +any changes could result in `compare` identifying a broken area as healthy. + +## How compare works + +It reconstructs the de-facto `Deployment` by walking the deployment area — reading every +`module.yaml`, checking which versions have a live versus a deprecated modulefile link, +and reading each `.version` file — then diffs that against the snapshot. Any mismatch (a +missing modulefile, a built module with no link, a wrong default version, a corrupt +metadata file) is raised as a clear error showing the differing lines. + +`compare` checks the area's *structure* — the modulefiles, links and metadata whose +breakage would disable many Modules at once. It does not inspect the contents of built +payloads such as Apptainer `.sif` images, so a `.sif` that is corrupt but present at the +expected path is not detected. That risk is avoided at build time instead: a Module is +built on the same filesystem as the deployment area and published by a single atomic +rename, so a partial or failed build is never moved into place. + +This is why CI should run `compare` *before* every `sync`: it confirms the area is in the +healthy state the last `sync` claimed to leave it in. + +## Recovery is manual + +With no automatic rollback, recovering from a failed `sync` is an administrator task: use +`compare` to see what differs, repair the area by hand (or restore the previous snapshot), +and re-run `compare` before another `sync`. Automatic self-healing was rejected to keep +the tool simple for a small development team. + +Two facilities help here: + +- `compare --use-ref ` compares the area against the snapshot stored at a previous + git commit of the deployment area (e.g. `HEAD~1`), since each `sync` commits the + snapshot. This answers "does the area still match the configuration from before the + last `sync`?" +- `compare --from-scratch` asserts only that the deployment root exists and is *empty* — + the check to run in CI before the very first `sync`, when no snapshot exists yet. + +The git repository in the deployment area exists only to give `compare --use-ref` this +reference point. It deliberately excludes the build area and Apptainer images, and is +**not** intended for reverting the area's state. + +Otherwise, it is expected that the deployment area is properly backed up, preferably with +some kind of filesystem-level snapshot system. From bc224f1222147309c7945e63bc2eba2c0bb40062 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 21 Jul 2026 12:49:06 +0000 Subject: [PATCH 3/9] Document compare --from-scratch in README --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 37bc8be4..3152397d 100644 --- a/README.md +++ b/README.md @@ -83,8 +83,14 @@ available as Tasks and Debug configurations for VSCode. These tasks (plus their inputs) should create a separate `demo-output` folder at the top-level of the workspace folder. -You will need to use the `--from-scratch` argument if starting from a clean Deployment -Area, as there is no snapshot from a prior Deploy step. +You will need to use the `--from-scratch` argument when starting from a clean Deployment +Area, as there is no snapshot from a prior Deploy step: + +- `sync`/`validate --from-scratch` work against an empty area without expecting a + snapshot, and imply `--allow-all` (there is no prior state to enforce lifecycle rules + against). +- `compare --from-scratch` just checks the Deployment Area exists and is **empty**. Run + this in CI before the first deploy, where a normal `compare` has no snapshot to use. It is recommended that you use `--help` to explore the commands, arguments and options in greater detail. From a73ffabd760a70f152c07d554b6cf7de02cf0633 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 21 Jul 2026 12:50:52 +0000 Subject: [PATCH 4/9] Update README to avoid confusion between Steps and Commands --- README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3152397d..f59c6d53 100644 --- a/README.md +++ b/README.md @@ -51,15 +51,19 @@ deploy-tools compare $deployment_root ## Deployment Steps -There are several key steps that make up the deployment process. Note that these are a -bit different to the CLI commands; see `deploy-tools --help` for more information. - -|**Step**|**Description** | -|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -|Compare |Compare the snapshot taken of the previous deployment with the modulefiles and built modules that already exist. This ensures that the Deployment Area is in a healthy state | -|Validate|Process the updated configuration. By comparing the new configuration files with a snapshot from the previous deployment, we determine the set of actions that need to be taken | -|Build |Generate entrypoint scripts, configuration files and environment variables for a given Module. These are output to the Build Area | -|Deploy |Move all built Modules from the Build Area into the Modules Area. A link to the built modulefile is moved to either the Modulefiles Folder or Deprecated Folder, depending on its deprecation status. Update default versions for the modulefiles| +There are several key conceptual steps that make up the deployment process. These are +**not** one-to-one with the CLI commands; the "Run by" column shows which command runs +each step. See `deploy-tools --help` for more detail. + +|**Step**|**Description** |**Run by**| +|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| +|Compare |Compare the snapshot taken of the previous deployment with the modulefiles and built modules that already exist. This ensures that the Deployment Area is in a healthy state |`compare` | +|Validate|Process the updated configuration. By comparing the new configuration files with a snapshot from the previous deployment, we determine the set of actions that need to be taken |`validate`, `sync`| +|Build |Generate entrypoint scripts, configuration files and environment variables for a given Module. These are output to the Build Area |`sync` (`validate --test-build`)| +|Deploy |Move all built Modules from the Build Area into the Modules Area. A link to the built modulefile is moved to either the Modulefiles Folder or Deprecated Folder, depending on its deprecation status. Update default versions for the modulefiles|`sync` | + +There is no standalone `build` or `deploy` command: `sync` always runs them together, so +a half-built Module is never exposed to users. ## JSON Schema From b663d0cd9a4528dee8d352d55f63bccb8c49b7de Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 21 Jul 2026 12:51:27 +0000 Subject: [PATCH 5/9] Add docs & explanations for the different schemas --- README.md | 3 +++ docs/schemas.md | 29 ++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f59c6d53..72c67d88 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ We strongly recommend that you provide a schema for configuration file validatio ```# yaml-language-server: $schema=https://raw.githubusercontent.com/DiamondLightSource/deploy-tools/main/src/deploy_tools/models/schemas/release.json``` +Use `release.json` for the per-version Module files (`/.yaml`) and +`deployment-settings.json` for the top-level `settings.yaml`. + As the demo_configuration is used during development, we instead set it to use the locally generated schemas via an absolute workspace path (e.g. `/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json`). This dev-container-only path should not be used for production configuration. Note that the 'Generate Schema' VSCode task will update the schemas according to any update of the code, but you need to trigger this manually and check the contents in. diff --git a/docs/schemas.md b/docs/schemas.md index bed77c9a..0ecd06a0 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -1,11 +1,27 @@ # JSON Schema files -These JSON schema files are generated from the Pydantic models in `src/deploy_tools/models`. +These JSON schema files are generated from the Pydantic models in +`src/deploy_tools/models`. The YAML language server and other tooling use them to validate +deployment configuration files. This section links to per-schema reference pages that +render each schema and list its properties. -They are used by the YAML language server and other tooling to validate deployment configuration files. +## Which file uses which schema -This section links to per-schema reference pages which render the schema and list the individual -properties as tables. +You author two kinds of configuration file, each validated against a different schema: + +| Configuration file | Schema | Pydantic model | +|--------------------|--------|----------------| +| `settings.yaml` (one per config folder) | `deployment-settings.json` | `DeploymentSettings` | +| `/.yaml` (one per Module version) | `release.json` | `Release` | + +Per-version files sit in a folder named after the Module, so the path is +`//.yaml` (folder = Module `name`, filename = `version`). +Add a `yaml-language-server` comment pointing at the matching schema so your editor +validates each file as you type. + +The other two generated schemas cover files you don't normally author by hand: +`deployment.json` (the `deployment.yaml` snapshot written by `sync`) and `module.json` +(the `Module` that a `Release` wraps). ## Schema pages @@ -18,9 +34,8 @@ schemas/release ## How the schemas are generated -The `deploy_tools.models.schema.generate_schema` function writes these files from the corresponding Pydantic models. - -To regenerate them manually: +`deploy_tools.models.schema.generate_schema` writes these files from the corresponding +Pydantic models. To regenerate them manually: ```bash deploy-tools schema path/to/output/folder From d9e105fd8df9f22b50861869e35319b58e6bd796 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Wed, 22 Jul 2026 17:11:01 +0000 Subject: [PATCH 6/9] Clarify docs based on feedback --- docs/explanations/default-versions.md | 8 ++++---- docs/explanations/snapshots-and-compare.md | 17 +++++++++-------- docs/schemas.md | 13 +++++++++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/explanations/default-versions.md b/docs/explanations/default-versions.md index 8a1a4c19..a70569a5 100644 --- a/docs/explanations/default-versions.md +++ b/docs/explanations/default-versions.md @@ -7,10 +7,10 @@ an explicit choice, and otherwise uses the `natsort` library for a more reasonab ## Every Module has a specific default configured -For every live module name, `deploy-tools` writes a `.version` file next to its modulefile -links (`modulefiles//.version`) with a single `set ModulesVersion ` line. -It is written whether or not the configuration names a default, so Environment Modules never -guesses. +For every live module name, `deploy-tools` writes a `.version` file in its modulefiles +folder (`modulefiles//.version`) with a single `set ModulesVersion ` line. +It is written even if the configuration does not have an explicit default, so Environment +Modules never guesses. The version it records is resolved as follows: diff --git a/docs/explanations/snapshots-and-compare.md b/docs/explanations/snapshots-and-compare.md index dab83102..264b5568 100644 --- a/docs/explanations/snapshots-and-compare.md +++ b/docs/explanations/snapshots-and-compare.md @@ -31,11 +31,11 @@ missing modulefile, a built module with no link, a wrong default version, a corr metadata file) is raised as a clear error showing the differing lines. `compare` checks the area's *structure* — the modulefiles, links and metadata whose -breakage would disable many Modules at once. It does not inspect the contents of built -payloads such as Apptainer `.sif` images, so a `.sif` that is corrupt but present at the -expected path is not detected. That risk is avoided at build time instead: a Module is -built on the same filesystem as the deployment area and published by a single atomic -rename, so a partial or failed build is never moved into place. +breakage would disable many Modules at once. It does not inspect the contents of large +built files such as Apptainer `.sif` images, so a corrupted `.sif` in the correct location +will not be detected. The risk of corruption is avoided at build time instead: a Module +is built on the same filesystem as the deployment area and published by a single atomic +rename, so a partial or failed build (including `.sif` files) is never moved into place. This is why CI should run `compare` *before* every `sync`: it confirms the area is in the healthy state the last `sync` claimed to leave it in. @@ -51,10 +51,11 @@ Two facilities help here: - `compare --use-ref ` compares the area against the snapshot stored at a previous git commit of the deployment area (e.g. `HEAD~1`), since each `sync` commits the - snapshot. This answers "does the area still match the configuration from before the - last `sync`?" + snapshot. This is helpful when attempting to fix a broken deployment area, as it is + often easier to rollback the configuration to a previous state rather than fix the + latest deployment. - `compare --from-scratch` asserts only that the deployment root exists and is *empty* — - the check to run in CI before the very first `sync`, when no snapshot exists yet. + this is the check to run in CI before the very first `sync`, when no snapshot exists yet. The git repository in the deployment area exists only to give `compare --use-ref` this reference point. It deliberately excludes the build area and Apptainer images, and is diff --git a/docs/schemas.md b/docs/schemas.md index 0ecd06a0..f1521fbe 100644 --- a/docs/schemas.md +++ b/docs/schemas.md @@ -16,8 +16,17 @@ You author two kinds of configuration file, each validated against a different s Per-version files sit in a folder named after the Module, so the path is `//.yaml` (folder = Module `name`, filename = `version`). -Add a `yaml-language-server` comment pointing at the matching schema so your editor -validates each file as you type. +Add a `yaml-language-server` comment as the first line of each file, pointing at the +matching schema, so your editor validates it as you type: + +```yaml +# yaml-language-server: $schema=https://raw.githubusercontent.com/DiamondLightSource/deploy-tools/main/src/deploy_tools/models/schemas/release.json +``` + +This requires an editor with a YAML language server — e.g. VS Code with the +[Red Hat YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml), +or any [LSP](https://microsoft.github.io/language-server-protocol/)-capable editor +running [`yaml-language-server`](https://github.com/redhat-developer/yaml-language-server). The other two generated schemas cover files you don't normally author by hand: `deployment.json` (the `deployment.yaml` snapshot written by `sync`) and `module.json` From 6018beb308b6b4262621551d843f9e8280afcfb4 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 23 Jul 2026 14:09:48 +0000 Subject: [PATCH 7/9] Clarify snapshot terminology in docs --- README.md | 6 +++--- docs/explanations/snapshots-and-compare.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 72c67d88..6682af93 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ deploy-tools validate $deployment_root $config_folder # validation as part of determining the required changes deploy-tools sync $deployment_root $config_folder -# Compare the previous deployment snapshot against what is actually deployed in the +# Compare the current deployment snapshot against what is actually deployed in the # deployment area. CI/CD should run this before a deploy to confirm a healthy state. deploy-tools compare $deployment_root @@ -57,8 +57,8 @@ each step. See `deploy-tools --help` for more detail. |**Step**|**Description** |**Run by**| |--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------| -|Compare |Compare the snapshot taken of the previous deployment with the modulefiles and built modules that already exist. This ensures that the Deployment Area is in a healthy state |`compare` | -|Validate|Process the updated configuration. By comparing the new configuration files with a snapshot from the previous deployment, we determine the set of actions that need to be taken |`validate`, `sync`| +|Compare |Compare the current deployment snapshot file with the modulefiles and built modules that currently exist. This ensures that the Deployment Area is in a healthy state |`compare` | +|Validate|Process the updated configuration. By comparing the new configuration files with the current deployment snapshot from the last sync, we determine the set of actions that need to be taken |`validate`, `sync`| |Build |Generate entrypoint scripts, configuration files and environment variables for a given Module. These are output to the Build Area |`sync` (`validate --test-build`)| |Deploy |Move all built Modules from the Build Area into the Modules Area. A link to the built modulefile is moved to either the Modulefiles Folder or Deprecated Folder, depending on its deprecation status. Update default versions for the modulefiles|`sync` | diff --git a/docs/explanations/snapshots-and-compare.md b/docs/explanations/snapshots-and-compare.md index 264b5568..3e55679f 100644 --- a/docs/explanations/snapshots-and-compare.md +++ b/docs/explanations/snapshots-and-compare.md @@ -50,8 +50,8 @@ the tool simple for a small development team. Two facilities help here: - `compare --use-ref ` compares the area against the snapshot stored at a previous - git commit of the deployment area (e.g. `HEAD~1`), since each `sync` commits the - snapshot. This is helpful when attempting to fix a broken deployment area, as it is + git commit of the deployment area (e.g. `HEAD~1`); each `sync` commits its snapshot. + This is helpful when attempting to fix a broken deployment area, as it is often easier to rollback the configuration to a previous state rather than fix the latest deployment. - `compare --from-scratch` asserts only that the deployment root exists and is *empty* — From 29b23cf928a8bd68422e20c8b76ed438a14a8ce8 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 23 Jul 2026 14:10:21 +0000 Subject: [PATCH 8/9] Fix docs to show that allow_updates can enable immediate removal --- docs/explanations/deprecation-lifecycle.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/explanations/deprecation-lifecycle.md b/docs/explanations/deprecation-lifecycle.md index a9c38446..39a6b957 100644 --- a/docs/explanations/deprecation-lifecycle.md +++ b/docs/explanations/deprecation-lifecycle.md @@ -34,9 +34,8 @@ Some transitions are blocked by default so mistakes aren't applied silently: - **An existing version may not change without a new version.** If its configuration differs from the last snapshot, validation fails unless the Module sets `allow_updates: true`. - **A live version may not be deleted directly.** Deprecate it in one `sync`, then - remove it in a later one. + remove it in a later one — unless the Module sets `allow_updates: true`. -`--allow-all` lifts the first and third guards, so you can force those transitions. It does -*not* affect updates: changing a version always requires `allow_updates: true`. +`--allow-all` lifts the first and third guards, so you can force those transitions. `--from-scratch` implies `--allow-all`. Neither relaxes the integrity checks — the area must still be healthy. From 0f88183a068c575d1d97ce956fcd9e0e2922357f Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 23 Jul 2026 14:52:01 +0000 Subject: [PATCH 9/9] Fix minor typos in explanations docs --- docs/explanations/default-versions.md | 2 +- docs/explanations/deployment-area.md | 2 +- docs/explanations/snapshots-and-compare.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/explanations/default-versions.md b/docs/explanations/default-versions.md index a70569a5..978f97b6 100644 --- a/docs/explanations/default-versions.md +++ b/docs/explanations/default-versions.md @@ -8,7 +8,7 @@ an explicit choice, and otherwise uses the `natsort` library for a more reasonab ## Every Module has a specific default configured For every live module name, `deploy-tools` writes a `.version` file in its modulefiles -folder (`modulefiles//.version`) with a single `set ModulesVersion ` line. +folder (`modulefiles//.version`) containing a `set ModulesVersion ` line. It is written even if the configuration does not have an explicit default, so Environment Modules never guesses. diff --git a/docs/explanations/deployment-area.md b/docs/explanations/deployment-area.md index baccfc3f..064a42cf 100644 --- a/docs/explanations/deployment-area.md +++ b/docs/explanations/deployment-area.md @@ -20,7 +20,7 @@ to be edited by hand. │ └── sif_files/ # Apptainer images, when the Module uses any ├── modulefiles/ # Modulefiles Folder — placed on MODULEPATH │ └── / -│ ├── # symlink → ../../modules///modulefile +│ ├── # absolute symlink into modules///modulefile │ └── .version # records the default version (see below) ├── deprecated/ │ └── modulefiles/ # Deprecated Folder — optionally on MODULEPATH diff --git a/docs/explanations/snapshots-and-compare.md b/docs/explanations/snapshots-and-compare.md index 3e55679f..7c864e35 100644 --- a/docs/explanations/snapshots-and-compare.md +++ b/docs/explanations/snapshots-and-compare.md @@ -7,7 +7,7 @@ it. This page explains how the two fit together. ## What the snapshot is The snapshot is a single file, `deployment.yaml`, at the root of the deployment area. -It is a serialised [`Deployment`](../schemas/release.rst) — the complete set of Releases +It is a serialised [`Deployment`](../schemas.md) — the complete set of Releases (live and deprecated) plus the settings, including each Module's default version. It records what the deployment area is supposed to contain.