[EXPERIMENTAL] Conda lockfile update PR to break up later

.github/pull_request_template.md

@@ -9,6 +9,5 @@ Aim to have all relevant checks ticked off before merging. See the [developer's
 - [ ] All tests and CI checks pass.
 - [ ] Ensured the pull request title is descriptive.
 - [ ] Ensure `rose-suite.conf.example` has been updated if new diagnostic added.
-- [ ] Conda lock files have been updated if dependencies have changed.
 - [ ] Attributed any Generative AI, such as GitHub Copilot, used in this PR.
 - [ ] Marked the PR as ready to review.

.github/workflows/scheduled-updates.yml

@@ -10,6 +10,7 @@ permissions: {}
 
 jobs:
   scheduled-updates:
+    if: "github.repository == 'MetOffice/CSET'"  # Skip on forks.
     name: Regular conda and pre-commit updates
     runs-on: ubuntu-latest
     timeout-minutes: 20
@@ -28,56 +29,27 @@ jobs:
         with:
           python-version: "3.x"
 
-      - name: Install dependencies
-        run: pip install conda-lock pre-commit
-
       - name: Setup git user and branch
         run: |
           git config --local user.name "github-actions[bot]"
           git config --local user.email "github-actions[bot]@users.noreply.github.com"
           git switch -c "scheduled-updates-$(date +%s)"
 
-      - name: Update CSET developer and workflow lock files
-        run: |
-          py_vers="3.12 3.13 3.14"
-          for py_ver in ${py_vers}
-          do
-            # Developer lock files.
-            cp "requirements/environment.yml" "${RUNNER_TEMP}/${py_ver}_dev_environment.yml"
-            echo -e "\n  - python = $py_ver" >> "${RUNNER_TEMP}/${py_ver}_dev_environment.yml"
-            conda-lock --channel conda-forge --kind explicit --file "${RUNNER_TEMP}/${py_ver}_dev_environment.yml" --platform linux-64 --filename-template "requirements/locks/py$(echo $py_ver | sed 's/\.//')-lock-linux-64.txt"
-          done
-
-          # Sort lock files to make diffs easier to review.
-          for file in requirements/locks/*.txt
-          do
-            # Leave the file header unsorted.
-            cat "$file" | (sed -u 4q; sort) > sorted.txt
-            mv sorted.txt "$file"
-          done
-
-      - name: Update pre-commit config
-        run: pre-commit autoupdate --freeze
-
-      - name: Generate GitHub App Token
-        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1
-        id: app-token
-        with:
-          app-id: ${{ secrets.AUTH_APP_ID }}
-          private-key: ${{ secrets.AUTH_APP_PRIVATE_KEY }}
+      - name: Update CSET developer lock files and pre-commit hooks
+        run: make update-dev-deps
 
       - name: Commit changes and create pull requests
         env:
           source_ref: ${{ github.ref }}
           source_ref_type: ${{ github.ref_type }}
-          gh_token: ${{ steps.app-token.outputs.token }}
+          gh_token: ${{ github.token }}
           repository: ${{ github.repository }}
         run: |
           original_commit="$(git rev-parse --verify HEAD)"
           git diff --stat
 
           # Commit changed lockfiles.
-          git add requirements/locks/*.txt
+          git add requirements/locks/
           git commit -m "[CI] Update conda lock files" || true
 
           # Commit changed pre-commit config.

.github/workflows/weekly-checks.yml

@@ -10,6 +10,7 @@ permissions: {}
 
 jobs:
   check-documentation-hyperlinks:
+    if: "github.repository == 'MetOffice/CSET'"  # Skip on forks.
     name: Check documentation hyperlinks
     runs-on: ubuntu-latest
     # Give plenty of time as link checking third-party sites might be slow.
@@ -43,6 +44,7 @@ jobs:
         run: sphinx-build -b linkcheck --color -W --keep-going "docs/source" "docs/build/linkcheck"
 
   full-tests:
+    if: "github.repository == 'MetOffice/CSET'"  # Skip on forks.
     runs-on: ubuntu-latest
     timeout-minutes: 20
     permissions:

.pre-commit-config.yaml

@@ -43,6 +43,15 @@ repos:
         additional_dependencies:
           - jinja2
 
+  - repo: local
+    hooks:
+      - id: check-conda-lockfiles
+        name: Check conda lockfiles are up-to-date
+        description: Ensure the conda lockfiles include new dependencies.
+        entry: ./scripts/check-conda-lockfiles.sh
+        language: script
+        files: requirements/
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c  # frozen: v6.0.0
     hooks:

Makefile

@@ -49,6 +49,9 @@ test-full: pre-commit ## Run all tests, including slow or network reliant.
 	playwright install --only-shell chromium
 	pytest -vv --cov --cov-append --cov-config=pyproject.toml --numprocesses logical
 
+update-dev-deps:  ## Update pre-commit hooks and conda lock files for the development environment.
+	scripts/update-developer-dependencies.sh
+
 # Mark targets as 'phony' to indicate they don't actually produce a file with
 # the same name as their target. Basically for actions rather than files.
-.PHONY: help setup docs test test-fast test-full prepare-lockfiles conda
+.PHONY: help setup docs test test-fast test-full prepare-lockfiles conda update-dev-deps

docs/source/contributing/dependencies.rst

@@ -33,12 +33,15 @@ Specifically it needs to be added to the ``pyproject.toml`` under the
 "dependencies" key, and ``requirements/environment.yaml`` under the appropriate
 dependencies section.
 
-After updating those two files and making a pull request, you'll need to rerun
-the conda lockfile generation action. In `Actions > Update conda lock files >
-Run workflow`_ select your branch, then run the workflow. A new PR will be
-created to update the lockfiles, which you can merge into your own branch.
+After updating those two files and making a pull request, you'll need to
+regenerate the conda lockfiles.
 
-.. _Actions > Update conda lock files > Run workflow: https://github.com/MetOffice/CSET/actions/workflows/conda-lock.yml
+.. code-block:: bash
+
+    # Regenerate the lock files.
+    make update-dev-deps
+    # Commit them to your branch.
+    git commit -m "Update conda lockfiles" requirements/
 
 Updating the conda-forge package
 --------------------------------

docs/source/contributing/index.rst

@@ -64,16 +64,6 @@ The pull request title can be edited by clicking the "Edit" button to its right.
 If you need more text to describe what the pull request does, please add it in
 the description.
 
-Conda lock files have been updated if dependencies changed
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you have changed the dependencies of CSET you will need to regenerate the
-conda dependency lock files. This can be done by running the `update conda lock
-files workflow`_ from your branch, then merging the PR it creates into your own
-branch. (Not the default ``main`` branch!)
-
-.. _update conda lock files workflow: https://github.com/MetOffice/CSET/actions/workflows/conda-lock.yml
-
 Attributed any Generative AI, such as GitHub Copilot, used in this PR
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

scripts/check-conda-lockfiles.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Check conda lockfiles are up-to-date.
+set -eu
+
+if ! sha256sum -c requirements/locks/sources
+then
+  echo "Conda lockfiles are outdated. Please run 'make update-dev-deps' to update them."
+  exit 1
+fi

scripts/update-developer-dependencies.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Update conda lock files and pre-commit hooks.
+set -euo pipefail
+
+# Create a temporary virtual environment.
+venv_location="$(mktemp -d)"
+echo "Creating temporary virtual environment at ${venv_location}"
+python3 -m venv "${venv_location}"
+# shellcheck disable=SC1091
+source "${venv_location}/bin/activate"
+
+echo "Installing update tools"
+pip install --quiet conda-lock pre-commit
+
+# Create lock files for supported python versions.
+py_vers="3.12 3.13 3.14"
+env_tmp="$(mktemp -d)"
+for py_ver in ${py_vers}
+do
+    # Developer lock files.
+    echo "Updating lock files for python ${py_ver}"
+    cp "requirements/environment.yml" "${env_tmp}/${py_ver}_dev_environment.yml"
+    echo -e "\n  - python = $py_ver" >> "${env_tmp}/${py_ver}_dev_environment.yml"
+    conda-lock --channel conda-forge --kind explicit --file "${env_tmp}/${py_ver}_dev_environment.yml" --platform linux-64 --filename-template "requirements/locks/py${py_ver//.}-lock-linux-64.txt"
+done
+
+# Replace Met Office specific URLs with normal ones.
+sed -i "s|metoffice.jfrog.io/metoffice/api/conda|conda.anaconda.org|" requirements/locks/*.txt
+
+# Sort lock files to make diffs easier to review.
+for file in requirements/locks/*.txt
+do
+    # Leave the file header unsorted.
+    cat "${file}" | (sed -u 4q; sort) > sorted.txt
+    mv sorted.txt "${file}"
+done
+
+# Record the environment.yml used to build the environment.
+sha256sum requirements/environment.yml > requirements/locks/sources
+
+echo "Updating pre-commit hooks"
+pre-commit autoupdate --freeze
+
+echo "Cleaning up temporary files"
+# Clean up temporary environment definitions.
+rm -r "${env_tmp}"
+# Clean up virtual environment.
+deactivate
+rm -r "${venv_location}"