diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 054c251e0..c18c40fbf 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,6 +9,5 @@ Aim to have all relevant checks ticked off before merging. See the [developer's - [ ] All tests and CI checks pass. - [ ] Ensured the pull request title is descriptive. - [ ] Ensure `rose-suite.conf.example` has been updated if new diagnostic added. -- [ ] Conda lock files have been updated if dependencies have changed. - [ ] Attributed any Generative AI, such as GitHub Copilot, used in this PR. - [ ] Marked the PR as ready to review. diff --git a/.github/workflows/scheduled-updates.yml b/.github/workflows/scheduled-updates.yml index dceb0f6e5..0fb1068a3 100644 --- a/.github/workflows/scheduled-updates.yml +++ b/.github/workflows/scheduled-updates.yml @@ -10,6 +10,7 @@ permissions: {} jobs: scheduled-updates: + if: "github.repository == 'MetOffice/CSET'" # Skip on forks. name: Regular conda and pre-commit updates runs-on: ubuntu-latest timeout-minutes: 20 @@ -28,56 +29,27 @@ jobs: with: python-version: "3.x" - - name: Install dependencies - run: pip install conda-lock pre-commit - - name: Setup git user and branch run: | git config --local user.name "github-actions[bot]" git config --local user.email "github-actions[bot]@users.noreply.github.com" git switch -c "scheduled-updates-$(date +%s)" - - name: Update CSET developer and workflow lock files - run: | - py_vers="3.12 3.13 3.14" - for py_ver in ${py_vers} - do - # Developer lock files. - cp "requirements/environment.yml" "${RUNNER_TEMP}/${py_ver}_dev_environment.yml" - echo -e "\n - python = $py_ver" >> "${RUNNER_TEMP}/${py_ver}_dev_environment.yml" - conda-lock --channel conda-forge --kind explicit --file "${RUNNER_TEMP}/${py_ver}_dev_environment.yml" --platform linux-64 --filename-template "requirements/locks/py$(echo $py_ver | sed 's/\.//')-lock-linux-64.txt" - done - - # Sort lock files to make diffs easier to review. - for file in requirements/locks/*.txt - do - # Leave the file header unsorted. - cat "$file" | (sed -u 4q; sort) > sorted.txt - mv sorted.txt "$file" - done - - - name: Update pre-commit config - run: pre-commit autoupdate --freeze - - - name: Generate GitHub App Token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 - id: app-token - with: - app-id: ${{ secrets.AUTH_APP_ID }} - private-key: ${{ secrets.AUTH_APP_PRIVATE_KEY }} + - name: Update CSET developer lock files and pre-commit hooks + run: make update-dev-deps - name: Commit changes and create pull requests env: source_ref: ${{ github.ref }} source_ref_type: ${{ github.ref_type }} - gh_token: ${{ steps.app-token.outputs.token }} + gh_token: ${{ github.token }} repository: ${{ github.repository }} run: | original_commit="$(git rev-parse --verify HEAD)" git diff --stat # Commit changed lockfiles. - git add requirements/locks/*.txt + git add requirements/locks/ git commit -m "[CI] Update conda lock files" || true # Commit changed pre-commit config. diff --git a/.github/workflows/weekly-checks.yml b/.github/workflows/weekly-checks.yml index 753e04038..3c147036e 100644 --- a/.github/workflows/weekly-checks.yml +++ b/.github/workflows/weekly-checks.yml @@ -10,6 +10,7 @@ permissions: {} jobs: check-documentation-hyperlinks: + if: "github.repository == 'MetOffice/CSET'" # Skip on forks. name: Check documentation hyperlinks runs-on: ubuntu-latest # Give plenty of time as link checking third-party sites might be slow. @@ -43,6 +44,7 @@ jobs: run: sphinx-build -b linkcheck --color -W --keep-going "docs/source" "docs/build/linkcheck" full-tests: + if: "github.repository == 'MetOffice/CSET'" # Skip on forks. runs-on: ubuntu-latest timeout-minutes: 20 permissions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eedce0303..a27a3b4e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,6 +43,15 @@ repos: additional_dependencies: - jinja2 + - repo: local + hooks: + - id: check-conda-lockfiles + name: Check conda lockfiles are up-to-date + description: Ensure the conda lockfiles include new dependencies. + entry: ./scripts/check-conda-lockfiles.sh + language: script + files: requirements/ + - repo: https://github.com/pre-commit/pre-commit-hooks rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: diff --git a/Makefile b/Makefile index f30163bf4..d478d48b1 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,9 @@ test-full: pre-commit ## Run all tests, including slow or network reliant. playwright install --only-shell chromium pytest -vv --cov --cov-append --cov-config=pyproject.toml --numprocesses logical +update-dev-deps: ## Update pre-commit hooks and conda lock files for the development environment. + scripts/update-developer-dependencies.sh + # Mark targets as 'phony' to indicate they don't actually produce a file with # the same name as their target. Basically for actions rather than files. -.PHONY: help setup docs test test-fast test-full prepare-lockfiles conda +.PHONY: help setup docs test test-fast test-full prepare-lockfiles conda update-dev-deps diff --git a/docs/source/contributing/dependencies.rst b/docs/source/contributing/dependencies.rst index 590fb69fa..7441337e7 100644 --- a/docs/source/contributing/dependencies.rst +++ b/docs/source/contributing/dependencies.rst @@ -33,12 +33,15 @@ Specifically it needs to be added to the ``pyproject.toml`` under the "dependencies" key, and ``requirements/environment.yaml`` under the appropriate dependencies section. -After updating those two files and making a pull request, you'll need to rerun -the conda lockfile generation action. In `Actions > Update conda lock files > -Run workflow`_ select your branch, then run the workflow. A new PR will be -created to update the lockfiles, which you can merge into your own branch. +After updating those two files and making a pull request, you'll need to +regenerate the conda lockfiles. -.. _Actions > Update conda lock files > Run workflow: https://github.com/MetOffice/CSET/actions/workflows/conda-lock.yml +.. code-block:: bash + + # Regenerate the lock files. + make update-dev-deps + # Commit them to your branch. + git commit -m "Update conda lockfiles" requirements/ Updating the conda-forge package -------------------------------- diff --git a/docs/source/contributing/index.rst b/docs/source/contributing/index.rst index bacfaec87..ee7b64f87 100644 --- a/docs/source/contributing/index.rst +++ b/docs/source/contributing/index.rst @@ -64,16 +64,6 @@ The pull request title can be edited by clicking the "Edit" button to its right. If you need more text to describe what the pull request does, please add it in the description. -Conda lock files have been updated if dependencies changed -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you have changed the dependencies of CSET you will need to regenerate the -conda dependency lock files. This can be done by running the `update conda lock -files workflow`_ from your branch, then merging the PR it creates into your own -branch. (Not the default ``main`` branch!) - -.. _update conda lock files workflow: https://github.com/MetOffice/CSET/actions/workflows/conda-lock.yml - Attributed any Generative AI, such as GitHub Copilot, used in this PR ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/check-conda-lockfiles.sh b/scripts/check-conda-lockfiles.sh new file mode 100755 index 000000000..114117499 --- /dev/null +++ b/scripts/check-conda-lockfiles.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Check conda lockfiles are up-to-date. +set -eu + +if ! sha256sum -c requirements/locks/sources +then + echo "Conda lockfiles are outdated. Please run 'make update-dev-deps' to update them." + exit 1 +fi diff --git a/scripts/update-developer-dependencies.sh b/scripts/update-developer-dependencies.sh new file mode 100755 index 000000000..b2d000218 --- /dev/null +++ b/scripts/update-developer-dependencies.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Update conda lock files and pre-commit hooks. +set -euo pipefail + +# Create a temporary virtual environment. +venv_location="$(mktemp -d)" +echo "Creating temporary virtual environment at ${venv_location}" +python3 -m venv "${venv_location}" +# shellcheck disable=SC1091 +source "${venv_location}/bin/activate" + +echo "Installing update tools" +pip install --quiet conda-lock pre-commit + +# Create lock files for supported python versions. +py_vers="3.12 3.13 3.14" +env_tmp="$(mktemp -d)" +for py_ver in ${py_vers} +do + # Developer lock files. + echo "Updating lock files for python ${py_ver}" + cp "requirements/environment.yml" "${env_tmp}/${py_ver}_dev_environment.yml" + echo -e "\n - python = $py_ver" >> "${env_tmp}/${py_ver}_dev_environment.yml" + conda-lock --channel conda-forge --kind explicit --file "${env_tmp}/${py_ver}_dev_environment.yml" --platform linux-64 --filename-template "requirements/locks/py${py_ver//.}-lock-linux-64.txt" +done + +# Replace Met Office specific URLs with normal ones. +sed -i "s|metoffice.jfrog.io/metoffice/api/conda|conda.anaconda.org|" requirements/locks/*.txt + +# Sort lock files to make diffs easier to review. +for file in requirements/locks/*.txt +do + # Leave the file header unsorted. + cat "${file}" | (sed -u 4q; sort) > sorted.txt + mv sorted.txt "${file}" +done + +# Record the environment.yml used to build the environment. +sha256sum requirements/environment.yml > requirements/locks/sources + +echo "Updating pre-commit hooks" +pre-commit autoupdate --freeze + +echo "Cleaning up temporary files" +# Clean up temporary environment definitions. +rm -r "${env_tmp}" +# Clean up virtual environment. +deactivate +rm -r "${venv_location}"