From 927773cc80cc4a90e6732a021afc67690d4a7348 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Thu, 28 May 2026 20:48:22 +0200 Subject: [PATCH 1/7] Relocate infra to $CODE/infra; keep $CODE as the workspace root The repository was the workspace root, so its .git tracked every sibling code checkout as untracked noise and carried two accidental gitlinks (SIMPLE, libneo) with no .gitmodules. Make it the infra directory of a workspace. activate.sh, activate.fish and activate export INFRA (this repo) and derive CODE as its parent. Infra-owned paths (scripts, .venv, external, modules, images, local, lib, bin) move to $INFRA. $CODE stays the workspace root, so the codes keep resolving dependencies as $CODE/ via find_or_fetch. Sibling references ($CODE/libneo/build) and workspace navigation (cdcode, vscode) stay on $CODE; set_branch reads the infra repo via $INFRA. Drop the accidental SIMPLE and libneo gitlinks. Update setup.sh, the devcontainer build and postCreateCommand.sh, the release CI in main.yml, the STELLOPT and VMEC config paths, and the README to the new layout. --- .devcontainer/postCreateCommand.sh | 8 +-- .github/workflows/main.yml | 25 +++++----- README.md | 41 ++++++++++----- SIMPLE | 1 - activate.fish | 42 ++++++++-------- activate.sh | 50 ++++++++++--------- images/base/cross_build_push.sh | 2 +- images/devcontainer/cross_build_push.sh | 2 +- images/devel-tex/cross_build_push.sh | 2 +- images/devel/cross_build_push.sh | 2 +- libneo | 1 - scripts/docker_build_push.sh | 2 +- scripts/docker_build_push_all.sh | 8 +-- scripts/machines/setup_viper.sh | 2 +- scripts/setup.sh | 26 ++++++---- scripts/setup/ascot5.sh | 2 +- scripts/setup/cgal.sh | 4 +- scripts/setup/compiler_intel.sh | 4 +- scripts/setup/compiler_nvidia.sh | 8 +-- scripts/setup/devcontainer.sh | 9 ++-- scripts/setup/fftw.sh | 4 +- scripts/setup/fgsl.sh | 4 +- scripts/setup/gpec.sh | 2 +- scripts/setup/gsl.sh | 4 +- scripts/setup/lfortran.sh | 8 +-- scripts/setup/libneo.sh | 4 +- scripts/setup/mars.sh | 2 +- scripts/setup/mephit.sh | 4 +- scripts/setup/netcdf.sh | 10 ++-- scripts/setup/omfit.sh | 4 +- scripts/setup/openblas.sh | 4 +- scripts/setup/simsopt.sh | 2 +- scripts/setup/stellopt.sh | 21 ++++---- scripts/setup/suitesparse.sh | 2 +- scripts/setup/vmec2000.sh | 6 +-- scripts/util.sh | 4 +- .../benchmark_with_NEO_2.ipynb | 2 +- 37 files changed, 180 insertions(+), 148 deletions(-) delete mode 160000 SIMPLE delete mode 160000 libneo diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 98c2244..2d7bb6d 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -1,10 +1,12 @@ #!/bin/bash CODE_TEMPLATE=/usr/local/src/code_template -CODE=/workspaces/code +INFRA=/workspaces/code +CODE=/workspaces -ln -s $CODE_TEMPLATE/.venv $CODE/.venv -ln -s $CODE_TEMPLATE/external/fgsl-1.6.0 $CODE/external/fgsl-1.6.0 +# .venv and external belong to the infra repo; libneo is a sibling checkout +ln -s $CODE_TEMPLATE/.venv $INFRA/.venv +ln -s $CODE_TEMPLATE/external/fgsl-1.6.0 $INFRA/external/fgsl-1.6.0 cp -r $CODE_TEMPLATE/libneo $CODE/libneo echo 'source /workspaces/code/activate.sh' >> $HOME/.bashrc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6154a9d..75d87d5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,49 +77,50 @@ jobs: - name: Extract tarball run: | - tar -xzvf /tmp/code.tar.gz + mkdir -p infra + tar -xzvf /tmp/code.tar.gz -C infra - name: NEO-2 run: | - source activate.sh + source infra/activate.sh clone_github NEO-2 cd NEO-2 - $CODE/scripts/checkout_branch.sh $CODE_BRANCH + $INFRA/scripts/checkout_branch.sh $CODE_BRANCH make - name: NEO-RT run: | - source activate.sh + source infra/activate.sh clone_github NEO-RT cd NEO-RT - $CODE/scripts/checkout_branch.sh $CODE_BRANCH + $INFRA/scripts/checkout_branch.sh $CODE_BRANCH make - name: SIMPLE run: | - source activate.sh + source infra/activate.sh clone_github SIMPLE cd SIMPLE - $CODE/scripts/checkout_branch.sh $CODE_BRANCH + $INFRA/scripts/checkout_branch.sh $CODE_BRANCH make - name: GORILLA run: | - source activate.sh + source infra/activate.sh clone_github GORILLA cd GORILLA - $CODE/scripts/checkout_branch.sh $CODE_BRANCH + $INFRA/scripts/checkout_branch.sh $CODE_BRANCH make - name: MEPHIT run: | - source activate.sh + source infra/activate.sh clone_github MEPHIT source /etc/profile.d/modules.sh - module use -a $CODE/modules + module use -a $INFRA/modules module load mephit cd MEPHIT - $CODE/scripts/checkout_branch.sh $CODE_BRANCH + $INFRA/scripts/checkout_branch.sh $CODE_BRANCH make pip install -e . diff --git a/README.md b/README.md index 4669b05..0f0338b 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,22 @@ CODE is based around our standard Debian bookworm system at ITPcp and provides - Container definitions - VSCode settings +## Layout + +This repository is the `infra` directory of a workspace. The workspace is the +environment variable `$CODE`; this repository is `$INFRA`, equal to +`$CODE/infra`. The code checkouts sit in the workspace next to `infra`: + + $CODE/ + infra/ this repository (activation, setup scripts, modules, .venv) + libneo/ code checkout + SIMPLE/ code checkout + ... + +Activation exports both. `$INFRA` holds the infra-owned paths (`scripts`, +`.venv`, `external`, `modules`). `$CODE` is the workspace root, so the codes +resolve their dependencies as `$CODE/`. + ## Getting Started If you haven't done so earlier, set up your SSH keys in `~/.ssh` via `ssh-keygen` @@ -35,30 +51,31 @@ to install Debian Linux via WSL2. Then follow the Linux instructions. ### Initial setup -Clone the repository to your working copy, at the institute this is +Clone this repository into an `infra` directory inside your workspace. At the +institute the workspace is `/proj/plasma/CODE/`: - git clone git@github.com:itpplasma/code /proj/plasma/CODE/ + git clone git@github.com:itpplasma/code /proj/plasma/CODE//infra -Then open the directory in VS Code with +Open the workspace in VS Code with - code code + code /proj/plasma/CODE/ When asked to initialize the devcontainer, remove the message. Run the setup script manually with - scripts/setup.sh + infra/scripts/setup.sh -The setup will install external dependencies and create -a Python virtual environment in the hidden `.venv` directory. +The setup installs external dependencies and creates the Python virtual +environment in `infra/.venv`. Finally, activate the environment with - source activate.sh + source infra/activate.sh To use this environment as a standard, put the activation script into bashrc with - echo "source $PWD/activate.sh" >> ~/.bashrc + echo "source /proj/plasma/CODE//infra/activate.sh" >> ~/.bashrc ## External codes @@ -72,7 +89,7 @@ to install the compiler and libraries HDF5 and NetCDFwith modules. Then the comm scripts/setup/gpec.sh scripts/setup/mars.sh -will install GPEC and MARS into `$CODE/external/intel`. They can then be loaded with +will install GPEC and MARS into `$INFRA/external/intel`. They can then be loaded with module load gpec module load mars @@ -82,7 +99,7 @@ codes based on GNU Fortran in the same shell. ### OMFIT OMFIT requires its own Python environment provided via `conda`. Both will be installed -to `$CODE/external` by running +to `$INFRA/external` by running deactivate scripts/setup/omfit.sh @@ -90,7 +107,7 @@ to `$CODE/external` by running OMFIT can then be loaded with deactivate - source $CODE/external/mambaforge/bin/activate omfit + source $INFRA/external/mambaforge/bin/activate omfit module load omfit and then started with `omfit` in the shell. Be careful not to work on codes diff --git a/SIMPLE b/SIMPLE deleted file mode 160000 index 8483ed8..0000000 --- a/SIMPLE +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8483ed8890ac5c69d2aa7565ceb6fb4b65887a47 diff --git a/activate.fish b/activate.fish index c8a0056..bb7cf70 100644 --- a/activate.fish +++ b/activate.fish @@ -3,21 +3,23 @@ # Fish shell activation script # Usage: source activate.fish -# Get the directory containing this script (absolute, without changing PWD) +# INFRA is this repository; CODE is the workspace one level up that holds it +# next to the code checkouts. The codes resolve dependencies as $CODE/. set SCRIPT_DIR (dirname (status --current-filename)) if type -q path - set -gx CODE (path resolve $SCRIPT_DIR) + set -gx INFRA (path resolve $SCRIPT_DIR) else if type -q realpath - set -gx CODE (realpath $SCRIPT_DIR) + set -gx INFRA (realpath $SCRIPT_DIR) else # Fallback: temporarily cd and restore set -l __oldpwd $PWD cd $SCRIPT_DIR - set -gx CODE $PWD + set -gx INFRA $PWD cd $__oldpwd end +set -gx CODE (path resolve $INFRA/..) -echo "Setting CODE environment to: $CODE" +echo "Setting CODE workspace to: $CODE (infra: $INFRA)" # Check if the OS is macOS if test (uname) = "Darwin" @@ -36,8 +38,8 @@ if test (uname) = "Darwin" echo "Unknown processor architecture." end - set -gx CMAKE_INCLUDE_PATH "/opt/homebrew/include/suitesparse:$CODE/external/triangle" - set -gx CMAKE_LIBRARY_PATH "/opt/homebrew/lib:$CODE/external/triangle/build" + set -gx CMAKE_INCLUDE_PATH "/opt/homebrew/include/suitesparse:$INFRA/external/triangle" + set -gx CMAKE_LIBRARY_PATH "/opt/homebrew/lib:$INFRA/external/triangle/build" set -gx CMAKE_ARGS "-DBLAS_LIBRARIES=$BLAS_LIBRARIES -DLAPACK_LIBRARIES=$LAPACK_LIBRARIES" else set -gx CMAKE_ARGS "" @@ -76,31 +78,31 @@ function set_branch_fish else if test -n "$CI_COMMIT_REF_NAME" set -gx CODE_BRANCH $CI_COMMIT_REF_NAME else - # Avoid directory changes; query git directly in $CODE - set -gx CODE_BRANCH (git -C $CODE branch --show-current) + # Avoid directory changes; query git directly in $INFRA + set -gx CODE_BRANCH (git -C $INFRA branch --show-current) if test -z "$CODE_BRANCH" - set -gx CODE_BRANCH (git -C $CODE rev-parse --short HEAD) + set -gx CODE_BRANCH (git -C $INFRA rev-parse --short HEAD) end end - echo "Activating $CODE on branch $CODE_BRANCH" + echo "Activating $INFRA on branch $CODE_BRANCH" end # Set up paths set_branch_fish -add_to_path_fish $CODE/scripts -add_to_path_fish $CODE/local/bin -add_to_path_fish $CODE/bin +add_to_path_fish $INFRA/scripts +add_to_path_fish $INFRA/local/bin +add_to_path_fish $INFRA/bin add_to_library_path_fish $CODE/libneo/build -add_to_library_path_fish $CODE/local/lib -add_to_library_path_fish $CODE/lib +add_to_library_path_fish $INFRA/local/lib +add_to_library_path_fish $INFRA/lib # Activate Python virtual environment -if test -f $CODE/.venv/bin/activate.fish - source $CODE/.venv/bin/activate.fish +if test -f $INFRA/.venv/bin/activate.fish + source $INFRA/.venv/bin/activate.fish else echo "Warning: Python virtual environment fish activation script not found" - echo "Run: python -m venv $CODE/.venv to create it" + echo "Run: python -m venv $INFRA/.venv to create it" end # Load modules if available @@ -115,6 +117,6 @@ if command -v code > /dev/null alias vscode="code $CODE" end -module use -a $CODE/modules +module use -a $INFRA/modules echo "Fish shell activation complete!" diff --git a/activate.sh b/activate.sh index 4e17102..899a7e5 100644 --- a/activate.sh +++ b/activate.sh @@ -12,7 +12,11 @@ else SCRIPT_SOURCE="$0" fi -export CODE="$( cd "$( dirname "$SCRIPT_SOURCE" )" && pwd )" +# INFRA is this repository. CODE is the workspace one level up that holds it +# next to the actual code checkouts (libneo, SIMPLE, ...). The codes resolve +# their dependencies as $CODE/ (find_or_fetch), so CODE is the parent. +export INFRA="$( cd "$( dirname "$SCRIPT_SOURCE" )" && pwd )" +export CODE="$( cd "$INFRA/.." && pwd )" # Check if the OS is macOS if [ "$(uname)" = "Darwin" ]; then @@ -31,15 +35,15 @@ if [ "$(uname)" = "Darwin" ]; then echo "Unknown processor architecture." fi - export CMAKE_INCLUDE_PATH="/opt/homebrew/include/suitesparse:$CODE/external/triangle" - export CMAKE_LIBRARY_PATH="/opt/homebrew/lib:$CODE/external/triangle/build" + export CMAKE_INCLUDE_PATH="/opt/homebrew/include/suitesparse:$INFRA/external/triangle" + export CMAKE_LIBRARY_PATH="/opt/homebrew/lib:$INFRA/external/triangle/build" export CMAKE_ARGS="-DBLAS_LIBRARIES=$BLAS_LIBRARIES -DLAPACK_LIBRARIES=$LAPACK_LIBRARIES" else export CMAKE_ARGS="" # Use locally built OpenBLAS if system BLAS headers not available if [ ! -f /usr/include/cblas.h ] && [ ! -f /usr/include/openblas/cblas.h ]; then - OPENBLAS_PREFIX="$CODE/external/OpenBLAS-0.3.28/install" + OPENBLAS_PREFIX="$INFRA/external/OpenBLAS-0.3.28/install" if [ -d "$OPENBLAS_PREFIX" ]; then export BLAS_LIBRARIES="$OPENBLAS_PREFIX/lib/libopenblas.so" export LAPACK_LIBRARIES="$OPENBLAS_PREFIX/lib/libopenblas.so" @@ -49,30 +53,30 @@ else fi # Use locally built GSL only if system GSL not available - if ! command -v gsl-config &> /dev/null && [ -d "$CODE/external/gsl-2.8/install" ]; then - export GSL_ROOT_DIR="$CODE/external/gsl-2.8/install" + if ! command -v gsl-config &> /dev/null && [ -d "$INFRA/external/gsl-2.8/install" ]; then + export GSL_ROOT_DIR="$INFRA/external/gsl-2.8/install" export PKG_CONFIG_PATH="$GSL_ROOT_DIR/lib/pkgconfig:$PKG_CONFIG_PATH" fi # Use locally built FFTW if system headers not available - if [ ! -f /usr/include/fftw3.h ] && [ -d "$CODE/external/fftw-3.3.10/install" ]; then - export FFTW_ROOT="$CODE/external/fftw-3.3.10/install" + if [ ! -f /usr/include/fftw3.h ] && [ -d "$INFRA/external/fftw-3.3.10/install" ]; then + export FFTW_ROOT="$INFRA/external/fftw-3.3.10/install" fi # Use locally built HDF5/NetCDF if system nf-config not available - if ! command -v nf-config &> /dev/null && [ -d "$CODE/external/netcdf-install" ]; then - export HDF5_ROOT="$CODE/external/netcdf-install" - export NETCDF_ROOT="$CODE/external/netcdf-install" - export PATH="$CODE/external/netcdf-install/bin:$PATH" + if ! command -v nf-config &> /dev/null && [ -d "$INFRA/external/netcdf-install" ]; then + export HDF5_ROOT="$INFRA/external/netcdf-install" + export NETCDF_ROOT="$INFRA/external/netcdf-install" + export PATH="$INFRA/external/netcdf-install/bin:$PATH" fi # Use locally installed NVIDIA HPC SDK if available (optional) - NVHPC_ROOT="$CODE/external/nvhpc/Linux_x86_64" + NVHPC_ROOT="$INFRA/external/nvhpc/Linux_x86_64" if [ -d "$NVHPC_ROOT" ]; then # Find the installed version NVHPC_VERSION=$(ls "$NVHPC_ROOT" 2>/dev/null | grep -E '^[0-9]+\.[0-9]+$' | sort -V | tail -1) if [ -n "$NVHPC_VERSION" ]; then - export NVHPC="$CODE/external/nvhpc" + export NVHPC="$INFRA/external/nvhpc" export NVHPC_ROOT="$NVHPC_ROOT/$NVHPC_VERSION" export PATH="$NVHPC_ROOT/compilers/bin:$PATH" export PATH="$NVHPC_ROOT/comm_libs/mpi/bin:$PATH" @@ -86,22 +90,22 @@ if [ -n "$FISH_VERSION" ]; then # Fish shell detected - redirect to dedicated fish script echo "Fish shell detected!" echo "Please use the dedicated fish script instead:" - echo " source $CODE/activate.fish" + echo " source $INFRA/activate.fish" echo "" echo "This script (activate.sh) is designed for bash/zsh compatibility." return 1 else # Bash/Zsh setup (original behavior) - source $CODE/scripts/util.sh + source $INFRA/scripts/util.sh set_branch - add_to_path $CODE/scripts - add_to_path $CODE/local/bin - add_to_path $CODE/bin + add_to_path $INFRA/scripts + add_to_path $INFRA/local/bin + add_to_path $INFRA/bin export PATH add_to_library_path $CODE/libneo/build - add_to_library_path $CODE/local/lib - add_to_library_path $CODE/lib + add_to_library_path $INFRA/local/lib + add_to_library_path $INFRA/lib if [ -n "$GSL_ROOT_DIR" ]; then add_to_library_path $GSL_ROOT_DIR/lib add_to_path $GSL_ROOT_DIR/bin @@ -126,7 +130,7 @@ else fi export LD_LIBRARY_PATH - source $CODE/.venv/bin/activate + source $INFRA/.venv/bin/activate if [ -f /etc/profile.d/modules.sh ]; then unalias ml 2>/dev/null @@ -134,6 +138,6 @@ else fi if command -v module >/dev/null 2>&1; then - module use -a $CODE/modules + module use -a $INFRA/modules fi fi diff --git a/images/base/cross_build_push.sh b/images/base/cross_build_push.sh index 5f5bcb8..fd9bdc9 100755 --- a/images/base/cross_build_push.sh +++ b/images/base/cross_build_push.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash docker pull debian:bookworm-slim -docker buildx build --platform linux/amd64,linux/arm64 -f $CODE/images/base/Dockerfile -t ghcr.io/itpplasma/base . --push +docker buildx build --platform linux/amd64,linux/arm64 -f $INFRA/images/base/Dockerfile -t ghcr.io/itpplasma/base . --push diff --git a/images/devcontainer/cross_build_push.sh b/images/devcontainer/cross_build_push.sh index fe00bdf..7db59fe 100755 --- a/images/devcontainer/cross_build_push.sh +++ b/images/devcontainer/cross_build_push.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash docker pull ghcr.io/itpplasma/devcontainer -docker buildx build --platform linux/amd64,linux/arm64 -f $CODE/images/devcontainer/Dockerfile -t ghcr.io/itpplasma/devcontainer . --push +docker buildx build --platform linux/amd64,linux/arm64 -f $INFRA/images/devcontainer/Dockerfile -t ghcr.io/itpplasma/devcontainer . --push diff --git a/images/devel-tex/cross_build_push.sh b/images/devel-tex/cross_build_push.sh index 9df3ce7..70bdb5f 100755 --- a/images/devel-tex/cross_build_push.sh +++ b/images/devel-tex/cross_build_push.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash docker pull ghcr.io/itpplasma/devel -docker buildx build --platform linux/amd64,linux/arm64 -f $CODE/images/devel-tex/Dockerfile -t ghcr.io/itpplasma/devel-tex . --push +docker buildx build --platform linux/amd64,linux/arm64 -f $INFRA/images/devel-tex/Dockerfile -t ghcr.io/itpplasma/devel-tex . --push diff --git a/images/devel/cross_build_push.sh b/images/devel/cross_build_push.sh index 6277f25..e1cbe38 100755 --- a/images/devel/cross_build_push.sh +++ b/images/devel/cross_build_push.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash docker pull ghcr.io/itpplasma/base -docker buildx build --platform linux/amd64,linux/arm64 -f $CODE/images/devel/Dockerfile -t ghcr.io/itpplasma/devel . --push +docker buildx build --platform linux/amd64,linux/arm64 -f $INFRA/images/devel/Dockerfile -t ghcr.io/itpplasma/devel . --push diff --git a/libneo b/libneo deleted file mode 160000 index d3ecd23..0000000 --- a/libneo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d3ecd23926973faeb5294d16273b36402373e4bd diff --git a/scripts/docker_build_push.sh b/scripts/docker_build_push.sh index f3f63b3..759eada 100755 --- a/scripts/docker_build_push.sh +++ b/scripts/docker_build_push.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/itpplasma/$1 -f $CODE/images/$1/Dockerfile . --push +docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/itpplasma/$1 -f $INFRA/images/$1/Dockerfile . --push diff --git a/scripts/docker_build_push_all.sh b/scripts/docker_build_push_all.sh index 78a5946..ebbaa9d 100755 --- a/scripts/docker_build_push_all.sh +++ b/scripts/docker_build_push_all.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -$CODE/images/base/cross_build_push.sh -$CODE/images/devel/cross_build_push.sh -$CODE/images/devel-tex/cross_build_push.sh -$CODE/images/devcontainer/cross_build_push.sh +$INFRA/images/base/cross_build_push.sh +$INFRA/images/devel/cross_build_push.sh +$INFRA/images/devel-tex/cross_build_push.sh +$INFRA/images/devcontainer/cross_build_push.sh docker pull ghcr.io/itpplasma/devcontainer diff --git a/scripts/machines/setup_viper.sh b/scripts/machines/setup_viper.sh index cbb0322..d80a27d 100644 --- a/scripts/machines/setup_viper.sh +++ b/scripts/machines/setup_viper.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source $CODE/machines/init_viper.sh +source $INFRA/machines/init_viper.sh mkdir -p $HOME/.local/lib ln -s $MKL_PARTS_HOME/lib/libmkl_blas.so $HOME/.local/lib/libblas.so diff --git a/scripts/setup.sh b/scripts/setup.sh index 4712452..f755817 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -4,21 +4,25 @@ set -e SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -export CODE=$SCRIPTPATH/.. +export INFRA="$( cd "$SCRIPTPATH/.." && pwd )" +export CODE="$( cd "$INFRA/.." && pwd )" export GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new" -$CODE/scripts/setup/venv.sh -source $CODE/.venv/bin/activate +# venv and requirements.txt live at the infra root +pushd $INFRA + $INFRA/scripts/setup/venv.sh + source $INFRA/.venv/bin/activate +popd -pushd $CODE/external - $CODE/scripts/setup/openblas.sh - $CODE/scripts/setup/gsl.sh - $CODE/scripts/setup/fgsl.sh - $CODE/scripts/setup/fftw.sh - $CODE/scripts/setup/netcdf.sh - $CODE/scripts/setup/triangle.sh +pushd $INFRA/external + $INFRA/scripts/setup/openblas.sh + $INFRA/scripts/setup/gsl.sh + $INFRA/scripts/setup/fgsl.sh + $INFRA/scripts/setup/fftw.sh + $INFRA/scripts/setup/netcdf.sh + $INFRA/scripts/setup/triangle.sh popd pushd $CODE - $CODE/scripts/setup/libneo.sh + $INFRA/scripts/setup/libneo.sh popd diff --git a/scripts/setup/ascot5.sh b/scripts/setup/ascot5.sh index 834b63d..df2b7fc 100755 --- a/scripts/setup/ascot5.sh +++ b/scripts/setup/ascot5.sh @@ -2,7 +2,7 @@ set -e -source $CODE/scripts/util.sh +source $INFRA/scripts/util.sh echo "Setting up ASCOT5..." diff --git a/scripts/setup/cgal.sh b/scripts/setup/cgal.sh index 9329ad4..ede584c 100755 --- a/scripts/setup/cgal.sh +++ b/scripts/setup/cgal.sh @@ -4,11 +4,11 @@ set -e -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 CGAL_VERSION="5.6.1" CGAL_DIR="CGAL-${CGAL_VERSION}" -CGAL_INSTALL="$CODE/external/cgal" +CGAL_INSTALL="$INFRA/external/cgal" if [ ! -d "$CGAL_INSTALL/include/CGAL" ]; then echo "Fetching and installing CGAL ${CGAL_VERSION}..." diff --git a/scripts/setup/compiler_intel.sh b/scripts/setup/compiler_intel.sh index 2ec892d..2ec0964 100755 --- a/scripts/setup/compiler_intel.sh +++ b/scripts/setup/compiler_intel.sh @@ -44,6 +44,6 @@ export FC=ifx export CC=icx export CXX=icpx -pushd $CODE/external/intel - source $CODE/scripts/setup/netcdf.sh +pushd $INFRA/external/intel + source $INFRA/scripts/setup/netcdf.sh popd diff --git a/scripts/setup/compiler_nvidia.sh b/scripts/setup/compiler_nvidia.sh index 50f245c..3628a1d 100755 --- a/scripts/setup/compiler_nvidia.sh +++ b/scripts/setup/compiler_nvidia.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # NVIDIA HPC SDK setup script (optional - not included in default setup.sh) -# Installs to $CODE/external/nvhpc without requiring root access +# Installs to $INFRA/external/nvhpc without requiring root access set -e @@ -8,9 +8,9 @@ NVHPC_VERSION=25.11 NVHPC_YEAR=2025 CUDA_VERSION=13.0 -INSTALL_DIR="$CODE/external/nvhpc" +INSTALL_DIR="$INFRA/external/nvhpc" -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 # Check if already installed if [ -f "$INSTALL_DIR/Linux_x86_64/$NVHPC_VERSION/compilers/bin/nvfortran" ]; then @@ -47,7 +47,7 @@ export NVHPC_INSTALL_LOCAL_DIR="" ./install -cd "$CODE/external" +cd "$INFRA/external" # Verify installation if [ -f "$INSTALL_DIR/Linux_x86_64/$NVHPC_VERSION/compilers/bin/nvfortran" ]; then diff --git a/scripts/setup/devcontainer.sh b/scripts/setup/devcontainer.sh index 93747bb..b32c429 100755 --- a/scripts/setup/devcontainer.sh +++ b/scripts/setup/devcontainer.sh @@ -2,17 +2,20 @@ export GIT_HTTPS=1 export CODE_TEMPLATE=/usr/local/src/code_template -export CODE=/workspaces/code mkdir -p /workspaces mkdir -p /usr/local/src +# infra repo at /workspaces/code, code checkouts as siblings under /workspaces cd /workspaces git clone https://github.com/itpplasma/code.git code -cd $CODE +cd /workspaces/code source scripts/setup.sh -mv $CODE $CODE_TEMPLATE +# setup.sh clones libneo to the workspace ($CODE = /workspaces); stash it inside +# the infra template so postCreateCommand.sh can restore it as a sibling. +mv /workspaces/libneo /workspaces/code/libneo +mv /workspaces/code $CODE_TEMPLATE rm -rf /workspaces diff --git a/scripts/setup/fftw.sh b/scripts/setup/fftw.sh index c5a9ebd..54a5d51 100755 --- a/scripts/setup/fftw.sh +++ b/scripts/setup/fftw.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 FFTW_VERSION=3.3.10 -INSTALL_PREFIX="$CODE/external/fftw-${FFTW_VERSION}/install" +INSTALL_PREFIX="$INFRA/external/fftw-${FFTW_VERSION}/install" # Check if system FFTW is available if [ -f /usr/include/fftw3.h ] || [ -f /usr/local/include/fftw3.h ]; then diff --git a/scripts/setup/fgsl.sh b/scripts/setup/fgsl.sh index bd3b07c..d13cc68 100755 --- a/scripts/setup/fgsl.sh +++ b/scripts/setup/fgsl.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 if [ ! -f "fgsl-1.6.0/.libs/libfgsl.a" ] ; then echo "Fetching and building FGSL..." @@ -9,7 +9,7 @@ if [ ! -f "fgsl-1.6.0/.libs/libfgsl.a" ] ; then if command -v gsl-config &> /dev/null; then GSL_CONFIG=gsl-config else - GSL_PREFIX=$CODE/external/gsl-2.8/install + GSL_PREFIX=$INFRA/external/gsl-2.8/install if [ -d "$GSL_PREFIX" ]; then GSL_CONFIG="$GSL_PREFIX/bin/gsl-config" export LD_LIBRARY_PATH="$GSL_PREFIX/lib:$LD_LIBRARY_PATH" diff --git a/scripts/setup/gpec.sh b/scripts/setup/gpec.sh index 49db761..96d8059 100755 --- a/scripts/setup/gpec.sh +++ b/scripts/setup/gpec.sh @@ -10,7 +10,7 @@ export CC=icx export CXX=icpx export FC=ifx export OMPFLAG="-qopenmp" -cd $CODE/external/intel +cd $INFRA/external/intel ../../scripts/setup/netcdf.sh diff --git a/scripts/setup/gsl.sh b/scripts/setup/gsl.sh index a80e0b7..c108dcb 100755 --- a/scripts/setup/gsl.sh +++ b/scripts/setup/gsl.sh @@ -2,7 +2,7 @@ GSL_VERSION=2.8 -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 if command -v gsl-config &> /dev/null; then echo "System GSL found: $(gsl-config --version)" @@ -12,7 +12,7 @@ elif [ ! -f "gsl-${GSL_VERSION}/install/lib/libgsl.a" ]; then curl -L https://ftp.gnu.org/gnu/gsl/gsl-${GSL_VERSION}.tar.gz -o - | tar xz fi pushd gsl-${GSL_VERSION} - ./configure --prefix=$CODE/external/gsl-${GSL_VERSION}/install + ./configure --prefix=$INFRA/external/gsl-${GSL_VERSION}/install make -j$(nproc) make install popd diff --git a/scripts/setup/lfortran.sh b/scripts/setup/lfortran.sh index 8942662..4155c9e 100755 --- a/scripts/setup/lfortran.sh +++ b/scripts/setup/lfortran.sh @@ -6,7 +6,7 @@ sudo apt update && sudo apt install -y --no-install-recommends llvm-dev curl -L https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz -o - | tar xz mkdir -p json-3.11.3/build pushd json-3.11.3/build -cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$CODE/.venv +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INFRA/.venv make -j$(nproc) make install popd @@ -14,7 +14,7 @@ popd curl -L https://github.com/jupyter-xeus/xeus/archive/refs/tags/5.1.0.tar.gz -o - | tar xz mkdir -p xeus-5.1.0/build pushd xeus-5.1.0/build -cmake .. -D CMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$CODE/.venv +cmake .. -D CMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INFRA/.venv make -j$(nproc) make install popd @@ -22,7 +22,7 @@ popd curl -L https://github.com/jupyter-xeus/xeus-zmq/archive/refs/tags/3.0.0.tar.gz -o - | tar xz mkdir -p xeus-zmq-3.0.0/build pushd xeus-zmq-3.0.0/build -cmake .. -D CMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$CODE/.venv +cmake .. -D CMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$INFRA/.venv make -j$(nproc) make install popd @@ -30,7 +30,7 @@ popd curl -L https://github.com/lfortran/lfortran/releases/download/v0.37.0/lfortran-0.37.0.tar.gz -o - | tar xz mkdir -p lfortran-0.37.0/build pushd lfortran-0.37.0/build -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_LLVM=yes -DCMAKE_INSTALL_PREFIX=$CODE/.venv -DWITH_XEUS=yes +cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_LLVM=yes -DCMAKE_INSTALL_PREFIX=$INFRA/.venv -DWITH_XEUS=yes make -j$(nproc) make install popd diff --git a/scripts/setup/libneo.sh b/scripts/setup/libneo.sh index 5227499..7ee1551 100755 --- a/scripts/setup/libneo.sh +++ b/scripts/setup/libneo.sh @@ -2,7 +2,7 @@ set -e -source $CODE/scripts/util.sh +source $INFRA/scripts/util.sh set_branch echo "Building and installing 'libneo'..." @@ -11,6 +11,6 @@ if [ ! -d "libneo" ] ; then clone_github libneo fi pushd libneo -$CODE/scripts/checkout_branch.sh $CODE_BRANCH +$INFRA/scripts/checkout_branch.sh $CODE_BRANCH pip install --verbose --no-build-isolation -e . popd diff --git a/scripts/setup/mars.sh b/scripts/setup/mars.sh index 8b3608a..3ed3339 100755 --- a/scripts/setup/mars.sh +++ b/scripts/setup/mars.sh @@ -2,7 +2,7 @@ set -e module load intel/compiler-rt/2024.2.1 intel/ifort intel/mpi -cd $CODE/external/intel +cd $INFRA/external/intel git clone git@gitlab.tugraz.at:plasma/codes/mars.git MARS cd MARS/MarsQ_2022 make diff --git a/scripts/setup/mephit.sh b/scripts/setup/mephit.sh index 964a0ed..896024b 100755 --- a/scripts/setup/mephit.sh +++ b/scripts/setup/mephit.sh @@ -2,11 +2,11 @@ set -e source /etc/profile.d/modules.sh -module use -a $CODE/modules +module use -a $INFRA/modules module load mephit pushd MEPHIT - $CODE/scripts/checkout_branch.sh $CODE_BRANCH + $INFRA/scripts/checkout_branch.sh $CODE_BRANCH make pip install -e . popd diff --git a/scripts/setup/netcdf.sh b/scripts/setup/netcdf.sh index bb623ae..9105e18 100755 --- a/scripts/setup/netcdf.sh +++ b/scripts/setup/netcdf.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 HDF5_VERSION=1.14.5 NETCDF_C_VERSION=4.9.2 NETCDF_F_VERSION=4.6.1 -INSTALL_PREFIX="$CODE/external/netcdf-install" +INSTALL_PREFIX="$INFRA/external/netcdf-install" # Check if system NetCDF-Fortran is available if command -v nf-config &> /dev/null; then @@ -40,7 +40,7 @@ if [ ! -f "$INSTALL_PREFIX/lib/libhdf5.so" ]; then -DBUILD_SHARED_LIBS=ON \ -DHDF5_BUILD_FORTRAN=ON .. make -j$(nproc) install - cd "$CODE/external" + cd "$INFRA/external" fi # Build NetCDF-C @@ -54,7 +54,7 @@ if [ ! -f "$INSTALL_PREFIX/lib/libnetcdf.so" ]; then LDFLAGS="-L$INSTALL_PREFIX/lib -Wl,-rpath,$INSTALL_PREFIX/lib" \ ./configure --prefix="$INSTALL_PREFIX" --disable-libxml2 --disable-byterange make -j$(nproc) install - cd "$CODE/external" + cd "$INFRA/external" fi # Build NetCDF-Fortran @@ -70,7 +70,7 @@ if [ ! -f "$INSTALL_PREFIX/lib/libnetcdff.so" ]; then -DNETCDF_C_INCLUDE_DIR="$INSTALL_PREFIX/include" \ -DCMAKE_PREFIX_PATH="$INSTALL_PREFIX" .. make -j$(nproc) install - cd "$CODE/external" + cd "$INFRA/external" fi echo "NetCDF stack installed to $INSTALL_PREFIX" diff --git a/scripts/setup/omfit.sh b/scripts/setup/omfit.sh index 68ee35f..8e9aae6 100755 --- a/scripts/setup/omfit.sh +++ b/scripts/setup/omfit.sh @@ -5,10 +5,10 @@ echo "Requires personal Git access to OMFIT repository." echo "Code access: https://omfit.io/install.html" export USER=`whoami` -cd $CODE/external +cd $INFRA/external git clone --filter=blob:none -b unstable git@gitlab.tugraz.at:plasma/codes/OMFIT-source.git cd OMFIT-source git submodule update --init omas install/install.sh mkdir -p $HOME/.LICENSES -cp $CODE/scripts/setup/omfit/LICENSES/* $HOME/.LICENSES/ +cp $INFRA/scripts/setup/omfit/LICENSES/* $HOME/.LICENSES/ diff --git a/scripts/setup/openblas.sh b/scripts/setup/openblas.sh index 14d024a..7731523 100755 --- a/scripts/setup/openblas.sh +++ b/scripts/setup/openblas.sh @@ -2,7 +2,7 @@ OPENBLAS_VERSION=0.3.28 -cd "$CODE/external" || exit 1 +cd "$INFRA/external" || exit 1 # Check if system BLAS/LAPACK dev headers exist if [ -f /usr/include/cblas.h ] || [ -f /usr/include/openblas/cblas.h ]; then @@ -15,5 +15,5 @@ elif [ ! -f "OpenBLAS-${OPENBLAS_VERSION}/install/lib/libopenblas.a" ]; then cd OpenBLAS-${OPENBLAS_VERSION} # Build with ZEN target for AMD EPYC make -j$(nproc) TARGET=ZEN USE_OPENMP=1 - make PREFIX=$CODE/external/OpenBLAS-${OPENBLAS_VERSION}/install install + make PREFIX=$INFRA/external/OpenBLAS-${OPENBLAS_VERSION}/install install fi diff --git a/scripts/setup/simsopt.sh b/scripts/setup/simsopt.sh index 2dfe367..a20bae7 100755 --- a/scripts/setup/simsopt.sh +++ b/scripts/setup/simsopt.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $CODE/external +cd $INFRA/external if [ ! -d "simsopt" ] ; then echo "Fetching and building simsopt..." diff --git a/scripts/setup/stellopt.sh b/scripts/setup/stellopt.sh index a8e09ae..55b30b7 100755 --- a/scripts/setup/stellopt.sh +++ b/scripts/setup/stellopt.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash set -e -# Ensure CODE_ROOT is exported for make configs (can't use CODE - build_all uses it) -export CODE_ROOT="$CODE" +# CODE_ROOT feeds the STELLOPT make configs, which read $(CODE_ROOT)/external/... +# That tree lives in the infra repo, so point it at $INFRA. +export CODE_ROOT="$INFRA" # Create ~/bin for libstell symlinks and ensure it's in PATH mkdir -p ~/bin @@ -17,24 +18,24 @@ fi select_machine() { case "$(uname -s)" in Darwin) - cp $CODE/scripts/setup/stellopt/make_osx_brew_m1.inc $CODE/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_osx_brew_m1.inc $INFRA/external/STELLOPT/SHARE export MACHINE=osx_brew_m1 ;; Linux) # Check for VSC5 cluster (hostname contains vsc or l5 node pattern) if [[ "$(hostname)" == *vsc* ]] || [[ "$(hostname)" == l5* ]] || [[ -n "$VSC_INSTITUTE" ]]; then - cp $CODE/scripts/setup/stellopt/make_vsc5.inc $CODE/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_vsc5.inc $INFRA/external/STELLOPT/SHARE export MACHINE=vsc5 # Check for scluster elif [[ "$(hostname)" == scluster* ]]; then - cp $CODE/scripts/setup/stellopt/make_scluster.inc $CODE/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_scluster.inc $INFRA/external/STELLOPT/SHARE export MACHINE=scluster # Detect distro family from /etc/os-release elif [ -f /etc/os-release ]; then . /etc/os-release case "$ID" in arch|manjaro|cachyos|endeavouros|garuda|artix) - cp $CODE/scripts/setup/stellopt/make_arch_linux.inc $CODE/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $INFRA/external/STELLOPT/SHARE export MACHINE=arch_linux ;; rhel|centos|fedora|almalinux|rocky) @@ -46,7 +47,7 @@ select_machine() { *) case "$ID_LIKE" in *arch*) - cp $CODE/scripts/setup/stellopt/make_arch_linux.inc $CODE/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $INFRA/external/STELLOPT/SHARE export MACHINE=arch_linux ;; *rhel*|*centos*|*fedora*) @@ -58,7 +59,7 @@ select_machine() { *) # Fallback: check for pacman (Arch-based) if command -v pacman &>/dev/null; then - cp $CODE/scripts/setup/stellopt/make_arch_linux.inc $CODE/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $INFRA/external/STELLOPT/SHARE export MACHINE=arch_linux else export MACHINE=ubuntu @@ -79,7 +80,7 @@ select_machine() { } apply_patches() { - local patches_dir="$CODE/scripts/setup/stellopt/patches" + local patches_dir="$INFRA/scripts/setup/stellopt/patches" if [ -d "$patches_dir" ]; then for patch in "$patches_dir"/*.patch; do [ -f "$patch" ] || continue @@ -93,7 +94,7 @@ apply_patches() { fi } -cd $CODE/external +cd $INFRA/external if [ ! -d "STELLOPT" ] ; then echo "Cloning STELLOPT..." diff --git a/scripts/setup/suitesparse.sh b/scripts/setup/suitesparse.sh index 1018438..62b32f9 100755 --- a/scripts/setup/suitesparse.sh +++ b/scripts/setup/suitesparse.sh @@ -1,6 +1,6 @@ #!/bin/sh -cd $CODE/external +cd $INFRA/external git clone https://github.com/DrTimothyAldenDavis/SuiteSparse cd SuiteSparse/build git checkout v7.10.2 diff --git a/scripts/setup/vmec2000.sh b/scripts/setup/vmec2000.sh index 58b7d95..4aa0695 100755 --- a/scripts/setup/vmec2000.sh +++ b/scripts/setup/vmec2000.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $CODE/external +cd $INFRA/external if [ ! -d "VMEC2000" ] ; then echo "Fetching and building VMEC2000..." @@ -8,8 +8,8 @@ if [ ! -d "VMEC2000" ] ; then fi pushd VMEC2000 -cp $CODE/scripts/setup/cmake_config_file_vmec2000.json cmake_config_file.json -sed -i "s|{CODE}|$CODE|g" cmake_config_file.json +cp $INFRA/scripts/setup/cmake_config_file_vmec2000.json cmake_config_file.json +sed -i "s|{CODE}|$INFRA|g" cmake_config_file.json cmake -S. -Bbuild -GNinja -DNETCDF_INC_PATH=/usr/include -DSCALAPACK_LIB_NAME=libscalapack-openmpi.so -DBLA_VENDOR=OpenBLAS pushd build ninja diff --git a/scripts/util.sh b/scripts/util.sh index d4886e0..a43740a 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -17,12 +17,12 @@ set_branch() { elif [ -n "$CI_COMMIT_REF_NAME" ]; then export CODE_BRANCH=$CI_COMMIT_REF_NAME else - pushd $CODE + pushd $INFRA export CODE_BRANCH=$(git branch --show-current) popd fi - echo "Activating $CODE on branch $CODE_BRANCH" + echo "Activating $INFRA on branch $CODE_BRANCH" } add_to_path() { diff --git a/tests/NEO-RT/benchmark_with_NEO_2/benchmark_with_NEO_2.ipynb b/tests/NEO-RT/benchmark_with_NEO_2/benchmark_with_NEO_2.ipynb index 64f93f2..a2558cc 100644 --- a/tests/NEO-RT/benchmark_with_NEO_2/benchmark_with_NEO_2.ipynb +++ b/tests/NEO-RT/benchmark_with_NEO_2/benchmark_with_NEO_2.ipynb @@ -20,7 +20,7 @@ "surfs = np.arange(0.78, 1.00, 0.02)\n", "runs = [f\"{surf:.2f}\".replace(\".\", \"p\") for surf in surfs]\n", "\n", - "run_dir = os.path.join(os.path.expandvars(\"$CODE/tests/NEO-RT/benchmark_with_NEO_2/RUN\"))\n", + "run_dir = os.path.join(os.path.expandvars(\"$INFRA/tests/NEO-RT/benchmark_with_NEO_2/RUN\"))\n", "if os.path.isdir(run_dir):\n", " os.system(f'rm -r {run_dir}')\n", "os.makedirs(run_dir)" From d91b1af5f1ac85669201d84ee0a1dcafda205095 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Thu, 28 May 2026 21:26:04 +0200 Subject: [PATCH 2/7] CI: overlay current infra sources onto the prebuilt-deps release main.yml downloaded a pinned release tarball whose frozen activate.sh predates the infra relocation, so $INFRA was unset and $INFRA/scripts/checkout_branch.sh resolved to /scripts/... (exit 127). Check out the branch and rsync its sources over the tarball, keeping the release's prebuilt .venv and external, and relocate the venv shebangs to the new path. --- .github/workflows/main.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75d87d5..5df7917 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,14 +71,30 @@ jobs: libmetis-dev \ libboost-all-dev + - name: Checkout infra sources + uses: actions/checkout@v4 + with: + path: infra-src + - name: Download Release # TODO: make this dynamic run: | wget https://github.com/itpplasma/code/releases/download/v2025.01.02/code-v2025.01.02.tar.gz -O /tmp/code.tar.gz - - name: Extract tarball + - name: Assemble infra run: | + # The release tarball carries the prebuilt deps (.venv, external) but + # its scripts are frozen at release time. Overlay the current checkout + # so activation and setup scripts match this branch. mkdir -p infra - tar -xzvf /tmp/code.tar.gz -C infra + tar -xzf /tmp/code.tar.gz -C infra + rsync -a --exclude='.git' --exclude='.venv' --exclude='external' infra-src/ infra/ + # The release .venv was built at a different path; relocate its + # console-script shebangs to where it now lives. + NEWV="$PWD/infra/.venv" + OLDV=$(sed -n '1s|^#!\(.*\)/bin/python.*|\1|p' infra/.venv/bin/f2py 2>/dev/null || true) + if [ -n "$OLDV" ] && [ "$OLDV" != "$NEWV" ]; then + grep -rIl "$OLDV" infra/.venv/bin | xargs -r sed -i "s|$OLDV|$NEWV|g" + fi - name: NEO-2 run: | From d3595851507d0d35ad2fcde3120eb5671d85af20 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Thu, 28 May 2026 21:28:50 +0200 Subject: [PATCH 3/7] CI: relax errexit while sourcing activate.sh activate.sh is meant for interactive shells; its add_to_path/add_to_library_path helpers return the guard status, so under the runner's bash -e the first absent dir ($INFRA/local/bin) aborted the step before the venv activated. Wrap the source in set +e/set -e so activation completes while build commands stay strict. --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5df7917..81f279f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,7 @@ jobs: - name: NEO-2 run: | - source infra/activate.sh + set +e; source infra/activate.sh; set -e clone_github NEO-2 cd NEO-2 $INFRA/scripts/checkout_branch.sh $CODE_BRANCH @@ -106,7 +106,7 @@ jobs: - name: NEO-RT run: | - source infra/activate.sh + set +e; source infra/activate.sh; set -e clone_github NEO-RT cd NEO-RT $INFRA/scripts/checkout_branch.sh $CODE_BRANCH @@ -114,7 +114,7 @@ jobs: - name: SIMPLE run: | - source infra/activate.sh + set +e; source infra/activate.sh; set -e clone_github SIMPLE cd SIMPLE $INFRA/scripts/checkout_branch.sh $CODE_BRANCH @@ -123,7 +123,7 @@ jobs: - name: GORILLA run: | - source infra/activate.sh + set +e; source infra/activate.sh; set -e clone_github GORILLA cd GORILLA $INFRA/scripts/checkout_branch.sh $CODE_BRANCH @@ -131,7 +131,7 @@ jobs: - name: MEPHIT run: | - source infra/activate.sh + set +e; source infra/activate.sh; set -e clone_github MEPHIT source /etc/profile.d/modules.sh module use -a $INFRA/modules From a14b19ce17023f170486a0c3628b98d6ea788a04 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Thu, 28 May 2026 21:36:41 +0200 Subject: [PATCH 4/7] Treat external as workspace-shared at $CODE/external The codes read prebuilt third-party libs from $CODE/external (e.g. NEO-2's build links $CODE/external/fgsl-1.6.0/.libs/libfgsl.a). external is workspace- shared like the code checkouts, not infra-private, so revert it from $INFRA to $CODE and stop tracking it in the infra repo. setup.sh creates it; CI moves the release's external to the workspace; CODE_ROOT (STELLOPT make configs) points at $CODE. setup.yml release packaging flagged for tag-time validation. --- .devcontainer/postCreateCommand.sh | 5 +++-- .github/workflows/main.yml | 2 ++ .github/workflows/setup.yml | 14 ++++++++++++-- README.md | 14 ++++++++------ activate.fish | 4 ++-- activate.sh | 26 +++++++++++++------------- external/.gitkeep | 0 external/intel/.gitkeep | 0 scripts/setup.sh | 4 +++- scripts/setup/cgal.sh | 4 ++-- scripts/setup/compiler_intel.sh | 2 +- scripts/setup/compiler_nvidia.sh | 8 ++++---- scripts/setup/devcontainer.sh | 5 +++-- scripts/setup/fftw.sh | 4 ++-- scripts/setup/fgsl.sh | 4 ++-- scripts/setup/gpec.sh | 2 +- scripts/setup/gsl.sh | 4 ++-- scripts/setup/mars.sh | 2 +- scripts/setup/netcdf.sh | 10 +++++----- scripts/setup/omfit.sh | 2 +- scripts/setup/openblas.sh | 4 ++-- scripts/setup/simsopt.sh | 2 +- scripts/setup/stellopt.sh | 16 ++++++++-------- scripts/setup/suitesparse.sh | 2 +- scripts/setup/vmec2000.sh | 2 +- 25 files changed, 80 insertions(+), 62 deletions(-) delete mode 100644 external/.gitkeep delete mode 100644 external/intel/.gitkeep diff --git a/.devcontainer/postCreateCommand.sh b/.devcontainer/postCreateCommand.sh index 2d7bb6d..d1cbc25 100644 --- a/.devcontainer/postCreateCommand.sh +++ b/.devcontainer/postCreateCommand.sh @@ -4,9 +4,10 @@ CODE_TEMPLATE=/usr/local/src/code_template INFRA=/workspaces/code CODE=/workspaces -# .venv and external belong to the infra repo; libneo is a sibling checkout +# .venv is infra-private; external and libneo are workspace-shared ln -s $CODE_TEMPLATE/.venv $INFRA/.venv -ln -s $CODE_TEMPLATE/external/fgsl-1.6.0 $INFRA/external/fgsl-1.6.0 +mkdir -p $CODE/external +ln -s $CODE_TEMPLATE/external/fgsl-1.6.0 $CODE/external/fgsl-1.6.0 cp -r $CODE_TEMPLATE/libneo $CODE/libneo echo 'source /workspaces/code/activate.sh' >> $HOME/.bashrc diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 81f279f..8742405 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,6 +88,8 @@ jobs: mkdir -p infra tar -xzf /tmp/code.tar.gz -C infra rsync -a --exclude='.git' --exclude='.venv' --exclude='external' infra-src/ infra/ + # external is workspace-shared; the codes read $CODE/external (= ./external here) + rm -rf external && mv infra/external external # The release .venv was built at a different path; relocate its # console-script shebangs to where it now lives. NEWV="$PWD/infra/.venv" diff --git a/.github/workflows/setup.yml b/.github/workflows/setup.yml index b7c7ce8..b51d944 100644 --- a/.github/workflows/setup.yml +++ b/.github/workflows/setup.yml @@ -76,17 +76,27 @@ jobs: libmetis-dev \ libboost-all-dev + # TODO(infra-relocation): this workflow runs on version tags only and is + # NOT exercised by PR CI. After the infra relocation, setup.sh builds the + # shared deps into $CODE/external (the workspace, parent of this checkout), + # so the steps below must build there and bundle external/ into the release + # tarball alongside the infra content. Validate on a real tag before + # cutting the next release; main.yml still consumes the pinned old release. - name: Setup run: | set -e source scripts/setup.sh - pushd external - ../scripts/setup/mfem.sh + CODE="$(cd .. && pwd)" + pushd "$CODE/external" + "$GITHUB_WORKSPACE/scripts/setup/mfem.sh" popd - name: Build tarball run: | set -e + CODE="$(cd .. && pwd)" + # flat release archive: infra content + the shared external/ at root + cp -r "$CODE/external" ./external tar -czf /tmp/code.tar.gz . - name: Create GitHub Release diff --git a/README.md b/README.md index 0f0338b..854b36a 100644 --- a/README.md +++ b/README.md @@ -22,13 +22,15 @@ environment variable `$CODE`; this repository is `$INFRA`, equal to $CODE/ infra/ this repository (activation, setup scripts, modules, .venv) + external/ prebuilt third-party libs, shared by all codes libneo/ code checkout SIMPLE/ code checkout ... -Activation exports both. `$INFRA` holds the infra-owned paths (`scripts`, -`.venv`, `external`, `modules`). `$CODE` is the workspace root, so the codes -resolve their dependencies as `$CODE/`. +Activation exports both. `$INFRA` holds the infra-private paths (`scripts`, +`.venv`, `modules`). `$CODE` is the workspace root, so the codes resolve their +dependencies as `$CODE/` and read prebuilt libraries from +`$CODE/external`. ## Getting Started @@ -89,7 +91,7 @@ to install the compiler and libraries HDF5 and NetCDFwith modules. Then the comm scripts/setup/gpec.sh scripts/setup/mars.sh -will install GPEC and MARS into `$INFRA/external/intel`. They can then be loaded with +will install GPEC and MARS into `$CODE/external/intel`. They can then be loaded with module load gpec module load mars @@ -99,7 +101,7 @@ codes based on GNU Fortran in the same shell. ### OMFIT OMFIT requires its own Python environment provided via `conda`. Both will be installed -to `$INFRA/external` by running +to `$CODE/external` by running deactivate scripts/setup/omfit.sh @@ -107,7 +109,7 @@ to `$INFRA/external` by running OMFIT can then be loaded with deactivate - source $INFRA/external/mambaforge/bin/activate omfit + source $CODE/external/mambaforge/bin/activate omfit module load omfit and then started with `omfit` in the shell. Be careful not to work on codes diff --git a/activate.fish b/activate.fish index bb7cf70..956ad2d 100644 --- a/activate.fish +++ b/activate.fish @@ -38,8 +38,8 @@ if test (uname) = "Darwin" echo "Unknown processor architecture." end - set -gx CMAKE_INCLUDE_PATH "/opt/homebrew/include/suitesparse:$INFRA/external/triangle" - set -gx CMAKE_LIBRARY_PATH "/opt/homebrew/lib:$INFRA/external/triangle/build" + set -gx CMAKE_INCLUDE_PATH "/opt/homebrew/include/suitesparse:$CODE/external/triangle" + set -gx CMAKE_LIBRARY_PATH "/opt/homebrew/lib:$CODE/external/triangle/build" set -gx CMAKE_ARGS "-DBLAS_LIBRARIES=$BLAS_LIBRARIES -DLAPACK_LIBRARIES=$LAPACK_LIBRARIES" else set -gx CMAKE_ARGS "" diff --git a/activate.sh b/activate.sh index 899a7e5..f6dfbb6 100644 --- a/activate.sh +++ b/activate.sh @@ -35,15 +35,15 @@ if [ "$(uname)" = "Darwin" ]; then echo "Unknown processor architecture." fi - export CMAKE_INCLUDE_PATH="/opt/homebrew/include/suitesparse:$INFRA/external/triangle" - export CMAKE_LIBRARY_PATH="/opt/homebrew/lib:$INFRA/external/triangle/build" + export CMAKE_INCLUDE_PATH="/opt/homebrew/include/suitesparse:$CODE/external/triangle" + export CMAKE_LIBRARY_PATH="/opt/homebrew/lib:$CODE/external/triangle/build" export CMAKE_ARGS="-DBLAS_LIBRARIES=$BLAS_LIBRARIES -DLAPACK_LIBRARIES=$LAPACK_LIBRARIES" else export CMAKE_ARGS="" # Use locally built OpenBLAS if system BLAS headers not available if [ ! -f /usr/include/cblas.h ] && [ ! -f /usr/include/openblas/cblas.h ]; then - OPENBLAS_PREFIX="$INFRA/external/OpenBLAS-0.3.28/install" + OPENBLAS_PREFIX="$CODE/external/OpenBLAS-0.3.28/install" if [ -d "$OPENBLAS_PREFIX" ]; then export BLAS_LIBRARIES="$OPENBLAS_PREFIX/lib/libopenblas.so" export LAPACK_LIBRARIES="$OPENBLAS_PREFIX/lib/libopenblas.so" @@ -53,30 +53,30 @@ else fi # Use locally built GSL only if system GSL not available - if ! command -v gsl-config &> /dev/null && [ -d "$INFRA/external/gsl-2.8/install" ]; then - export GSL_ROOT_DIR="$INFRA/external/gsl-2.8/install" + if ! command -v gsl-config &> /dev/null && [ -d "$CODE/external/gsl-2.8/install" ]; then + export GSL_ROOT_DIR="$CODE/external/gsl-2.8/install" export PKG_CONFIG_PATH="$GSL_ROOT_DIR/lib/pkgconfig:$PKG_CONFIG_PATH" fi # Use locally built FFTW if system headers not available - if [ ! -f /usr/include/fftw3.h ] && [ -d "$INFRA/external/fftw-3.3.10/install" ]; then - export FFTW_ROOT="$INFRA/external/fftw-3.3.10/install" + if [ ! -f /usr/include/fftw3.h ] && [ -d "$CODE/external/fftw-3.3.10/install" ]; then + export FFTW_ROOT="$CODE/external/fftw-3.3.10/install" fi # Use locally built HDF5/NetCDF if system nf-config not available - if ! command -v nf-config &> /dev/null && [ -d "$INFRA/external/netcdf-install" ]; then - export HDF5_ROOT="$INFRA/external/netcdf-install" - export NETCDF_ROOT="$INFRA/external/netcdf-install" - export PATH="$INFRA/external/netcdf-install/bin:$PATH" + if ! command -v nf-config &> /dev/null && [ -d "$CODE/external/netcdf-install" ]; then + export HDF5_ROOT="$CODE/external/netcdf-install" + export NETCDF_ROOT="$CODE/external/netcdf-install" + export PATH="$CODE/external/netcdf-install/bin:$PATH" fi # Use locally installed NVIDIA HPC SDK if available (optional) - NVHPC_ROOT="$INFRA/external/nvhpc/Linux_x86_64" + NVHPC_ROOT="$CODE/external/nvhpc/Linux_x86_64" if [ -d "$NVHPC_ROOT" ]; then # Find the installed version NVHPC_VERSION=$(ls "$NVHPC_ROOT" 2>/dev/null | grep -E '^[0-9]+\.[0-9]+$' | sort -V | tail -1) if [ -n "$NVHPC_VERSION" ]; then - export NVHPC="$INFRA/external/nvhpc" + export NVHPC="$CODE/external/nvhpc" export NVHPC_ROOT="$NVHPC_ROOT/$NVHPC_VERSION" export PATH="$NVHPC_ROOT/compilers/bin:$PATH" export PATH="$NVHPC_ROOT/comm_libs/mpi/bin:$PATH" diff --git a/external/.gitkeep b/external/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/external/intel/.gitkeep b/external/intel/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/setup.sh b/scripts/setup.sh index f755817..06f19fb 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -14,7 +14,9 @@ pushd $INFRA source $INFRA/.venv/bin/activate popd -pushd $INFRA/external +# external prebuilt deps are workspace-shared; the codes read $CODE/external +mkdir -p $CODE/external +pushd $CODE/external $INFRA/scripts/setup/openblas.sh $INFRA/scripts/setup/gsl.sh $INFRA/scripts/setup/fgsl.sh diff --git a/scripts/setup/cgal.sh b/scripts/setup/cgal.sh index ede584c..9329ad4 100755 --- a/scripts/setup/cgal.sh +++ b/scripts/setup/cgal.sh @@ -4,11 +4,11 @@ set -e -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 CGAL_VERSION="5.6.1" CGAL_DIR="CGAL-${CGAL_VERSION}" -CGAL_INSTALL="$INFRA/external/cgal" +CGAL_INSTALL="$CODE/external/cgal" if [ ! -d "$CGAL_INSTALL/include/CGAL" ]; then echo "Fetching and installing CGAL ${CGAL_VERSION}..." diff --git a/scripts/setup/compiler_intel.sh b/scripts/setup/compiler_intel.sh index 2ec0964..a13e4c3 100755 --- a/scripts/setup/compiler_intel.sh +++ b/scripts/setup/compiler_intel.sh @@ -44,6 +44,6 @@ export FC=ifx export CC=icx export CXX=icpx -pushd $INFRA/external/intel +pushd $CODE/external/intel source $INFRA/scripts/setup/netcdf.sh popd diff --git a/scripts/setup/compiler_nvidia.sh b/scripts/setup/compiler_nvidia.sh index 3628a1d..50f245c 100755 --- a/scripts/setup/compiler_nvidia.sh +++ b/scripts/setup/compiler_nvidia.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # NVIDIA HPC SDK setup script (optional - not included in default setup.sh) -# Installs to $INFRA/external/nvhpc without requiring root access +# Installs to $CODE/external/nvhpc without requiring root access set -e @@ -8,9 +8,9 @@ NVHPC_VERSION=25.11 NVHPC_YEAR=2025 CUDA_VERSION=13.0 -INSTALL_DIR="$INFRA/external/nvhpc" +INSTALL_DIR="$CODE/external/nvhpc" -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 # Check if already installed if [ -f "$INSTALL_DIR/Linux_x86_64/$NVHPC_VERSION/compilers/bin/nvfortran" ]; then @@ -47,7 +47,7 @@ export NVHPC_INSTALL_LOCAL_DIR="" ./install -cd "$INFRA/external" +cd "$CODE/external" # Verify installation if [ -f "$INSTALL_DIR/Linux_x86_64/$NVHPC_VERSION/compilers/bin/nvfortran" ]; then diff --git a/scripts/setup/devcontainer.sh b/scripts/setup/devcontainer.sh index b32c429..3a28d51 100755 --- a/scripts/setup/devcontainer.sh +++ b/scripts/setup/devcontainer.sh @@ -13,9 +13,10 @@ git clone https://github.com/itpplasma/code.git code cd /workspaces/code source scripts/setup.sh -# setup.sh clones libneo to the workspace ($CODE = /workspaces); stash it inside -# the infra template so postCreateCommand.sh can restore it as a sibling. +# setup.sh puts libneo and external in the workspace ($CODE = /workspaces); +# stash them inside the infra template so postCreateCommand.sh can restore them. mv /workspaces/libneo /workspaces/code/libneo +mv /workspaces/external /workspaces/code/external mv /workspaces/code $CODE_TEMPLATE rm -rf /workspaces diff --git a/scripts/setup/fftw.sh b/scripts/setup/fftw.sh index 54a5d51..c5a9ebd 100755 --- a/scripts/setup/fftw.sh +++ b/scripts/setup/fftw.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 FFTW_VERSION=3.3.10 -INSTALL_PREFIX="$INFRA/external/fftw-${FFTW_VERSION}/install" +INSTALL_PREFIX="$CODE/external/fftw-${FFTW_VERSION}/install" # Check if system FFTW is available if [ -f /usr/include/fftw3.h ] || [ -f /usr/local/include/fftw3.h ]; then diff --git a/scripts/setup/fgsl.sh b/scripts/setup/fgsl.sh index d13cc68..bd3b07c 100755 --- a/scripts/setup/fgsl.sh +++ b/scripts/setup/fgsl.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 if [ ! -f "fgsl-1.6.0/.libs/libfgsl.a" ] ; then echo "Fetching and building FGSL..." @@ -9,7 +9,7 @@ if [ ! -f "fgsl-1.6.0/.libs/libfgsl.a" ] ; then if command -v gsl-config &> /dev/null; then GSL_CONFIG=gsl-config else - GSL_PREFIX=$INFRA/external/gsl-2.8/install + GSL_PREFIX=$CODE/external/gsl-2.8/install if [ -d "$GSL_PREFIX" ]; then GSL_CONFIG="$GSL_PREFIX/bin/gsl-config" export LD_LIBRARY_PATH="$GSL_PREFIX/lib:$LD_LIBRARY_PATH" diff --git a/scripts/setup/gpec.sh b/scripts/setup/gpec.sh index 96d8059..49db761 100755 --- a/scripts/setup/gpec.sh +++ b/scripts/setup/gpec.sh @@ -10,7 +10,7 @@ export CC=icx export CXX=icpx export FC=ifx export OMPFLAG="-qopenmp" -cd $INFRA/external/intel +cd $CODE/external/intel ../../scripts/setup/netcdf.sh diff --git a/scripts/setup/gsl.sh b/scripts/setup/gsl.sh index c108dcb..a80e0b7 100755 --- a/scripts/setup/gsl.sh +++ b/scripts/setup/gsl.sh @@ -2,7 +2,7 @@ GSL_VERSION=2.8 -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 if command -v gsl-config &> /dev/null; then echo "System GSL found: $(gsl-config --version)" @@ -12,7 +12,7 @@ elif [ ! -f "gsl-${GSL_VERSION}/install/lib/libgsl.a" ]; then curl -L https://ftp.gnu.org/gnu/gsl/gsl-${GSL_VERSION}.tar.gz -o - | tar xz fi pushd gsl-${GSL_VERSION} - ./configure --prefix=$INFRA/external/gsl-${GSL_VERSION}/install + ./configure --prefix=$CODE/external/gsl-${GSL_VERSION}/install make -j$(nproc) make install popd diff --git a/scripts/setup/mars.sh b/scripts/setup/mars.sh index 3ed3339..8b3608a 100755 --- a/scripts/setup/mars.sh +++ b/scripts/setup/mars.sh @@ -2,7 +2,7 @@ set -e module load intel/compiler-rt/2024.2.1 intel/ifort intel/mpi -cd $INFRA/external/intel +cd $CODE/external/intel git clone git@gitlab.tugraz.at:plasma/codes/mars.git MARS cd MARS/MarsQ_2022 make diff --git a/scripts/setup/netcdf.sh b/scripts/setup/netcdf.sh index 9105e18..bb623ae 100755 --- a/scripts/setup/netcdf.sh +++ b/scripts/setup/netcdf.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 HDF5_VERSION=1.14.5 NETCDF_C_VERSION=4.9.2 NETCDF_F_VERSION=4.6.1 -INSTALL_PREFIX="$INFRA/external/netcdf-install" +INSTALL_PREFIX="$CODE/external/netcdf-install" # Check if system NetCDF-Fortran is available if command -v nf-config &> /dev/null; then @@ -40,7 +40,7 @@ if [ ! -f "$INSTALL_PREFIX/lib/libhdf5.so" ]; then -DBUILD_SHARED_LIBS=ON \ -DHDF5_BUILD_FORTRAN=ON .. make -j$(nproc) install - cd "$INFRA/external" + cd "$CODE/external" fi # Build NetCDF-C @@ -54,7 +54,7 @@ if [ ! -f "$INSTALL_PREFIX/lib/libnetcdf.so" ]; then LDFLAGS="-L$INSTALL_PREFIX/lib -Wl,-rpath,$INSTALL_PREFIX/lib" \ ./configure --prefix="$INSTALL_PREFIX" --disable-libxml2 --disable-byterange make -j$(nproc) install - cd "$INFRA/external" + cd "$CODE/external" fi # Build NetCDF-Fortran @@ -70,7 +70,7 @@ if [ ! -f "$INSTALL_PREFIX/lib/libnetcdff.so" ]; then -DNETCDF_C_INCLUDE_DIR="$INSTALL_PREFIX/include" \ -DCMAKE_PREFIX_PATH="$INSTALL_PREFIX" .. make -j$(nproc) install - cd "$INFRA/external" + cd "$CODE/external" fi echo "NetCDF stack installed to $INSTALL_PREFIX" diff --git a/scripts/setup/omfit.sh b/scripts/setup/omfit.sh index 8e9aae6..cf1beff 100755 --- a/scripts/setup/omfit.sh +++ b/scripts/setup/omfit.sh @@ -5,7 +5,7 @@ echo "Requires personal Git access to OMFIT repository." echo "Code access: https://omfit.io/install.html" export USER=`whoami` -cd $INFRA/external +cd $CODE/external git clone --filter=blob:none -b unstable git@gitlab.tugraz.at:plasma/codes/OMFIT-source.git cd OMFIT-source git submodule update --init omas diff --git a/scripts/setup/openblas.sh b/scripts/setup/openblas.sh index 7731523..14d024a 100755 --- a/scripts/setup/openblas.sh +++ b/scripts/setup/openblas.sh @@ -2,7 +2,7 @@ OPENBLAS_VERSION=0.3.28 -cd "$INFRA/external" || exit 1 +cd "$CODE/external" || exit 1 # Check if system BLAS/LAPACK dev headers exist if [ -f /usr/include/cblas.h ] || [ -f /usr/include/openblas/cblas.h ]; then @@ -15,5 +15,5 @@ elif [ ! -f "OpenBLAS-${OPENBLAS_VERSION}/install/lib/libopenblas.a" ]; then cd OpenBLAS-${OPENBLAS_VERSION} # Build with ZEN target for AMD EPYC make -j$(nproc) TARGET=ZEN USE_OPENMP=1 - make PREFIX=$INFRA/external/OpenBLAS-${OPENBLAS_VERSION}/install install + make PREFIX=$CODE/external/OpenBLAS-${OPENBLAS_VERSION}/install install fi diff --git a/scripts/setup/simsopt.sh b/scripts/setup/simsopt.sh index a20bae7..2dfe367 100755 --- a/scripts/setup/simsopt.sh +++ b/scripts/setup/simsopt.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $INFRA/external +cd $CODE/external if [ ! -d "simsopt" ] ; then echo "Fetching and building simsopt..." diff --git a/scripts/setup/stellopt.sh b/scripts/setup/stellopt.sh index 55b30b7..bbc070b 100755 --- a/scripts/setup/stellopt.sh +++ b/scripts/setup/stellopt.sh @@ -3,7 +3,7 @@ set -e # CODE_ROOT feeds the STELLOPT make configs, which read $(CODE_ROOT)/external/... # That tree lives in the infra repo, so point it at $INFRA. -export CODE_ROOT="$INFRA" +export CODE_ROOT="$CODE" # Create ~/bin for libstell symlinks and ensure it's in PATH mkdir -p ~/bin @@ -18,24 +18,24 @@ fi select_machine() { case "$(uname -s)" in Darwin) - cp $INFRA/scripts/setup/stellopt/make_osx_brew_m1.inc $INFRA/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_osx_brew_m1.inc $CODE/external/STELLOPT/SHARE export MACHINE=osx_brew_m1 ;; Linux) # Check for VSC5 cluster (hostname contains vsc or l5 node pattern) if [[ "$(hostname)" == *vsc* ]] || [[ "$(hostname)" == l5* ]] || [[ -n "$VSC_INSTITUTE" ]]; then - cp $INFRA/scripts/setup/stellopt/make_vsc5.inc $INFRA/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_vsc5.inc $CODE/external/STELLOPT/SHARE export MACHINE=vsc5 # Check for scluster elif [[ "$(hostname)" == scluster* ]]; then - cp $INFRA/scripts/setup/stellopt/make_scluster.inc $INFRA/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_scluster.inc $CODE/external/STELLOPT/SHARE export MACHINE=scluster # Detect distro family from /etc/os-release elif [ -f /etc/os-release ]; then . /etc/os-release case "$ID" in arch|manjaro|cachyos|endeavouros|garuda|artix) - cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $INFRA/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $CODE/external/STELLOPT/SHARE export MACHINE=arch_linux ;; rhel|centos|fedora|almalinux|rocky) @@ -47,7 +47,7 @@ select_machine() { *) case "$ID_LIKE" in *arch*) - cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $INFRA/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $CODE/external/STELLOPT/SHARE export MACHINE=arch_linux ;; *rhel*|*centos*|*fedora*) @@ -59,7 +59,7 @@ select_machine() { *) # Fallback: check for pacman (Arch-based) if command -v pacman &>/dev/null; then - cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $INFRA/external/STELLOPT/SHARE + cp $INFRA/scripts/setup/stellopt/make_arch_linux.inc $CODE/external/STELLOPT/SHARE export MACHINE=arch_linux else export MACHINE=ubuntu @@ -94,7 +94,7 @@ apply_patches() { fi } -cd $INFRA/external +cd $CODE/external if [ ! -d "STELLOPT" ] ; then echo "Cloning STELLOPT..." diff --git a/scripts/setup/suitesparse.sh b/scripts/setup/suitesparse.sh index 62b32f9..1018438 100755 --- a/scripts/setup/suitesparse.sh +++ b/scripts/setup/suitesparse.sh @@ -1,6 +1,6 @@ #!/bin/sh -cd $INFRA/external +cd $CODE/external git clone https://github.com/DrTimothyAldenDavis/SuiteSparse cd SuiteSparse/build git checkout v7.10.2 diff --git a/scripts/setup/vmec2000.sh b/scripts/setup/vmec2000.sh index 4aa0695..b80ffe2 100755 --- a/scripts/setup/vmec2000.sh +++ b/scripts/setup/vmec2000.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -cd $INFRA/external +cd $CODE/external if [ ! -d "VMEC2000" ] ; then echo "Fetching and building VMEC2000..." From 1faaa2ddd49ffcb7d9789a71efc86d8ba454229e Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Thu, 28 May 2026 23:41:48 +0200 Subject: [PATCH 5/7] Retire umbrella integration CI; document the development model main.yml built every downstream code on each push to infra and never passed (0 of 94 runs): it built the unpinned tips of six repos in one sequential job against a frozen release tarball. Remove it. Add docs/development-model.md (proposed): workspace layout, the self-describing dependency graph, the per-code + release-time reverse-dependency testing model, the _BRANCH convention, the release-branch flow, and YY.MINOR.PATCH versioning. The genuine external -> $CODE fix from earlier commits stays; the umbrella build it was chasing is gone. --- .github/workflows/main.yml | 144 ------------------------------------- docs/development-model.md | 114 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 144 deletions(-) delete mode 100644 .github/workflows/main.yml create mode 100644 docs/development-model.md diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 8742405..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,144 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: CI - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -env: - GIT_HTTPS: "true" - CI_MERGE_REQUEST_SOURCE_BRANCH_NAME: "${{ github.head_ref }}" - -jobs: - code: - name: Build and test - runs-on: ubuntu-24.04 - - steps: - - name: Install dependencies - run: | - sudo apt-get update -y && sudo apt-get install -y -q --no-install-recommends \ - apt-transport-https \ - build-essential \ - ca-certificates \ - curl \ - libssl-dev \ - openssh-client \ - rsync \ - wget \ - git \ - gfortran \ - ninja-build \ - cmake \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv - - sudo apt-get install -y -q --no-install-recommends \ - python3-tk \ - idle3 \ - findent \ - expect \ - environment-modules - - sudo apt-get install -y -q --no-install-recommends \ - pkg-config \ - libsuitesparse-dev \ - libopenblas-dev \ - libsuperlu-dev \ - libhdf5-dev \ - libhdf5-openmpi-dev \ - libnetcdf-dev \ - libnetcdff-dev \ - libfftw3-dev \ - libgsl-dev \ - libopenmpi-dev \ - libscalapack-openmpi-dev \ - libpcre3-dev \ - libreadline-dev \ - h5utils \ - hdf5-tools \ - netcdf-bin \ - libmetis-dev \ - libboost-all-dev - - - name: Checkout infra sources - uses: actions/checkout@v4 - with: - path: infra-src - - - name: Download Release # TODO: make this dynamic - run: | - wget https://github.com/itpplasma/code/releases/download/v2025.01.02/code-v2025.01.02.tar.gz -O /tmp/code.tar.gz - - - name: Assemble infra - run: | - # The release tarball carries the prebuilt deps (.venv, external) but - # its scripts are frozen at release time. Overlay the current checkout - # so activation and setup scripts match this branch. - mkdir -p infra - tar -xzf /tmp/code.tar.gz -C infra - rsync -a --exclude='.git' --exclude='.venv' --exclude='external' infra-src/ infra/ - # external is workspace-shared; the codes read $CODE/external (= ./external here) - rm -rf external && mv infra/external external - # The release .venv was built at a different path; relocate its - # console-script shebangs to where it now lives. - NEWV="$PWD/infra/.venv" - OLDV=$(sed -n '1s|^#!\(.*\)/bin/python.*|\1|p' infra/.venv/bin/f2py 2>/dev/null || true) - if [ -n "$OLDV" ] && [ "$OLDV" != "$NEWV" ]; then - grep -rIl "$OLDV" infra/.venv/bin | xargs -r sed -i "s|$OLDV|$NEWV|g" - fi - - - name: NEO-2 - run: | - set +e; source infra/activate.sh; set -e - clone_github NEO-2 - cd NEO-2 - $INFRA/scripts/checkout_branch.sh $CODE_BRANCH - make - - - name: NEO-RT - run: | - set +e; source infra/activate.sh; set -e - clone_github NEO-RT - cd NEO-RT - $INFRA/scripts/checkout_branch.sh $CODE_BRANCH - make - - - name: SIMPLE - run: | - set +e; source infra/activate.sh; set -e - clone_github SIMPLE - cd SIMPLE - $INFRA/scripts/checkout_branch.sh $CODE_BRANCH - make - - - - name: GORILLA - run: | - set +e; source infra/activate.sh; set -e - clone_github GORILLA - cd GORILLA - $INFRA/scripts/checkout_branch.sh $CODE_BRANCH - make - - - name: MEPHIT - run: | - set +e; source infra/activate.sh; set -e - clone_github MEPHIT - source /etc/profile.d/modules.sh - module use -a $INFRA/modules - module load mephit - cd MEPHIT - $INFRA/scripts/checkout_branch.sh $CODE_BRANCH - make - pip install -e . diff --git a/docs/development-model.md b/docs/development-model.md new file mode 100644 index 0000000..0b95d89 --- /dev/null +++ b/docs/development-model.md @@ -0,0 +1,114 @@ +# Development model (proposed) + +Status: proposed, 2026-05. Open for group discussion before adoption. + +Conventions for how the plasma codes and their shared dependencies are built and +released. This is a proposal from a design session, not settled policy. + +## Workspace layout + +`$CODE` is a plain workspace directory. It holds: + + $CODE/ + infra/ this repository: activation, setup scripts, modules, .venv + external/ prebuilt third-party libraries, shared by all codes + libneo/ code checkout + SIMPLE/ code checkout + ... + +Activation exports `INFRA=$CODE/infra` and `CODE` as its parent. Infra-private +paths (`scripts`, `.venv`, `modules`) live under `$INFRA`. The codes resolve +their dependencies as `$CODE/` and read prebuilt libraries from +`$CODE/external`. `infra` is no longer the workspace root, so its git tree stops +seeing the sibling checkouts. + +## Dependency graph + +Each code declares its first-party dependencies in CMake through `find_or_fetch` +(use `$CODE/` if present, else fetch from GitHub). The graph is therefore +self-describing: read it by inverting those declarations rather than maintaining +a list by hand. + +Current hub: libneo. Consumers with a declared edge: SIMPLE, NEO-2, NEO-RT, +MEPHIT, KAMEL, rabe. Secondary first-party dependency: fortplot (SIMPLE, +NEO-RT). Single-consumer leaves: spline (NEO-RT), vode (NEO-RT), quadpack +(KAMEL). GORILLA and most small projects declare no first-party edge. + +## Integration testing + +The old umbrella CI built every code on each push to infra. It never passed (0 +of 94 runs) because it built the unpinned tips of six repos in one sequential +job against a frozen release tarball. We retire it. + +Replacement, in two layers: + +1. Per-code CI at the source. Each code builds against the current libneo in its + own CI, so a breaking change surfaces in the pull request that caused it. +2. Release-time reverse-dependency validation, owned by the upstream. Before + libneo releases, it builds its tracked consumers against the candidate and + gates the release on the result. + +A tracked downstream is a code with a real `find_or_fetch` edge to the upstream +and its own CI/CD. Both are checkable, so the set is computed, not curated, and +the long tail of small projects drops out automatically. + +## The one convention + +Every tracked downstream honors `_BRANCH` (environment or CMake cache +variable): when set, `find_or_fetch` fetches that dependency at the given ref +instead of the committed default. `LIBNEO_BRANCH` is the instance for libneo, +and SIMPLE already implements it. This single hook lets the upstream build any +downstream against any candidate ref with no commit to the downstream. It is the +multi-repo stand-in for a global build graph, and the only piece that has to be +standardized across the codes. + +## Release model + +Release branches, no release-candidate tags. + +1. libneo cuts `release/YY.MINOR` off main. +2. For each tracked downstream, dispatch its integration workflow with + `LIBNEO_BRANCH=release/YY.MINOR` (a `workflow_dispatch` input: no commit, no + pull request). The downstream builds its own main against the candidate. +3. libneo polls the dispatched runs and gates the tag on all of them passing. +4. On all-green, tag `YY.MINOR.0` at the release-branch head. +5. A bot opens a bump pull request in each downstream pinning the tag, never the + branch. The maintainer reviews and merges on their own schedule; no + auto-merge. + +Invariants: + +- A downstream's main only ever pins released tags. Branch refs live only in the + ephemeral dispatch override, never in a committed pin. +- Validation needs green CI, not merged pull requests, so waiting for + maintainers never blocks the upstream release. +- No automatic downstream releases. A downstream releases on its own cadence. + Propagate a release only along a real edge, and only when behavior changed + (for example a libneo change that alters NEO-2 output, NEO-2 being itself an + upstream of NEO-RT). + +Patch releases reuse the branch: cherry-pick onto `release/YY.MINOR`, re-run the +downstream dispatch, tag `YY.MINOR.PATCH`. + +## Versioning + +`YY.MINOR.PATCH`. `YY` is the two-digit year the release branch is cut, not the +year it finishes. MINOR counts releases within that year; PATCH counts fixes on +a release branch. Derive the version from the git tag (for example via +`setuptools_scm`) so the tag is the single source of truth, and feed CMake from +the same source. + +No leading zeros in the version. PEP 440 normalizes `26.05` to `26.5`, so a +packaged library must use `26.5` or its metadata disagrees with the tag. Every +component is then a plain integer, which keeps git, CMake `project(VERSION)`, +Spack, and PEP 440 in agreement. + +Teaching codes keep the semester scheme `YY.0M` (`26.03`, `26.10`). They are +end-user deliverables, not packaged libraries, so the leading-zero rule does not +reach them. + +Rationale for CalVer over SemVer: the release process verifies downstream +compatibility by building, so the number need not carry SemVer's compatibility +promise. Its remaining job is to identify the dated snapshot, which is what +citation and reproducibility want. Precedent in scientific software includes +GROMACS (`2026.2`) and FEniCS (`2019.1.0`). From 4dce46ff1954c573c2ee9e73322528f73a1da7c1 Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Thu, 28 May 2026 23:45:55 +0200 Subject: [PATCH 6/7] Add proposed workflow templates for the development model docs/templates/downstream-integration.yml: per-consumer reverse-dep validation, dispatched with a libneo ref, no commit. docs/templates/upstream-release.yml: libneo release-branch + gate + bump-PR skeleton (untested, TODOs marked). Referenced by the development-model design note. --- docs/templates/downstream-integration.yml | 35 ++++++++++++ docs/templates/upstream-release.yml | 66 +++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 docs/templates/downstream-integration.yml create mode 100644 docs/templates/upstream-release.yml diff --git a/docs/templates/downstream-integration.yml b/docs/templates/downstream-integration.yml new file mode 100644 index 0000000..f65c14f --- /dev/null +++ b/docs/templates/downstream-integration.yml @@ -0,0 +1,35 @@ +# Template: downstream reverse-dependency validation (proposed). +# +# Commit this once into each tracked consumer's .github/workflows/. An upstream +# release workflow dispatches it with the upstream ref to build against. It +# builds THIS code's own default branch against that ref, with no commit and no +# pin change, and the upstream polls the run conclusion to gate its release tag. +# +# Prerequisite: this code's find_or_fetch honors LIBNEO_BRANCH (see the +# _BRANCH convention in docs/development-model.md). + +name: integration + +on: + workflow_dispatch: + inputs: + libneo_ref: + description: libneo branch or tag to build against + required: true + default: main + +jobs: + build: + runs-on: ubuntu-24.04 + env: + # find_or_fetch fetches libneo at this ref; the committed pin stays untouched + LIBNEO_BRANCH: ${{ inputs.libneo_ref }} + steps: + - uses: actions/checkout@v4 + - name: Build and test + run: | + # Per-code build. CMake codes: + cmake -S . -B build -G Ninja + cmake --build build -j + ctest --test-dir build --output-on-failure + # Make-based codes substitute their own `make && make test`. diff --git a/docs/templates/upstream-release.yml b/docs/templates/upstream-release.yml new file mode 100644 index 0000000..ae44729 --- /dev/null +++ b/docs/templates/upstream-release.yml @@ -0,0 +1,66 @@ +# Template: upstream release orchestration (proposed skeleton, untested). +# +# Lives in the hub repo (libneo). Cuts a release branch, validates the tracked +# downstreams against it, gates the tag on all-green, then opens tag-pinned bump +# PRs. The downstream list is the inverted find_or_fetch graph; the set below is +# libneo's current consumers. See docs/development-model.md for the full model. + +name: release + +on: + workflow_dispatch: + inputs: + version: + description: release version YY.MINOR.PATCH (e.g. 26.1.0) + required: true + +env: + DOWNSTREAMS: "SIMPLE NEO-2 NEO-RT MEPHIT KAMEL rabe" + +jobs: + release: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cut release branch + run: | + V="${{ inputs.version }}" # 26.1.0 + BRANCH="release/${V%.*}" # release/26.1 + git switch -c "$BRANCH" || git switch "$BRANCH" + git push -u origin "$BRANCH" + echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" + + - name: Dispatch downstream validation + env: + GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN }} # needs actions:write on the downstreams + run: | + for d in $DOWNSTREAMS; do + gh workflow run integration.yml -R "itpplasma/$d" -f "libneo_ref=$BRANCH" + done + # TODO: capture each dispatched run id (poll the runs API right after dispatch) + + - name: Gate on all downstreams green + env: + GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN }} + run: | + # TODO: poll each downstream's latest integration run for libneo_ref=$BRANCH + # until conclusion != null; fail this job if any conclusion != success. + echo "TODO: poll conclusions, fail on any non-success" + + - name: Tag the release + run: | + # Only reached when the gate passed. Tag the exact branch head that + # the downstreams validated against. + git tag "${{ inputs.version }}" + git push origin "${{ inputs.version }}" + + - name: Open bump PRs (pin the tag, never the branch) + env: + GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN }} + run: | + # TODO: for each downstream, edit its standardized libneo pin to the + # tag ${{ inputs.version }}, open a PR, assign code owners. No merge. + echo "TODO: open tag-pinned bump PRs; maintainers merge on their own schedule" From be6a7c0bbf5e0f344b06d48ffdd532f4b2349b2d Mon Sep 17 00:00:00 2001 From: Christopher Albert Date: Mon, 8 Jun 2026 16:11:10 +0200 Subject: [PATCH 7/7] Update development model to release-branch tracking and LIBNEO_REF Codes track the libneo release branch by default (committed LIBNEO_RELEASE); tags are reproducibility snapshots. Rename the override to LIBNEO_REF (branch, tag or commit), validated against the remote. Drop the $CODE local-path shortcut: first-party deps are always fetched at the resolved ref. The release gate dispatches each downstream's existing CI and opens auto-merge bump PRs to the new release branch; templates reflect reuse of existing CI and point at the canonical libneo release.yml. --- docs/development-model.md | 97 ++++++++++++++--------- docs/templates/downstream-integration.yml | 48 +++++------ docs/templates/upstream-release.yml | 81 ++++--------------- 3 files changed, 95 insertions(+), 131 deletions(-) diff --git a/docs/development-model.md b/docs/development-model.md index 0b95d89..69a6034 100644 --- a/docs/development-model.md +++ b/docs/development-model.md @@ -17,17 +17,20 @@ released. This is a proposal from a design session, not settled policy. ... Activation exports `INFRA=$CODE/infra` and `CODE` as its parent. Infra-private -paths (`scripts`, `.venv`, `modules`) live under `$INFRA`. The codes resolve -their dependencies as `$CODE/` and read prebuilt libraries from -`$CODE/external`. `infra` is no longer the workspace root, so its git tree stops -seeing the sibling checkouts. +paths (`scripts`, `.venv`, `modules`) live under `$INFRA`. Codes read prebuilt +third-party libraries from `$CODE/external`. First-party dependencies are fetched +at a resolved ref, not read from the sibling checkouts (see Dependency graph), so +a local checkout is for editing that code, not for wiring one code against +another. `infra` is no longer the workspace root, so its git tree stops seeing +the sibling checkouts. ## Dependency graph -Each code declares its first-party dependencies in CMake through `find_or_fetch` -(use `$CODE/` if present, else fetch from GitHub). The graph is therefore -self-describing: read it by inverting those declarations rather than maintaining -a list by hand. +Each code declares its first-party dependencies in CMake through `find_or_fetch`, +which fetches each one from GitHub at a resolved ref (see The one convention). +There is no local-path shortcut: a sibling checkout is not consulted, so the +build is reproducible from the declaration alone. The graph is self-describing: +read it by inverting those declarations rather than keeping a list by hand. Current hub: libneo. Consumers with a declared edge: SIMPLE, NEO-2, NEO-RT, MEPHIT, KAMEL, rabe. Secondary first-party dependency: fortplot (SIMPLE, @@ -42,11 +45,11 @@ job against a frozen release tarball. We retire it. Replacement, in two layers: -1. Per-code CI at the source. Each code builds against the current libneo in its - own CI, so a breaking change surfaces in the pull request that caused it. +1. Per-code CI at the source. Each code builds against the libneo release branch + it tracks, so a breaking change surfaces in the pull request that caused it. 2. Release-time reverse-dependency validation, owned by the upstream. Before - libneo releases, it builds its tracked consumers against the candidate and - gates the release on the result. + libneo releases, it dispatches each tracked consumer's own CI against the + candidate and gates the release on the result. A tracked downstream is a code with a real `find_or_fetch` edge to the upstream and its own CI/CD. Both are checkable, so the set is computed, not curated, and @@ -54,38 +57,54 @@ the long tail of small projects drops out automatically. ## The one convention -Every tracked downstream honors `_BRANCH` (environment or CMake cache -variable): when set, `find_or_fetch` fetches that dependency at the given ref -instead of the committed default. `LIBNEO_BRANCH` is the instance for libneo, -and SIMPLE already implements it. This single hook lets the upstream build any -downstream against any candidate ref with no commit to the downstream. It is the -multi-repo stand-in for a global build graph, and the only piece that has to be -standardized across the codes. +Every tracked downstream resolves each first-party dependency through one ladder +in `find_or_fetch`: + +1. `_REF` (environment or CMake cache): a branch, tag, or commit. It is + validated against the remote and ignored if absent. The upstream sets it to + build the downstream against a candidate. +2. `_RELEASE` (committed cache variable): the release branch the code tracks + by default. Never main. +3. the current branch if it exists in the remote, otherwise main. + +`LIBNEO_REF` and `LIBNEO_RELEASE` are the instances for libneo. The override lets +the upstream build any downstream against any candidate ref with no commit to the +downstream. It is the multi-repo stand-in for a global build graph, and the only +piece standardized across the codes. ## Release model -Release branches, no release-candidate tags. +Codes track the latest libneo release branch. Tags are reproducibility +snapshots, not what a code follows. 1. libneo cuts `release/YY.MINOR` off main. -2. For each tracked downstream, dispatch its integration workflow with - `LIBNEO_BRANCH=release/YY.MINOR` (a `workflow_dispatch` input: no commit, no - pull request). The downstream builds its own main against the candidate. -3. libneo polls the dispatched runs and gates the tag on all of them passing. -4. On all-green, tag `YY.MINOR.0` at the release-branch head. -5. A bot opens a bump pull request in each downstream pinning the tag, never the - branch. The maintainer reviews and merges on their own schedule; no - auto-merge. - -Invariants: - -- A downstream's main only ever pins released tags. Branch refs live only in the - ephemeral dispatch override, never in a committed pin. -- Validation needs green CI, not merged pull requests, so waiting for - maintainers never blocks the upstream release. -- No automatic downstream releases. A downstream releases on its own cadence. - Propagate a release only along a real edge, and only when behavior changed - (for example a libneo change that alters NEO-2 output, NEO-2 being itself an - upstream of NEO-RT). +2. For each tracked downstream, the release workflow dispatches the downstream's + own CI with `LIBNEO_REF=release/YY.MINOR` (a `workflow_dispatch` input: no + commit, no pull request). The downstream builds and tests its main against the + candidate, golden records included. +3. libneo polls the dispatched runs and gates on all of them passing. +4. On all-green, a bot opens a bump pull request in each pinning downstream that + sets `LIBNEO_RELEASE` to the new branch, and enables auto-merge. Each + downstream's own CI gates that merge; no human step. +5. Tagging `YY.MINOR.PATCH` at the branch head is a separate, on-demand step. The + tag is a citation and reproducibility snapshot; downstreams keep tracking the + branch. + +Properties: + +- A downstream's committed pin is a release branch, never main. Exact reproduction + uses a tag. +- Validation needs green CI, not merged pull requests, so a slow downstream never + blocks the upstream. +- The gate dispatches each downstream's existing CI by name (`release/downstreams` + lists the workflow file per repo). There is no separate integration workflow to + maintain. +- Transitive consumers are listed explicitly. NEO-RT reaches libneo through NEO-2, + so its CI sets `LIBNEO_REF` as an environment variable and it propagates to the + libneo fetch inside the NEO-2 that NEO-RT builds. +- No automatic downstream releases. A downstream releases on its own cadence, and + only when behavior changed (for example a libneo change that alters NEO-2 + output, NEO-2 being itself an upstream of NEO-RT). Patch releases reuse the branch: cherry-pick onto `release/YY.MINOR`, re-run the downstream dispatch, tag `YY.MINOR.PATCH`. diff --git a/docs/templates/downstream-integration.yml b/docs/templates/downstream-integration.yml index f65c14f..6073c40 100644 --- a/docs/templates/downstream-integration.yml +++ b/docs/templates/downstream-integration.yml @@ -1,35 +1,29 @@ -# Template: downstream reverse-dependency validation (proposed). +# Snippet: make a downstream's existing CI dispatchable by an upstream release. # -# Commit this once into each tracked consumer's .github/workflows/. An upstream -# release workflow dispatches it with the upstream ref to build against. It -# builds THIS code's own default branch against that ref, with no commit and no -# pin change, and the upstream polls the run conclusion to gate its release tag. +# There is no separate integration workflow. Add these three pieces to the +# downstream's existing CI workflow (the one that already builds and tests it). +# An upstream libneo release then dispatches that workflow by name with a +# candidate ref; the run builds the downstream's own main against it, with no +# commit and no pin change, and the upstream polls the conclusion to gate. # -# Prerequisite: this code's find_or_fetch honors LIBNEO_BRANCH (see the -# _BRANCH convention in docs/development-model.md). - -name: integration +# Prerequisite: this code's find_or_fetch honors LIBNEO_REF (see the _REF +# convention in docs/development-model.md). +# 1. Accept a ref from the dispatcher (add to `on:`). on: workflow_dispatch: inputs: libneo_ref: - description: libneo branch or tag to build against - required: true - default: main + description: libneo branch, tag or commit to build against + required: false + default: '' + +# 2. Map it to the resolver override. Empty on push/PR, so normal builds keep the +# committed LIBNEO_RELEASE. Set it at job/workflow level, except where the job +# also builds a reference version against a different libneo (then scope it to +# the single build step under test). +env: + LIBNEO_REF: ${{ inputs.libneo_ref }} -jobs: - build: - runs-on: ubuntu-24.04 - env: - # find_or_fetch fetches libneo at this ref; the committed pin stays untouched - LIBNEO_BRANCH: ${{ inputs.libneo_ref }} - steps: - - uses: actions/checkout@v4 - - name: Build and test - run: | - # Per-code build. CMake codes: - cmake -S . -B build -G Ninja - cmake --build build -j - ctest --test-dir build --output-on-failure - # Make-based codes substitute their own `make && make test`. +# 3. Run the full job (golden records included) on dispatch. If a job `if:` guard +# only admits push/pull_request, add `|| github.event_name == 'workflow_dispatch'`. diff --git a/docs/templates/upstream-release.yml b/docs/templates/upstream-release.yml index ae44729..a4788ef 100644 --- a/docs/templates/upstream-release.yml +++ b/docs/templates/upstream-release.yml @@ -1,66 +1,17 @@ -# Template: upstream release orchestration (proposed skeleton, untested). +# Pointer, not a template. # -# Lives in the hub repo (libneo). Cuts a release branch, validates the tracked -# downstreams against it, gates the tag on all-green, then opens tag-pinned bump -# PRs. The downstream list is the inverted find_or_fetch graph; the set below is -# libneo's current consumers. See docs/development-model.md for the full model. - -name: release - -on: - workflow_dispatch: - inputs: - version: - description: release version YY.MINOR.PATCH (e.g. 26.1.0) - required: true - -env: - DOWNSTREAMS: "SIMPLE NEO-2 NEO-RT MEPHIT KAMEL rabe" - -jobs: - release: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Cut release branch - run: | - V="${{ inputs.version }}" # 26.1.0 - BRANCH="release/${V%.*}" # release/26.1 - git switch -c "$BRANCH" || git switch "$BRANCH" - git push -u origin "$BRANCH" - echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" - - - name: Dispatch downstream validation - env: - GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN }} # needs actions:write on the downstreams - run: | - for d in $DOWNSTREAMS; do - gh workflow run integration.yml -R "itpplasma/$d" -f "libneo_ref=$BRANCH" - done - # TODO: capture each dispatched run id (poll the runs API right after dispatch) - - - name: Gate on all downstreams green - env: - GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN }} - run: | - # TODO: poll each downstream's latest integration run for libneo_ref=$BRANCH - # until conclusion != null; fail this job if any conclusion != success. - echo "TODO: poll conclusions, fail on any non-success" - - - name: Tag the release - run: | - # Only reached when the gate passed. Tag the exact branch head that - # the downstreams validated against. - git tag "${{ inputs.version }}" - git push origin "${{ inputs.version }}" - - - name: Open bump PRs (pin the tag, never the branch) - env: - GH_TOKEN: ${{ secrets.RELEASE_BOT_TOKEN }} - run: | - # TODO: for each downstream, edit its standardized libneo pin to the - # tag ${{ inputs.version }}, open a PR, assign code owners. No merge. - echo "TODO: open tag-pinned bump PRs; maintainers merge on their own schedule" +# The upstream release gate is implemented, not templated. The canonical version +# lives in the hub repo: +# +# itpplasma/libneo: .github/workflows/release.yml +# release/downstreams (blocking set + per-repo CI file) +# release/check-downstreams.sh (advisory drift check) +# +# It dispatches each downstream's existing CI (by the workflow file named in +# release/downstreams) with LIBNEO_REF set to the release branch, polls each run, +# and gates on all-green. On a new release branch it opens auto-merge bump PRs +# that set each pinning downstream's LIBNEO_RELEASE to that branch; tagging +# YY.MINOR.PATCH is a separate reproducibility snapshot. +# +# Copy that workflow into another hub (for example fortplot) and swap libneo for +# the hub name. See docs/development-model.md for the model.