Rift o4d junior calmarg in loop: AFTER main 'distance' merge

.github/workflows/ci.yml

@@ -2,10 +2,14 @@ name: CI
 
 on:
   push:
-    branches: [rift_O4d, rift_O4d_junior]
+    branches: [rift_O4d, rift_O4d_junior, rift_O4d_gmm_gpu]
   pull_request:
-    branches: [rift_O4d, rift_O4d_junior]
+    branches: [rift_O4d, rift_O4d_junior, rift_O4d_gmm_gpu]
   workflow_dispatch:
+  schedule:
+    # Weekly canary so a fresh UPSTREAM release (e.g. swig>=4.4.0 -- see #136)
+    # is caught even when there is no RIFT commit. Mondays 06:00 UTC.
+    - cron: '0 6 * * 1'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -17,7 +21,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.10', '3.11', '3.12', '3.13']
+        # Smoke-test that the package resolves and installs across the supported
+        # Python range, including the legacy py3.9 lane (paired with the pinned
+        # numpy==1.24.4 used by the integrator gate below).
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -60,11 +67,27 @@ jobs:
   import-check:
     needs: install
     runs-on: ubuntu-latest
+    # Verify every declared module imports cleanly under both the pinned
+    # legacy lane (py3.9 + numpy 1.24.4) and the modern lane (py3.12 +
+    # numpy 2.x). Catches platform-portability regressions like the
+    # np.float128 import-time crash on numpy 2.x systems without an
+    # extended-precision long double.
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - lane: legacy
+            python-version: '3.9'
+            numpy-pin: 'numpy==1.24.4'
+          - lane: modern
+            python-version: '3.12'
+            numpy-pin: 'numpy>=2.0,<3.0'
+    name: import-check (${{ matrix.lane }} py${{ matrix.python-version }})
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: ${{ matrix.python-version }}
           cache: 'pip'
           cache-dependency-path: requirements.txt
       - name: Enable symlink
@@ -73,8 +96,14 @@ jobs:
         run: |
           python -m pip install --upgrade pip --break-system-packages
           python -m pip install -r requirements.txt --break-system-packages
+          # Pin numpy AFTER requirements.txt so it overrides the unpinned
+          # 'numpy' line in requirements.txt without changing the file.
+          python -m pip install '${{ matrix.numpy-pin }}' --break-system-packages
           python -m pip install coverage pytest --break-system-packages
           python -m pip install --editable . --break-system-packages
+      - name: Show resolved versions
+        run: |
+          python -c "import sys, numpy, scipy; print('python', sys.version); print('numpy', numpy.__version__); print('scipy', scipy.__version__)"
       - name: Run import check
         run: python .travis/test-all-mod.py
 
@@ -108,11 +137,29 @@ jobs:
   test-run:
     needs: install
     runs-on: ubuntu-latest
+    # Integrator + posterior gate. We run this in two CI lanes:
+    #   - legacy : py3.9 + numpy 1.24.4 -- the historically known-good
+    #              configuration on Linux x86_64 where np.float128 is real.
+    #   - modern : py3.12 + numpy 2.x -- the forward-looking target. Catches
+    #              numpy 2.x removals (np.product, np.cumproduct, np.in1d,
+    #              np.alltrue, np.float_) and scipy >= 1.16 mvnun removal.
+    # Both lanes must pass test-integrate.sh's GMM/AC/AV consistency check.
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - lane: legacy
+            python-version: '3.9'
+            numpy-pin: 'numpy==1.24.4'
+          - lane: modern
+            python-version: '3.12'
+            numpy-pin: 'numpy>=2.0,<3.0'
+    name: test-run (${{ matrix.lane }} py${{ matrix.python-version }})
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: ${{ matrix.python-version }}
           cache: 'pip'
           cache-dependency-path: requirements.txt
       - name: Enable symlink
@@ -121,8 +168,14 @@ jobs:
         run: |
           python -m pip install --upgrade pip --break-system-packages
           python -m pip install -r requirements.txt --break-system-packages
+          # Pin numpy AFTER requirements.txt so it overrides the unpinned
+          # 'numpy' line in requirements.txt without changing the file.
+          python -m pip install '${{ matrix.numpy-pin }}' --break-system-packages
           python -m pip install coverage pytest pytest-cov --break-system-packages
           python -m pip install --editable . --break-system-packages
+      - name: Show resolved versions
+        run: |
+          python -c "import sys, numpy, scipy; print('python', sys.version); print('numpy', numpy.__version__); print('scipy', scipy.__version__)"
       - name: Run test scripts
         run: |
           . .travis/test-coord.sh
@@ -135,11 +188,70 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v4
         with:
-          name: test-logs
+          name: test-logs-${{ matrix.lane }}-py${{ matrix.python-version }}
           path: |
             **/*.log
             **/test-results/*.xml
 
+  container-dep-canary:
+    # Dependency-resolution canary for the container build. The container ships
+    # an UNPINNED dependency set (containers/requirements-container.txt) and
+    # clones RIFT at build time, so a fresh upstream release can silently break
+    # RIFT and only surface when a container rebuild fails (e.g. swig>=4.4.0,
+    # issue #136). This job installs that same unpinned set + exercises the pixi
+    # swig-post44 deployment lane and runs the import check, so we get an early
+    # warning. Runs on push/PR AND weekly (see on.schedule).
+    #
+    # Non-blocking: it tracks UPSTREAM changes outside any PR author's control,
+    # so a red run should alert maintainers, not block unrelated PRs.
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'   # matches the container's python3.10
+      - name: Enable symlink
+        run: sudo ln -sf $(which python3) /usr/bin/python
+      - name: Install system dependencies
+        run: sudo apt-get update && sudo apt-get install -y libgsl-dev
+      - name: Install UNPINNED container dependency set (latest upstream)
+        run: |
+          python -m pip install --upgrade pip --break-system-packages
+          # Strip the GPU-only cupy line: there is no GPU/driver in CI and the
+          # canary's goal is dependency RESOLUTION + RIFT import, not cupy exec.
+          # (cupy is not actually listed in requirements-container.txt, but be
+          # defensive in case it is added later.)
+          grep -viE '^\s*cupy' containers/requirements-container.txt > /tmp/req-canary.txt
+          python -m pip install -r /tmp/req-canary.txt --break-system-packages
+          python -m pip install --editable . --break-system-packages
+      - name: Show resolved versions
+        run: |
+          python -c "import sys, numpy, scipy; print('python', sys.version); print('numpy', numpy.__version__); print('scipy', scipy.__version__)"
+      - name: Import check (latest container deps)
+        run: python .travis/test-all-mod.py
+
+  container-swig-canary:
+    # Companion to container-dep-canary: exercise the pixi swig-post44 lane,
+    # which is the direct issue-#136 detector (swig>=4.4.0 breaking RIFT's
+    # generated bindings). Kept as its own job so a swig failure is distinct
+    # from a general dependency-resolution failure. Also non-blocking.
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install system dependencies
+        run: sudo apt-get update && sudo apt-get install -y libgsl-dev
+      - uses: prefix-dev/setup-pixi@v0.8.1
+        with:
+          environments: swig-post44
+      - name: swig version (post-4.4 lane)
+        run: pixi run -e swig-post44 swig-version
+      - name: Install RIFT (post-4.4 swig lane)
+        run: pixi run -e swig-post44 install-rift
+      - name: Import check (post-4.4 swig lane)
+        run: pixi run -e swig-post44 import-check
+
   docs:
     runs-on: ubuntu-latest
     permissions:

.gitlab-ci.yml

@@ -16,6 +16,29 @@ stages:
 #  - docker image
 #  - deploy
 
+.pixi_template:
+  stage: system tests
+  before_script: []
+  variables:
+    PIXI_HOME: "$CI_PROJECT_DIR/.pixi"
+    PIXI_CACHE_DIR: "$CI_PROJECT_DIR/.pixi-cache"
+    XDG_CACHE_HOME: "$CI_PROJECT_DIR/.pixi-cache/xdg"
+    PATH: "$CI_PROJECT_DIR/.pixi/bin:$PATH"
+  cache:
+    key: pixi-$CI_JOB_NAME
+    paths:
+      - .pixi/
+      - .pixi-cache/
+      - .pixi/envs/
+  script:
+    - apt-get update --assume-yes && apt-get install --assume-yes ca-certificates curl git
+    - curl -fsSL https://pixi.sh/install.sh | bash
+    - export PATH="$PIXI_HOME/bin:$PATH"
+    - pixi --version
+    - pixi run -e "$PIXI_ENV" swig-version
+    - pixi run -e "$PIXI_ENV" install-rift
+    - pixi run -e "$PIXI_ENV" import-check
+
 .install_docker_dependencies:
   before_script:
      - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
@@ -66,6 +89,16 @@ test_run:
     -  bash .travis/test-run-alts.sh
     -  bash .travis/test-build.sh
 
+pixi_swig_pre44:
+  extends: .pixi_template
+  variables:
+    PIXI_ENV: swig-pre44
+
+pixi_swig_post44:
+  extends: .pixi_template
+  variables:
+    PIXI_ENV: swig-post44
+
 # build:test:
 #   image: docker:latest
 #   stage: docker image

.travis/test-build.sh

@@ -1,10 +1,64 @@
 #! /bin/bash
-# This is just a pipeline build test. The coinc file is from a synthetic event.
+# Pipeline build test. The coinc file is from a synthetic event.
+#
+# Builds (does not submit) RIFT DAGs from a reference ini + coinc using fake
+# data, and verifies that the per-distance likelihood export flags (Plan A
+# density grid, Plan B fixed-distance slices) thread through
+# util_RIFT_pseudo_pipe.py -> create_event_parameter_pipeline_* and land in the
+# correct condor submit file (ILE_extr.sub, the extrinsic stage).
+
+set -e
 
 export RIFT_LOWLATENCY=True
 export SINGULARITY_RIFT_IMAGE=foo
 # SINGULARITY_RIFT_IMAGE=/cvmfs/singularity.opensciencegrid.org/james-clark/research-projects-rit/rift:test
 export SINGULARITY_BASE_EXE_DIR=/usr/bin/
 alias gw_data_find=/bin/true  # don't want to reall do the datafind job
 touch foo.cache
-util_RIFT_pseudo_pipe.py --use-ini  `pwd`/.travis/ref_ini/GW150914.ini --use-coinc `pwd`/.travis/ref_ini/coinc.xml --use-rundir `pwd`/test_build_pipe --fake-data-cache `pwd`/foo.cache
+
+REF_INI=`pwd`/.travis/ref_ini/GW150914.ini
+COINC=`pwd`/.travis/ref_ini/coinc.xml
+
+# require a flag to be present in a file
+assert_has() {  # file pattern
+    if ! grep -q -- "$2" "$1"; then
+        echo "FAIL: expected '$2' in $1"; exit 1
+    fi
+}
+# require a flag to be absent from a file
+assert_absent() {  # file pattern
+    if grep -q -- "$2" "$1"; then
+        echo "FAIL: did not expect '$2' in $1"; exit 1
+    fi
+}
+
+# --- 1. baseline build (original smoke test) ---
+util_RIFT_pseudo_pipe.py --use-ini $REF_INI --use-coinc $COINC --use-rundir `pwd`/test_build_pipe --fake-data-cache `pwd`/foo.cache
+
+# --- 2. Plan-A distance-grid export, threaded onto the extrinsic stage ---
+# Distance marginalization must stay ON for the intrinsic ILE jobs (speedup)
+# and be disabled ONLY at the extrinsic export stage. The trailing space in
+# the pattern matches the standalone --distance-marginalization flag but not
+# --distance-marginalization-lookup-table.
+util_RIFT_pseudo_pipe.py --use-ini $REF_INI --use-coinc $COINC --use-rundir `pwd`/test_build_grid --fake-data-cache `pwd`/foo.cache --add-extrinsic --export-marginal-distance-grid
+assert_has    `pwd`/test_build_grid/ILE_extr.sub "--export-marginal-distance-grid"
+assert_has    `pwd`/test_build_grid/ILE_extr.sub "--internal-use-lnL"
+assert_absent `pwd`/test_build_grid/ILE.sub      "--export-marginal-distance-grid"
+assert_has    `pwd`/test_build_grid/args_ile.txt "--distance-marginalization "
+assert_has    `pwd`/test_build_grid/ILE.sub      "--distance-marginalization "
+assert_absent `pwd`/test_build_grid/ILE_extr.sub "--distance-marginalization "
+echo "OK: Plan-A grid export only on ILE_extr.sub; distance marginalization disabled only at the extrinsic stage"
+
+# --- 3. Plan-B distance-slice export, threaded onto the extrinsic stage ---
+util_RIFT_pseudo_pipe.py --use-ini $REF_INI --use-coinc $COINC --use-rundir `pwd`/test_build_slices --fake-data-cache `pwd`/foo.cache --add-extrinsic --export-distance-slices 10 --export-distance-slices-wing-delta-lnL 7.0 --export-distance-slices-skip-threshold 1.0
+assert_has    `pwd`/test_build_slices/ILE_extr.sub "--export-distance-slices 10"
+assert_has    `pwd`/test_build_slices/ILE_extr.sub "--distance-slice-wing-delta-lnL 7.0"
+assert_has    `pwd`/test_build_slices/ILE_extr.sub "--distance-slice-skip-threshold 1.0"
+assert_has    `pwd`/test_build_slices/ILE_extr.sub "--internal-use-lnL"
+assert_absent `pwd`/test_build_slices/ILE.sub      "--export-distance-slices"
+assert_has    `pwd`/test_build_slices/args_ile.txt "--distance-marginalization "
+assert_has    `pwd`/test_build_slices/ILE.sub      "--distance-marginalization "
+assert_absent `pwd`/test_build_slices/ILE_extr.sub "--distance-marginalization "
+echo "OK: Plan-B slice export only on ILE_extr.sub; distance marginalization disabled only at the extrinsic stage"
+
+echo "test-build.sh: all pipeline-build checks passed"