ipopt_outer_bound: a certified outer-bound spoke for convex NLP subproblems

.github/workflows/test_pr_and_main.yml

@@ -1262,7 +1262,9 @@ jobs:
             mpisppy/tests/test_prox_approx.py \
             mpisppy/tests/test_sep_rho.py \
             mpisppy/tests/test_reduced_costs_fixer.py \
-            mpisppy/tests/test_slammer.py
+            mpisppy/tests/test_slammer.py \
+            mpisppy/tests/test_dual_certificate.py \
+            mpisppy/tests/test_ipopt_outer_bound.py
 
       - name: Upload coverage data
         if: always()
@@ -1273,6 +1275,128 @@ jobs:
           if-no-files-found: ignore
           include-hidden-files: true
 
+  ipopt-tests:
+    name: Ipopt tests (idaes-ext build, with HSL)
+    runs-on: ubuntu-latest
+    needs: [ruff]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: test_env
+          python-version: 3.11
+      - name: Install dependencies
+        run: |
+          conda install mpi4py pandas setuptools
+
+      - name: Install Ipopt's system libraries
+        # The idaes-ext binary carries no RPATH and resolves libgfortran,
+        # liblapack and libblas from the system.  A bare runner has none of
+        # them, and the failure surfaces as an unhelpful load error at first
+        # solve rather than at install.  Pyomo's workflow installs the same
+        # three for the same reason.
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libopenblas-dev gfortran liblapack-dev
+          # glpk is not needed by Ipopt; it gives the certificate a second,
+          # independent opinion to be checked against. See
+          # TestAgreesWithLagrangian -- the Lagrangian spoke needs a solver that
+          # reports a dual bound, which Ipopt by definition does not, and which
+          # the bundle's cbc turns out not to do on this runner.
+          sudo apt-get install -y glpk-utils
+
+      - name: Install Ipopt from idaes-ext
+        # The pip/conda ipopt is built against MUMPS only.  The idaes-ext
+        # release bundle additionally carries the HSL linear solvers (ma27,
+        # ma57, ma97), which is what makes it usable on real NLPs.  This is the
+        # same source Pyomo's and Egret's own CI pull from.
+        run: |
+          SOLVER_DIR="${GITHUB_WORKSPACE}/cache/solvers"
+          mkdir -p "$SOLVER_DIR"
+          echo "$SOLVER_DIR" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SOLVER_DIR" >> $GITHUB_ENV
+          URL=https://github.com/IDAES/idaes-ext
+          RELEASE=$(curl --max-time 150 --retry 8 -L -s \
+              -H 'Accept: application/json' ${URL}/releases/latest)
+          VER=$(echo $RELEASE | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/')
+          # Fall back to a known-good release if the API is unavailable or
+          # rate-limited, so a GitHub hiccup does not read as a test failure.
+          if test -z "$VER" -o "$VER" = "$RELEASE"; then VER=3.4.2; fi
+          echo "idaes-ext version: $VER"
+          curl --max-time 300 --retry 8 -L \
+              ${URL}/releases/download/$VER/idaes-solvers-ubuntu2204-x86_64.tar.gz \
+              > solvers.tar.gz
+          tar -xzf solvers.tar.gz -C "$SOLVER_DIR"
+          "$SOLVER_DIR/ipopt" -v
+
+      - name: setup the program
+        run: |
+          pip install -e ".[test]"
+
+      - name: confirm the HSL linear solvers are really present
+        # Guards against silently falling back to a MUMPS-only build: a missing
+        # HSL solver makes ipopt exit abnormally rather than warn.
+        run: |
+          python -c "
+          import pyomo.environ as pyo
+          for ls in ('ma27', 'ma57', 'ma97'):
+              m = pyo.ConcreteModel()
+              m.x = pyo.Var(bounds=(-10, 10), initialize=0.0)
+              m.o = pyo.Objective(expr=(m.x - 3)**2)
+              m.c = pyo.Constraint(expr=m.x <= 1)
+              opt = pyo.SolverFactory('ipopt')
+              opt.options['linear_solver'] = ls
+              opt.solve(m)
+              assert abs(pyo.value(m.x) - 1.0) < 1e-6, ls
+              print(ls, 'OK')
+          "
+          echo ""
+          echo "=============================================================================="
+          echo "This build of Ipopt links HSL, and defaults to the ma27 linear solver, so"
+          echo "the tests below solve with HSL rather than with MUMPS."
+          echo "HSL, a collection of Fortran codes for large-scale scientific computation."
+          echo "See https://www.hsl.rl.ac.uk/"
+          echo "=============================================================================="
+
+      - name: run ipopt-dependent tests
+        timeout-minutes: 10
+        run: |
+          coverage run $COV_ARGS -m pytest -v \
+            mpisppy/tests/test_dual_certificate.py \
+            mpisppy/tests/test_ipopt_outer_bound.py
+
+      - name: run the ipopt_outer_bound spoke on two ranks
+        timeout-minutes: 10
+        run: |
+          mpiexec -np 2 coverage run $COV_ARGS -m mpi4py -m pytest -v \
+            mpisppy/tests/test_ipopt_outer_bound.py
+
+      - name: smoke-test the driver command line
+        # Stands in for a run_all.py entry: it exercises the same thing (the
+        # documented command line still works end to end) without forcing an
+        # ipopt install into the two run_all CI jobs, which have no other use
+        # for one.
+        timeout-minutes: 10
+        run: |
+          cd examples/farmer
+          mpiexec -np 2 coverage run $COV_ARGS -m mpi4py \
+            ../../mpisppy/generic_cylinders.py \
+            --module-name farmer --num-scens 3 --solver-name ipopt \
+            --max-iterations 5 --default-rho 1.0 --ipopt-outer-bound
+
+      - name: List coverage files (debug)
+        if: always()
+        run: ls -la ${{ github.workspace }}/.coverage* || echo "No .coverage files found"
+
+      - name: Upload coverage data
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-ipopt-tests
+          path: ${{ github.workspace }}/.coverage.*
+          if-no-files-found: ignore
+          include-hidden-files: true
+
   coverage-report:
     name: Coverage Report
     runs-on: ubuntu-latest
@@ -1303,6 +1427,7 @@ jobs:
       - test-cg
       - test-agnostic
       - unit-tests
+      - ipopt-tests
     if: always()
     steps:
       - uses: actions/checkout@v3

.gitignore

@@ -158,8 +158,18 @@ examples/**/*_full_solution/
 examples/**/*_pickles/
 examples/**/*_cyl_nonants.npy
 
-# LaTeX build products under doc/slides (the .tex and the built .pdf are
-# committed; everything else pdflatex/biber leaves behind is not)
+# LaTeX build products under doc/slides and doc/designs (the .tex and the
+# built .pdf are committed; everything else pdflatex/biber leaves behind is not)
+doc/designs/**/*.aux
+doc/designs/**/*.bbl
+doc/designs/**/*.bcf
+doc/designs/**/*.blg
+doc/designs/**/*.fdb_latexmk
+doc/designs/**/*.fls
+doc/designs/**/*.out
+doc/designs/**/*.run.xml
+doc/designs/**/*.synctex.gz
+doc/designs/**/*.toc
 doc/slides/**/*.aux
 doc/slides/**/*.bbl
 doc/slides/**/*.bcf