debugging

Author: ggorman

.github/workflows/pytest-gpu.yml

@@ -15,18 +15,17 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  OUTPUT_PATH: ${{ github.workspace }}
   RESOURCE_GROUP: CI-gpu
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
   workflow_dispatch:
     inputs:
       tags:
-        description: 'Run GPU tests'
+        description: "Run GPU tests"
 
 jobs:
   build:
@@ -38,42 +37,44 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [
-          pytest-gpu-acc-nvidia,
-          pytest-gpu-omp-amd
-        ]
-        test_examples: ["examples/seismic/tti/tti_example.py examples/seismic/acoustic/acoustic_example.py examples/seismic/viscoacoustic/viscoacoustic_example.py examples/seismic/viscoelastic/viscoelastic_example.py examples/seismic/elastic/elastic_example.py"]
+        name: [pytest-gpu-acc-nvidia, pytest-gpu-omp-amd]
+        test_examples:
+          - examples/seismic/tti/tti_example.py
+          - examples/seismic/acoustic/acoustic_example.py
+          - examples/seismic/viscoacoustic/viscoacoustic_example.py
+          - examples/seismic/viscoelastic/viscoelastic_example.py
+          - examples/seismic/elastic/elastic_example.py
 
         include:
-        # -------------------- NVIDIA job --------------------
-        - name: pytest-gpu-acc-nvidia
-          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py"
-          base: "devitocodes/bases:nvidia-nvc"
-          runner_label: nvidiagpu
-          test_drive_cmd: "nvidia-smi"
-          # Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device.
-          # NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.).
-          flags: >-
-            --init --rm -t
-            --name ${CONTAINER_BASENAME}
-            --env CUDA_VISIBLE_DEVICES
-            --gpus "device=${CUDA_VISIBLE_DEVICES:-all}"
-
-        # -------------------- AMD job -----------------------
-        - name: pytest-gpu-omp-amd
-          test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py"
-          runner_label: amdgpu
-          base: "devitocodes/bases:amd"
-          test_drive_cmd: "rocm-smi"
-          # Unchanged, still passes through required /dev nodes etc.
-          flags: >-
-            --init --network=host
-            --device=/dev/kfd --device=/dev/dri
-            --ipc=host
-            --group-add video --group-add "$(getent group render | cut -d: -f3)"
-            --cap-add=SYS_PTRACE --security-opt seccomp=unconfined
-            --rm -t
-            --name ${CONTAINER_BASENAME}
+          # -------------------- NVIDIA job --------------------
+          - name: pytest-gpu-acc-nvidia
+            test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openacc.py"
+            base: "devitocodes/bases:nvidia-nvc"
+            runner_label: nvidiagpu
+            test_drive_cmd: "nvidia-smi"
+            # Respect CUDA_VISIBLE_DEVICES and also hard-limit Docker to that device.
+            # NOTE: CUDA_VISIBLE_DEVICES must be set by the runner (systemd drop-in etc.).
+            flags: >-
+              --init --rm -t
+              --name ${CONTAINER_BASENAME}
+              --env CUDA_VISIBLE_DEVICES
+              --gpus "device=${CUDA_VISIBLE_DEVICES:-all}"
+
+          # -------------------- AMD job -----------------------
+          - name: pytest-gpu-omp-amd
+            test_files: "tests/test_adjoint.py tests/test_gpu_common.py tests/test_gpu_openmp.py"
+            runner_label: amdgpu
+            base: "devitocodes/bases:amd"
+            test_drive_cmd: "rocm-smi"
+            # Unchanged, still passes through required /dev nodes etc.
+            flags: >-
+              --init --network=host
+              --device=/dev/kfd --device=/dev/dri
+              --ipc=host
+              --group-add video --group-add "$(getent group render | cut -d: -f3)"
+              --cap-add=SYS_PTRACE --security-opt seccomp=unconfined
+              --rm -t
+              --name ${CONTAINER_BASENAME}
 
     steps:
       - name: Checkout devito
@@ -84,11 +85,16 @@ jobs:
           echo "DOCKER_IMAGE=${{ matrix.name }}-${RUNNER_NAME// /_}" >> $GITHUB_ENV
           echo "CONTAINER_BASENAME=testrun-${{ matrix.name }}-${RUNNER_NAME// /_}" >> $GITHUB_ENV
 
+      - name: Ensure buildx builder
+        run: |
+          docker buildx inspect "$RUNNER_NAME" >/dev/null 2>&1 || \
+            docker buildx create --name "$RUNNER_NAME" --driver docker-container
+          docker buildx use "$RUNNER_NAME"
+
       - name: Build docker image
         run: |
-          docker buildx create --use --name $RUNNER_NAME || true
-          docker build . \
-            --builder $RUNNER_NAME \
+          docker buildx build . \
+            --builder "$RUNNER_NAME" \
             --label ci-run=$GITHUB_RUN_ID \
             --rm --pull \
             --file docker/Dockerfile.devito \
@@ -138,7 +144,9 @@ jobs:
             -e CODECOV_TOKEN \
             ${NVIDIA_ENV_FLAGS} \
             "${DOCKER_IMAGE}" \
-            pytest -vv -ra -l -s --full-trace --maxfail=1 \
+            pytest -vvv --capture=no --showlocals \
+                   --log-cli-level=DEBUG -o log_cli=true \
+                   --full-trace --durations=10 \
                    --cov --cov-config=.coveragerc --cov-report=xml \
                    ${{ matrix.test_files }}
 
@@ -160,3 +168,8 @@ jobs:
           # Guard global prune the layers we created
           # (label ci-run=$GITHUB_RUN_ID)
           docker image prune -f --filter label=ci-run=$GITHUB_RUN_ID
+
+          docker builder prune -f --filter "name=$RUNNER_NAME" --filter "until=168h"
+
+          # May be overkill - commented out for now but left here as a reminder
+          # docker buildx rm $RUNNER_NAME || true