diff --git a/gpu-validation/defaults/main.yaml b/gpu-validation/defaults/main.yaml index 529ad5e..2e6f724 100644 --- a/gpu-validation/defaults/main.yaml +++ b/gpu-validation/defaults/main.yaml @@ -121,3 +121,52 @@ gpu_validation_reboot_force: false gpu_validation_reboot_timeout: 3600 # [int] Post-reboot delay in seconds gpu_validation_reboot_post_delay: 60 + +# ── NVSwitch validation defaults (nvswitch_validation.yaml playbook) ────────── +# gpu_validation_nvswitch_validation_fabric_mode is intentionally NOT defaulted here. +# tasks_from in include_role is rendered in the play context before role defaults +# are loaded, so a role default cannot serve as a reliable selector. +# Callers MUST supply gpu_validation_nvswitch_validation_fabric_mode via extra-vars (-e), +# inventory/group_vars, or an AnsibleTest CR ansibleVarFiles entry. +# Valid values: 0 (PCI grouping), 1 (Shared NVSwitch), 2 (vGPU mdev). + +# [list] Systemd services expected active per mode. +# Override if your installation uses different service names. +gpu_validation_nvswitch_validation_mode0_fm_services: + - nvidia-fabricmanager + - nvlsm +gpu_validation_nvswitch_validation_mode1_fm_services: + - nvidia-fabricmanager + - nvlsm + - fm-partition-restore +gpu_validation_nvswitch_validation_mode2_fm_services: + - nvidia-fabricmanager + - nvlsm + - nvidia-vgpu-mgr + +# [int] Mode 2 only: expected number of mdev instances (from mdevctl list). +# Set to 0 to skip the assertion. Typically equals gpu_validation_nvswitch_validation_expected_gpu_count. +gpu_validation_nvswitch_validation_mode2_expected_mdev_count: 0 + +# [string] GPU PCI vendor ID +gpu_validation_nvswitch_validation_gpu_vendor_id: "10de" +# [string] GPU PCI product ID — HGX B300 GB110 SXM6 AC +gpu_validation_nvswitch_validation_gpu_product_id: "3182" +# [int] Total number of GPUs expected on the compute host +gpu_validation_nvswitch_validation_expected_gpu_count: 8 +# [bool] Check that all GPUs reach the expected Fabric State +gpu_validation_nvswitch_validation_fabric_state_check: true +# [string] Expected nvidia-smi Fabric State value +gpu_validation_nvswitch_validation_fabric_state_expected: "Completed" +# [int] Expected number of activated NVSwitch partitions; set after first fmpm -l run +gpu_validation_nvswitch_validation_expected_partition_count: 0 +# [int] Number of GPUs per passthrough VM flavor +gpu_validation_nvswitch_validation_gpus_per_vm: 1 +# [string] Nova resource class for the GPU passthrough flavor +gpu_validation_nvswitch_validation_passthrough_resource_class: "CUSTOM_HGX-B300-GPU" +# [string] Expected libvirt device type for passthrough GPUs in virsh dumpxml +gpu_validation_nvswitch_validation_expected_device_type: "hostdev" +# [bool] Check that FABRIC_MODE in fabricmanager.cfg matches gpu_validation_nvswitch_validation_fabric_mode +gpu_validation_nvswitch_validation_check_fabric_mode_cfg: false +# [string] Path to the FM configuration file +gpu_validation_nvswitch_validation_fabric_mode_cfg_file: /usr/share/nvidia/nvswitch/fabricmanager.cfg diff --git a/gpu-validation/tasks/nvswitch.yaml b/gpu-validation/tasks/nvswitch.yaml new file mode 100644 index 0000000..7e1baa6 --- /dev/null +++ b/gpu-validation/tasks/nvswitch.yaml @@ -0,0 +1,83 @@ +--- +# host-side checks: FM services, fabric state, partition activation. + +- name: Collect FM service status + become: true + ansible.builtin.systemd: + name: "{{ item }}" + loop: "{{ gpu_validation_nvswitch_validation_fm_services }}" + register: _fm_service_status + failed_when: false + +- name: Set FM service status facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fm_service_results: "{{ _fm_service_status.results + | map(attribute='item') | zip(_fm_service_status.results + | map(attribute='status') | map(attribute='ActiveState')) + | list }}" + +- name: Show FM service states + ansible.builtin.debug: + msg: "{{ item[0] }}: {{ item[1] }}" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Collect Fabric State for all GPUs + become: true + ansible.builtin.shell: | + set -o pipefail + nvidia-smi -q | grep -A2 'Fabric' | grep 'State' | awk '{print $NF}' + args: + executable: /bin/bash + register: _fabric_states + changed_when: false + failed_when: false + +- name: Show fabric states + ansible.builtin.debug: + msg: "GPU Fabric States: {{ _fabric_states.stdout_lines }}" + +- name: Set fabric state facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fabric_states: "{{ _fabric_states.stdout_lines | map('trim') | list }}" + gpu_validation_nvswitch_fabric_completed_count: >- + {{ _fabric_states.stdout_lines + | map('trim') + | select('equalto', gpu_validation_nvswitch_validation_fabric_state_expected) + | list | length }} + +- name: Collect fmpm partition list + become: true + ansible.builtin.command: + cmd: fmpm -l + register: _fmpm_list + changed_when: false + failed_when: false + +- name: Show fmpm partition list + ansible.builtin.debug: + msg: "{{ _fmpm_list.stdout | default('fmpm not available or no output') }}" + +- name: Set partition facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fmpm_output: "{{ _fmpm_list.stdout | default('') }}" + gpu_validation_nvswitch_activated_partition_count: >- + {{ (_fmpm_list.stdout | default('')) + .splitlines() + | select('search', '\\bactivated\\b') + | list | length }} + +- name: Collect NVLink topology matrix + become: true + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _nvlink_topo + changed_when: false + failed_when: false + +- name: Show NVLink topology matrix + ansible.builtin.debug: + msg: "{{ _nvlink_topo.stdout | default('nvidia-smi topo not available') }}" + +- name: Set topology fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_topo_output: "{{ _nvlink_topo.stdout | default('') }}" diff --git a/gpu-validation/tasks/nvswitch_assertions.yaml b/gpu-validation/tasks/nvswitch_assertions.yaml new file mode 100644 index 0000000..aa131f0 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_assertions.yaml @@ -0,0 +1,41 @@ +--- +# assertions: FM services active, fabric state, partition count. + +- name: Assert FM services are active + ansible.builtin.assert: + that: + - item[1] == 'active' + fail_msg: "Service {{ item[0] }} is not active (state: {{ item[1] }})" + success_msg: "Service {{ item[0] }} is active" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Assert all GPUs reached Fabric State Completed + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_fabric_states | length == gpu_validation_nvswitch_validation_expected_gpu_count | int + - gpu_validation_nvswitch_fabric_completed_count | int == gpu_validation_nvswitch_validation_expected_gpu_count | int + fail_msg: >- + Discovered {{ gpu_validation_nvswitch_fabric_states | length }} GPU(s), expected {{ gpu_validation_nvswitch_validation_expected_gpu_count }}. + Only {{ gpu_validation_nvswitch_fabric_completed_count }} of + {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }}. + States seen: {{ gpu_validation_nvswitch_fabric_states }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }} + when: gpu_validation_nvswitch_validation_fabric_state_check | bool + +- name: Assert expected number of activated partitions + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_activated_partition_count | int == gpu_validation_nvswitch_validation_expected_partition_count | int + fail_msg: >- + fmpm -l shows {{ gpu_validation_nvswitch_activated_partition_count }} activated partitions, + expected {{ gpu_validation_nvswitch_validation_expected_partition_count }}. + Full fmpm output: + {{ gpu_validation_nvswitch_fmpm_output }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_expected_partition_count }} NVSwitch partitions + are activated + when: + - gpu_validation_nvswitch_validation_expected_partition_count | int > 0 diff --git a/gpu-validation/tasks/nvswitch_isolation.yaml b/gpu-validation/tasks/nvswitch_isolation.yaml new file mode 100644 index 0000000..0d931e2 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_isolation.yaml @@ -0,0 +1,43 @@ +--- +# In-VM isolation checks: GPU count, NVLink topology per VM. +# Runs on the [gpu_vms] group (the passthrough-flavor VMs). + +- name: Count GPUs visible inside VM + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=name --format=csv,noheader + register: _vm_gpus + changed_when: false + failed_when: false + +- name: Show GPU list inside VM + ansible.builtin.debug: + msg: "GPUs in VM {{ inventory_hostname }}: {{ _vm_gpus.stdout_lines }}" + +- name: Set GPU count fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_gpu_count: "{{ _vm_gpus.stdout_lines | length }}" + +- name: Collect in-VM NVLink topology matrix + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _vm_topo + changed_when: false + failed_when: false + +- name: Show in-VM NVLink topology matrix + ansible.builtin.debug: + msg: "{{ _vm_topo.stdout | default('nvidia-smi topo not available') }}" + +- name: Set in-VM topology facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_topo_output: "{{ _vm_topo.stdout | default('') }}" + gpu_validation_nvswitch_vm_nvlink_lines: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('search', '\\bNV[0-9]') + | list }} + gpu_validation_nvswitch_vm_topo_gpu_count: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('match', '^GPU\\d') + | list | length | int }} diff --git a/gpu-validation/tasks/nvswitch_isolation_assertions.yaml b/gpu-validation/tasks/nvswitch_isolation_assertions.yaml new file mode 100644 index 0000000..e1e8e2c --- /dev/null +++ b/gpu-validation/tasks/nvswitch_isolation_assertions.yaml @@ -0,0 +1,44 @@ +--- +# assertions: GPU count per VM matches flavor, no cross-partition NVLink. + +- name: Assert GPU count matches passthrough flavor + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s), + expected {{ gpu_validation_nvswitch_validation_gpus_per_vm }} per the passthrough flavor. + GPUs seen: {{ _vm_gpus.stdout_lines }} + success_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s) as expected + +- name: Assert NVLink entries are present (GPUs within partition are connected) + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_nvlink_lines | length > 0 + fail_msg: >- + No NVLink entries found in nvidia-smi topo -m output for + VM {{ inventory_hostname }}. GPUs within the same partition + should show NVLink connectivity. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + NVLink connectivity confirmed within partition for + VM {{ inventory_hostname }} + when: + - gpu_validation_nvswitch_validation_gpus_per_vm | int > 1 + +- name: Assert topo GPU row count matches flavor (no extra GPUs visible from topo matrix) + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_topo_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + nvidia-smi topo -m shows {{ gpu_validation_nvswitch_vm_topo_gpu_count }} GPU row(s) in + VM {{ inventory_hostname }} but flavor specifies + {{ gpu_validation_nvswitch_validation_gpus_per_vm }}. + A mismatch may indicate cross-partition GPU visibility. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + nvidia-smi topo -m GPU row count ({{ gpu_validation_nvswitch_vm_topo_gpu_count }}) matches flavor + — no cross-partition GPU leakage detected in VM {{ inventory_hostname }} diff --git a/gpu-validation/tasks/nvswitch_mode0.yaml b/gpu-validation/tasks/nvswitch_mode0.yaml new file mode 100644 index 0000000..e3724e0 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode0.yaml @@ -0,0 +1,89 @@ +--- +# Mode 0 (FABRIC_MODE=0) host-side checks. +# NVSwitch fabric is open/all-to-all — fmpm is NOT called. +# FM + NVLSM are required to initialize the fabric; no partition tables are programmed. +# Isolation is software-only (CUDA + IOMMU). Suitable for trusted tenants only. + +- name: Collect FABRIC_MODE from fabricmanager.cfg + become: true + ansible.builtin.command: + cmd: "grep -E '^FABRIC_MODE' {{ gpu_validation_nvswitch_validation_fabric_mode_cfg_file }}" + register: _fabric_mode_cfg + changed_when: false + failed_when: false + when: gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool + +- name: Show FABRIC_MODE config value + ansible.builtin.debug: + msg: "fabricmanager.cfg: {{ _fabric_mode_cfg.stdout | default('not found') }}" + when: gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool + +- name: Set FABRIC_MODE config fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_fabric_mode_cfg_value: >- + {{ (_fabric_mode_cfg.stdout | default('') | regex_search('FABRIC_MODE\s*=\s*(\d+)', '\\1') | default([]) | first | default('-1')) | int }} + when: gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool + +- name: Collect FM service status (mode 0) + become: true + ansible.builtin.systemd: + name: "{{ item }}" + loop: "{{ gpu_validation_nvswitch_validation_mode0_fm_services }}" + register: _fm_service_status + failed_when: false + +- name: Set FM service status facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fm_service_results: "{{ _fm_service_status.results + | map(attribute='item') | zip( + _fm_service_status.results + | map(attribute='status') | map(attribute='ActiveState')) + | list }}" + +- name: Show FM service states + ansible.builtin.debug: + msg: "{{ item[0] }}: {{ item[1] }}" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Collect Fabric State for all GPUs + become: true + ansible.builtin.shell: | + set -o pipefail + nvidia-smi -q | grep -A2 'Fabric' | grep 'State' | awk '{print $NF}' + args: + executable: /bin/bash + register: _fabric_states + changed_when: false + failed_when: false + +- name: Show fabric states + ansible.builtin.debug: + msg: "GPU Fabric States: {{ _fabric_states.stdout_lines }}" + +- name: Set fabric state facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fabric_states: "{{ _fabric_states.stdout_lines | map('trim') | list }}" + gpu_validation_nvswitch_fabric_completed_count: >- + {{ _fabric_states.stdout_lines + | map('trim') + | select('equalto', gpu_validation_nvswitch_validation_fabric_state_expected) + | list | length }} + +- name: Collect NVLink topology matrix (open fabric — informational only) + become: true + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _nvlink_topo + changed_when: false + failed_when: false + +- name: Show NVLink topology matrix + ansible.builtin.debug: + msg: | + NOTE: Mode 0 NVSwitch fabric is open/all-to-all. All GPUs will show NVLink + connectivity to each other. Cross-VM NVLink is NOT blocked at the switch level. + {{ _nvlink_topo.stdout | default('nvidia-smi topo not available') }} + +- name: Set topology fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_topo_output: "{{ _nvlink_topo.stdout | default('') }}" diff --git a/gpu-validation/tasks/nvswitch_mode0_assertions.yaml b/gpu-validation/tasks/nvswitch_mode0_assertions.yaml new file mode 100644 index 0000000..5ec13b6 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode0_assertions.yaml @@ -0,0 +1,51 @@ +--- +# Mode 0 (FABRIC_MODE=0) host-side assertions. +# NVSwitch partition check is skipped — fmpm is not used in Mode 0. +# Isolation is CUDA + IOMMU only; NVSwitch routing is open/all-to-all. + +- name: Assert FABRIC_MODE=0 in fabricmanager.cfg + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_fabric_mode_cfg_value | int == gpu_validation_nvswitch_validation_fabric_mode | int + fail_msg: >- + fabricmanager.cfg has FABRIC_MODE={{ gpu_validation_nvswitch_fabric_mode_cfg_value }} + but expected FABRIC_MODE={{ gpu_validation_nvswitch_validation_fabric_mode }}. + A leftover mode from a prior deployment will cause incorrect fabric behaviour. + success_msg: >- + fabricmanager.cfg FABRIC_MODE={{ gpu_validation_nvswitch_fabric_mode_cfg_value }} + matches expected mode {{ gpu_validation_nvswitch_validation_fabric_mode }} + when: + - gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool + - gpu_validation_nvswitch_fabric_mode_cfg_value is defined + +- name: Assert FM services are active (mode 0) + ansible.builtin.assert: + that: + - item[1] == 'active' + fail_msg: "Service {{ item[0] }} is not active (state: {{ item[1] }})" + success_msg: "Service {{ item[0] }} is active" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Assert all GPUs reached Fabric State Completed + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_fabric_states | length == gpu_validation_nvswitch_validation_expected_gpu_count | int + - gpu_validation_nvswitch_fabric_completed_count | int == gpu_validation_nvswitch_validation_expected_gpu_count | int + fail_msg: >- + Discovered {{ gpu_validation_nvswitch_fabric_states | length }} GPU(s), expected {{ gpu_validation_nvswitch_validation_expected_gpu_count }}. + Only {{ gpu_validation_nvswitch_fabric_completed_count }} of + {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }}. + States seen: {{ gpu_validation_nvswitch_fabric_states }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }} + when: gpu_validation_nvswitch_validation_fabric_state_check | bool + +- name: Note — NVSwitch partition isolation is not enforced in Mode 0 + ansible.builtin.debug: + msg: >- + Mode 0 (FABRIC_MODE=0): NVSwitch routing table is open/all-to-all. + fmpm partition check is skipped. + NVLink isolation between tenant VMs relies on CUDA device ownership and + IOMMU DMA boundaries only. This mode is suitable for trusted tenants only. diff --git a/gpu-validation/tasks/nvswitch_mode0_isolation.yaml b/gpu-validation/tasks/nvswitch_mode0_isolation.yaml new file mode 100644 index 0000000..536fb41 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode0_isolation.yaml @@ -0,0 +1,48 @@ +--- +# Mode 0 (FABRIC_MODE=0) in-VM isolation checks. +# Runs on the [gpu_vms] group (VMs booted with the PCI-group flavor). +# Collects GPU count and in-VM NVLink topology. +# NOTE: NVSwitch routing is open — cross-VM NVLink is NOT blocked at the hardware level. + +- name: Count GPUs visible inside VM (mode 0) + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=name --format=csv,noheader + register: _vm_gpus + changed_when: false + failed_when: false + +- name: Show GPU list inside VM + ansible.builtin.debug: + msg: "GPUs in VM {{ inventory_hostname }}: {{ _vm_gpus.stdout_lines }}" + +- name: Set GPU count fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_gpu_count: "{{ _vm_gpus.stdout_lines | length }}" + +- name: Collect in-VM NVLink topology matrix + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _vm_topo + changed_when: false + failed_when: false + +- name: Show in-VM NVLink topology matrix + ansible.builtin.debug: + msg: >- + NOTE: Mode 0 open fabric — NVLink visible here is within the assigned PCI group only + (the VM cannot access GPUs it does not own via PCI). + {{ _vm_topo.stdout | default('nvidia-smi topo not available') }} + +- name: Set in-VM topology facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_topo_output: "{{ _vm_topo.stdout | default('') }}" + gpu_validation_nvswitch_vm_nvlink_lines: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('search', '\\bNV[0-9]') + | list }} + gpu_validation_nvswitch_vm_topo_gpu_count: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('match', '^GPU\\d') + | list | length | int }} diff --git a/gpu-validation/tasks/nvswitch_mode0_isolation_assertions.yaml b/gpu-validation/tasks/nvswitch_mode0_isolation_assertions.yaml new file mode 100644 index 0000000..808525c --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode0_isolation_assertions.yaml @@ -0,0 +1,36 @@ +--- +# Mode 0 (FABRIC_MODE=0) in-VM isolation assertions. +# Asserts the VM sees the full PCI group (all GPUs in CUSTOM_GPU_GROUP_N). +# NVLink cross-VM isolation is NOT asserted — NVSwitch routing is open in Mode 0. + +- name: Assert GPU count matches PCI group size + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s), + expected {{ gpu_validation_nvswitch_validation_gpus_per_vm }} (full PCI group size). + GPUs seen: {{ _vm_gpus.stdout_lines }} + success_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s) — full PCI group assigned + +- name: Assert NVLink entries are present within the PCI group + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_nvlink_lines | length > 0 + fail_msg: >- + No NVLink entries found in nvidia-smi topo -m for VM {{ inventory_hostname }}. + GPUs within the same PCI group should show NVLink connectivity. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + NVLink connectivity confirmed within PCI group for VM {{ inventory_hostname }} + when: + - gpu_validation_nvswitch_validation_gpus_per_vm | int > 1 + +- name: Note — cross-VM NVLink isolation is software-only in Mode 0 + ansible.builtin.debug: + msg: >- + Mode 0 (FABRIC_MODE=0): NVSwitch routing table is open/all-to-all. + Cross-VM NVLink hardware isolation is NOT enforced. This mode is + suitable for trusted tenants only. For hardware isolation use Mode 1 or Mode 2. diff --git a/gpu-validation/tasks/nvswitch_mode1.yaml b/gpu-validation/tasks/nvswitch_mode1.yaml new file mode 100644 index 0000000..0925c7c --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode1.yaml @@ -0,0 +1,84 @@ +--- +# host-side checks: FM services, fabric state, partition activation. + +- name: Collect FM service status + become: true + ansible.builtin.systemd: + name: "{{ item }}" + loop: "{{ gpu_validation_nvswitch_validation_mode1_fm_services }}" + register: _fm_service_status + failed_when: false + +- name: Set FM service status facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fm_service_results: "{{ _fm_service_status.results + | map(attribute='item') | zip( + _fm_service_status.results + | map(attribute='status') | map(attribute='ActiveState')) + | list }}" + +- name: Show FM service states + ansible.builtin.debug: + msg: "{{ item[0] }}: {{ item[1] }}" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Collect Fabric State for all GPUs + become: true + ansible.builtin.shell: | + set -o pipefail + nvidia-smi -q | grep -A2 'Fabric' | grep 'State' | awk '{print $NF}' + args: + executable: /bin/bash + register: _fabric_states + changed_when: false + failed_when: false + +- name: Show fabric states + ansible.builtin.debug: + msg: "GPU Fabric States: {{ _fabric_states.stdout_lines }}" + +- name: Set fabric state facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fabric_states: "{{ _fabric_states.stdout_lines | map('trim') | list }}" + gpu_validation_nvswitch_fabric_completed_count: >- + {{ _fabric_states.stdout_lines + | map('trim') + | select('equalto', gpu_validation_nvswitch_validation_fabric_state_expected) + | list | length }} + +- name: Collect fmpm partition list + become: true + ansible.builtin.command: + cmd: fmpm -l + register: _fmpm_list + changed_when: false + failed_when: false + +- name: Show fmpm partition list + ansible.builtin.debug: + msg: "{{ _fmpm_list.stdout | default('fmpm not available or no output') }}" + +- name: Set partition facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fmpm_output: "{{ _fmpm_list.stdout | default('') }}" + gpu_validation_nvswitch_activated_partition_count: >- + {{ (_fmpm_list.stdout | default('')) + .splitlines() + | select('search', '\\bactivated\\b') + | list | length }} + +- name: Collect NVLink topology matrix + become: true + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _nvlink_topo + changed_when: false + failed_when: false + +- name: Show NVLink topology matrix + ansible.builtin.debug: + msg: "{{ _nvlink_topo.stdout | default('nvidia-smi topo not available') }}" + +- name: Set topology fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_topo_output: "{{ _nvlink_topo.stdout | default('') }}" diff --git a/gpu-validation/tasks/nvswitch_mode1_assertions.yaml b/gpu-validation/tasks/nvswitch_mode1_assertions.yaml new file mode 100644 index 0000000..aa131f0 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode1_assertions.yaml @@ -0,0 +1,41 @@ +--- +# assertions: FM services active, fabric state, partition count. + +- name: Assert FM services are active + ansible.builtin.assert: + that: + - item[1] == 'active' + fail_msg: "Service {{ item[0] }} is not active (state: {{ item[1] }})" + success_msg: "Service {{ item[0] }} is active" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Assert all GPUs reached Fabric State Completed + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_fabric_states | length == gpu_validation_nvswitch_validation_expected_gpu_count | int + - gpu_validation_nvswitch_fabric_completed_count | int == gpu_validation_nvswitch_validation_expected_gpu_count | int + fail_msg: >- + Discovered {{ gpu_validation_nvswitch_fabric_states | length }} GPU(s), expected {{ gpu_validation_nvswitch_validation_expected_gpu_count }}. + Only {{ gpu_validation_nvswitch_fabric_completed_count }} of + {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }}. + States seen: {{ gpu_validation_nvswitch_fabric_states }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }} + when: gpu_validation_nvswitch_validation_fabric_state_check | bool + +- name: Assert expected number of activated partitions + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_activated_partition_count | int == gpu_validation_nvswitch_validation_expected_partition_count | int + fail_msg: >- + fmpm -l shows {{ gpu_validation_nvswitch_activated_partition_count }} activated partitions, + expected {{ gpu_validation_nvswitch_validation_expected_partition_count }}. + Full fmpm output: + {{ gpu_validation_nvswitch_fmpm_output }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_expected_partition_count }} NVSwitch partitions + are activated + when: + - gpu_validation_nvswitch_validation_expected_partition_count | int > 0 diff --git a/gpu-validation/tasks/nvswitch_mode1_isolation.yaml b/gpu-validation/tasks/nvswitch_mode1_isolation.yaml new file mode 100644 index 0000000..0d931e2 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode1_isolation.yaml @@ -0,0 +1,43 @@ +--- +# In-VM isolation checks: GPU count, NVLink topology per VM. +# Runs on the [gpu_vms] group (the passthrough-flavor VMs). + +- name: Count GPUs visible inside VM + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=name --format=csv,noheader + register: _vm_gpus + changed_when: false + failed_when: false + +- name: Show GPU list inside VM + ansible.builtin.debug: + msg: "GPUs in VM {{ inventory_hostname }}: {{ _vm_gpus.stdout_lines }}" + +- name: Set GPU count fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_gpu_count: "{{ _vm_gpus.stdout_lines | length }}" + +- name: Collect in-VM NVLink topology matrix + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _vm_topo + changed_when: false + failed_when: false + +- name: Show in-VM NVLink topology matrix + ansible.builtin.debug: + msg: "{{ _vm_topo.stdout | default('nvidia-smi topo not available') }}" + +- name: Set in-VM topology facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_topo_output: "{{ _vm_topo.stdout | default('') }}" + gpu_validation_nvswitch_vm_nvlink_lines: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('search', '\\bNV[0-9]') + | list }} + gpu_validation_nvswitch_vm_topo_gpu_count: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('match', '^GPU\\d') + | list | length | int }} diff --git a/gpu-validation/tasks/nvswitch_mode1_isolation_assertions.yaml b/gpu-validation/tasks/nvswitch_mode1_isolation_assertions.yaml new file mode 100644 index 0000000..e1e8e2c --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode1_isolation_assertions.yaml @@ -0,0 +1,44 @@ +--- +# assertions: GPU count per VM matches flavor, no cross-partition NVLink. + +- name: Assert GPU count matches passthrough flavor + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s), + expected {{ gpu_validation_nvswitch_validation_gpus_per_vm }} per the passthrough flavor. + GPUs seen: {{ _vm_gpus.stdout_lines }} + success_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s) as expected + +- name: Assert NVLink entries are present (GPUs within partition are connected) + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_nvlink_lines | length > 0 + fail_msg: >- + No NVLink entries found in nvidia-smi topo -m output for + VM {{ inventory_hostname }}. GPUs within the same partition + should show NVLink connectivity. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + NVLink connectivity confirmed within partition for + VM {{ inventory_hostname }} + when: + - gpu_validation_nvswitch_validation_gpus_per_vm | int > 1 + +- name: Assert topo GPU row count matches flavor (no extra GPUs visible from topo matrix) + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_topo_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + nvidia-smi topo -m shows {{ gpu_validation_nvswitch_vm_topo_gpu_count }} GPU row(s) in + VM {{ inventory_hostname }} but flavor specifies + {{ gpu_validation_nvswitch_validation_gpus_per_vm }}. + A mismatch may indicate cross-partition GPU visibility. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + nvidia-smi topo -m GPU row count ({{ gpu_validation_nvswitch_vm_topo_gpu_count }}) matches flavor + — no cross-partition GPU leakage detected in VM {{ inventory_hostname }} diff --git a/gpu-validation/tasks/nvswitch_mode2.yaml b/gpu-validation/tasks/nvswitch_mode2.yaml new file mode 100644 index 0000000..c8c8f91 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode2.yaml @@ -0,0 +1,82 @@ +--- +# Mode 2 (FABRIC_MODE=2) host-side checks. +# nvidia-vgpu-mgr programs NVSwitch partition routing automatically per mdev instance. +# mdevctl list verifies one mdev instance per GPU PF BDF. + +- name: Collect FM service status (mode 2) + become: true + ansible.builtin.systemd: + name: "{{ item }}" + loop: "{{ gpu_validation_nvswitch_validation_mode2_fm_services }}" + register: _fm_service_status + failed_when: false + +- name: Set FM service status facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fm_service_results: "{{ _fm_service_status.results + | map(attribute='item') | zip( + _fm_service_status.results + | map(attribute='status') | map(attribute='ActiveState')) + | list }}" + +- name: Show FM service states + ansible.builtin.debug: + msg: "{{ item[0] }}: {{ item[1] }}" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Collect Fabric State for all GPUs + become: true + ansible.builtin.shell: | + set -o pipefail + nvidia-smi -q | grep -A2 'Fabric' | grep 'State' | awk '{print $NF}' + args: + executable: /bin/bash + register: _fabric_states + changed_when: false + failed_when: false + +- name: Show fabric states + ansible.builtin.debug: + msg: "GPU Fabric States: {{ _fabric_states.stdout_lines }}" + +- name: Set fabric state facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_fabric_states: "{{ _fabric_states.stdout_lines | map('trim') | list }}" + gpu_validation_nvswitch_fabric_completed_count: >- + {{ _fabric_states.stdout_lines + | map('trim') + | select('equalto', gpu_validation_nvswitch_validation_fabric_state_expected) + | list | length }} + +- name: Collect mdev instance list + become: true + ansible.builtin.command: + cmd: mdevctl list + register: _mdevctl_list + changed_when: false + failed_when: false + +- name: Show mdev instance list + ansible.builtin.debug: + msg: "{{ _mdevctl_list.stdout | default('mdevctl list returned no output') }}" + +- name: Set mdev instance count fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_mdev_instance_count: "{{ _mdevctl_list.stdout_lines | default([]) | length }}" + gpu_validation_nvswitch_mdevctl_output: "{{ _mdevctl_list.stdout | default('') }}" + +- name: Collect NVLink topology matrix + become: true + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _nvlink_topo + changed_when: false + failed_when: false + +- name: Show NVLink topology matrix + ansible.builtin.debug: + msg: "{{ _nvlink_topo.stdout | default('nvidia-smi topo not available') }}" + +- name: Set topology fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_topo_output: "{{ _nvlink_topo.stdout | default('') }}" diff --git a/gpu-validation/tasks/nvswitch_mode2_assertions.yaml b/gpu-validation/tasks/nvswitch_mode2_assertions.yaml new file mode 100644 index 0000000..6fd01ca --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode2_assertions.yaml @@ -0,0 +1,40 @@ +--- +# Mode 2 (FABRIC_MODE=2) host-side assertions. +# Verifies FM + nvidia-vgpu-mgr are active, all GPUs completed, and mdev instances exist. + +- name: Assert FM services are active (mode 2) + ansible.builtin.assert: + that: + - item[1] == 'active' + fail_msg: "Service {{ item[0] }} is not active (state: {{ item[1] }})" + success_msg: "Service {{ item[0] }} is active" + loop: "{{ gpu_validation_nvswitch_fm_service_results }}" + +- name: Assert all GPUs reached Fabric State Completed + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_fabric_states | length == gpu_validation_nvswitch_validation_expected_gpu_count | int + - gpu_validation_nvswitch_fabric_completed_count | int == gpu_validation_nvswitch_validation_expected_gpu_count | int + fail_msg: >- + Discovered {{ gpu_validation_nvswitch_fabric_states | length }} GPU(s), expected {{ gpu_validation_nvswitch_validation_expected_gpu_count }}. + Only {{ gpu_validation_nvswitch_fabric_completed_count }} of + {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }}. + States seen: {{ gpu_validation_nvswitch_fabric_states }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached + Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }} + when: gpu_validation_nvswitch_validation_fabric_state_check | bool + +- name: Assert expected number of mdev instances exist + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_mdev_instance_count | int == gpu_validation_nvswitch_validation_mode2_expected_mdev_count | int + fail_msg: >- + mdevctl list shows {{ gpu_validation_nvswitch_mdev_instance_count }} mdev instance(s), + expected {{ gpu_validation_nvswitch_validation_mode2_expected_mdev_count }}. + Full mdevctl output: + {{ gpu_validation_nvswitch_mdevctl_output }} + success_msg: >- + All {{ gpu_validation_nvswitch_validation_mode2_expected_mdev_count }} mdev instance(s) found + when: gpu_validation_nvswitch_validation_mode2_expected_mdev_count | int > 0 diff --git a/gpu-validation/tasks/nvswitch_mode2_isolation.yaml b/gpu-validation/tasks/nvswitch_mode2_isolation.yaml new file mode 100644 index 0000000..2dbebe9 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode2_isolation.yaml @@ -0,0 +1,45 @@ +--- +# Mode 2 (FABRIC_MODE=2) in-VM isolation checks. +# Runs on the [gpu_vms] group (VMs booted with the mdev flavor). +# From inside the VM the mdev device presents as the physical GPU model; +# nvidia-smi output is equivalent to Mode 1 passthrough from the guest's perspective. + +- name: Count GPUs visible inside VM (mode 2) + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=name --format=csv,noheader + register: _vm_gpus + changed_when: false + failed_when: false + +- name: Show GPU list inside VM + ansible.builtin.debug: + msg: "GPUs in VM {{ inventory_hostname }}: {{ _vm_gpus.stdout_lines }}" + +- name: Set GPU count fact + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_gpu_count: "{{ _vm_gpus.stdout_lines | length }}" + +- name: Collect in-VM NVLink topology matrix + ansible.builtin.command: + cmd: nvidia-smi topo -m + register: _vm_topo + changed_when: false + failed_when: false + +- name: Show in-VM NVLink topology matrix + ansible.builtin.debug: + msg: "{{ _vm_topo.stdout | default('nvidia-smi topo not available') }}" + +- name: Set in-VM topology facts + ansible.builtin.set_fact: + gpu_validation_nvswitch_vm_topo_output: "{{ _vm_topo.stdout | default('') }}" + gpu_validation_nvswitch_vm_nvlink_lines: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('search', '\\bNV[0-9]') + | list }} + gpu_validation_nvswitch_vm_topo_gpu_count: >- + {{ (_vm_topo.stdout | default('')) + .splitlines() + | select('match', '^GPU\\d') + | list | length | int }} diff --git a/gpu-validation/tasks/nvswitch_mode2_isolation_assertions.yaml b/gpu-validation/tasks/nvswitch_mode2_isolation_assertions.yaml new file mode 100644 index 0000000..d3c4857 --- /dev/null +++ b/gpu-validation/tasks/nvswitch_mode2_isolation_assertions.yaml @@ -0,0 +1,43 @@ +--- +# Mode 2 (FABRIC_MODE=2) in-VM isolation assertions. +# nvidia-vgpu-mgr programs NVSwitch partition routing per mdev — hardware-enforced isolation. + +- name: Assert GPU count matches mdev flavor slot size + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s), + expected {{ gpu_validation_nvswitch_validation_gpus_per_vm }} per the mdev flavor. + GPUs seen: {{ _vm_gpus.stdout_lines }} + success_msg: >- + VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s) as expected + +- name: Assert NVLink entries are present within the mdev partition + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_nvlink_lines | length > 0 + fail_msg: >- + No NVLink entries found in nvidia-smi topo -m for VM {{ inventory_hostname }}. + GPUs within the same mdev partition should show NVLink connectivity. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + NVLink connectivity confirmed within mdev partition for VM {{ inventory_hostname }} + when: + - gpu_validation_nvswitch_validation_gpus_per_vm | int > 1 + +- name: Assert topo GPU row count matches mdev slot (no extra GPUs visible from topo matrix) + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_vm_topo_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int + fail_msg: >- + nvidia-smi topo -m shows {{ gpu_validation_nvswitch_vm_topo_gpu_count }} GPU row(s) in + VM {{ inventory_hostname }} but mdev flavor specifies + {{ gpu_validation_nvswitch_validation_gpus_per_vm }}. + A mismatch may indicate cross-partition GPU visibility. + Topology output: + {{ gpu_validation_nvswitch_vm_topo_output }} + success_msg: >- + nvidia-smi topo -m GPU row count ({{ gpu_validation_nvswitch_vm_topo_gpu_count }}) matches mdev flavor + — no cross-partition GPU leakage detected in VM {{ inventory_hostname }} diff --git a/nvswitch_validation.yaml b/nvswitch_validation.yaml new file mode 100644 index 0000000..40794f0 --- /dev/null +++ b/nvswitch_validation.yaml @@ -0,0 +1,112 @@ +--- +# NVSwitch validation playbook — dispatches to mode-specific task files. +# +# REQUIRED: gpu_validation_nvswitch_validation_fabric_mode must be supplied by the caller. +# It cannot come from role defaults because tasks_from is rendered in the play +# context before role defaults are loaded. Set it via: +# - extra-vars: -e gpu_validation_nvswitch_validation_fabric_mode=0 ← preferred (single source) +# - group_vars/all ← also single source +# - AnsibleTest CR ansibleVarFiles +# +# WARNING: do NOT set it in group_vars/edpm_hosts or group_vars/gpu_vms separately. +# The two plays resolve the variable independently from their own host context; a +# per-group assignment silently dispatches each play to a different mode task file. +# +# Valid values: +# 0 — PCI Grouping (FABRIC_MODE=0, open NVSwitch fabric, trusted tenants only) +# 1 — Shared NVSwitch (FABRIC_MODE=1, fmpm partitions, hardware isolation) +# 2 — vGPU Multitenancy (FABRIC_MODE=2, mdev, hardware isolation) +# +# Host-side checks (runs on edpm_hosts): +# Mode 0: FM + NVLSM active, all GPUs Fabric State Completed, open-fabric topo display +# Mode 1: FM + NVLSM + fm-partition-restore active, fmpm -l partitions activated +# Mode 2: FM + NVLSM + nvidia-vgpu-mgr active, mdevctl list count matches expected +# +# In-VM isolation checks (runs on gpu_vms): +# Mode 0: GPU count matches PCI group size, NVLink within group (no switch-level isolation) +# Mode 1: GPU count matches partition size, NVLink within partition (hardware isolation) +# Mode 2: GPU count matches mdev slot size, NVLink within mdev partition (hardware isolation) +# +# Inventory groups required: +# [edpm_hosts] — the EDPM compute node(s) +# [gpu_vms] — test VMs booted with the GPU flavor (two minimum) +# +# See gpu-validation/defaults/main.yaml for all other gpu_validation_nvswitch_validation_* defaults. + +- name: Pre-flight — validate inventory groups + hosts: localhost + gather_subset: + - min + tasks: + - name: Assert edpm_hosts is non-empty + ansible.builtin.assert: + that: groups['edpm_hosts'] | default([]) | length >= 1 + fail_msg: >- + Inventory group [edpm_hosts] is empty or undefined. + Add at least one EDPM compute node to this group. + + - name: Assert gpu_vms has at least two hosts + ansible.builtin.assert: + that: groups['gpu_vms'] | default([]) | length >= 2 + fail_msg: >- + Inventory group [gpu_vms] has {{ groups['gpu_vms'] | default([]) | length }} host(s); + at least two are required to validate cross-VM NVSwitch isolation. + +- name: Host-side checks + hosts: edpm_hosts + gather_subset: + - min + pre_tasks: + - name: Assert gpu_validation_nvswitch_validation_fabric_mode is provided by caller + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_validation_fabric_mode is defined + - (gpu_validation_nvswitch_validation_fabric_mode | string) in ['0', '1', '2'] + fail_msg: >- + gpu_validation_nvswitch_validation_fabric_mode must be set to 0, 1, or 2 via + extra-vars (-e gpu_validation_nvswitch_validation_fabric_mode=N), inventory/group_vars, + or an AnsibleTest CR ansibleVarFiles entry. + It cannot come from role defaults because tasks_from is resolved in the + play context before role defaults are loaded. + tasks: + - name: "Collect host-side data (mode {{ gpu_validation_nvswitch_validation_fabric_mode }})" + ansible.builtin.include_role: + name: gpu-validation + tasks_from: "nvswitch_mode{{ gpu_validation_nvswitch_validation_fabric_mode }}.yaml" + + - name: "Assert host-side state (mode {{ gpu_validation_nvswitch_validation_fabric_mode }})" + ansible.builtin.include_role: + name: gpu-validation + tasks_from: "nvswitch_mode{{ gpu_validation_nvswitch_validation_fabric_mode }}_assertions.yaml" + +- name: In-VM isolation checks + hosts: gpu_vms + gather_subset: + - min + pre_tasks: + - name: Assert fabric mode is valid and consistent with edpm_hosts + ansible.builtin.assert: + that: + - gpu_validation_nvswitch_validation_fabric_mode is defined + - (gpu_validation_nvswitch_validation_fabric_mode | string) in ['0', '1', '2'] + - groups['edpm_hosts'] | map('extract', hostvars, 'gpu_validation_nvswitch_validation_fabric_mode') + | list | unique | length == 1 + - (groups['edpm_hosts'] | map('extract', hostvars, 'gpu_validation_nvswitch_validation_fabric_mode') + | list | first | string) == (gpu_validation_nvswitch_validation_fabric_mode | string) + fail_msg: >- + gpu_validation_nvswitch_validation_fabric_mode is not consistent across inventory groups. + gpu_vms resolved {{ gpu_validation_nvswitch_validation_fabric_mode }}; + edpm_hosts resolved + {{ groups['edpm_hosts'] | map('extract', hostvars, 'gpu_validation_nvswitch_validation_fabric_mode') | list }}. + Set the variable via extra-vars or group_vars/all (not group-specific vars) + so both plays dispatch to the same mode task file. + tasks: + - name: "Collect in-VM data (mode {{ gpu_validation_nvswitch_validation_fabric_mode }})" + ansible.builtin.include_role: + name: gpu-validation + tasks_from: "nvswitch_mode{{ gpu_validation_nvswitch_validation_fabric_mode }}_isolation.yaml" + + - name: "Assert in-VM isolation (mode {{ gpu_validation_nvswitch_validation_fabric_mode }})" + ansible.builtin.include_role: + name: gpu-validation + tasks_from: "nvswitch_mode{{ gpu_validation_nvswitch_validation_fabric_mode }}_isolation_assertions.yaml"