Add nwswitch/link validations for host and VM side

gpu-validation/defaults/main.yaml

@@ -121,3 +121,52 @@ gpu_validation_reboot_force: false
 gpu_validation_reboot_timeout: 3600
 # [int] Post-reboot delay in seconds
 gpu_validation_reboot_post_delay: 60
+
+# ── NVSwitch validation defaults (nvswitch_validation.yaml playbook) ──────────
+# gpu_validation_nvswitch_validation_fabric_mode is intentionally NOT defaulted here.
+# tasks_from in include_role is rendered in the play context before role defaults
+# are loaded, so a role default cannot serve as a reliable selector.
+# Callers MUST supply gpu_validation_nvswitch_validation_fabric_mode via extra-vars (-e),
+# inventory/group_vars, or an AnsibleTest CR ansibleVarFiles entry.
+# Valid values: 0 (PCI grouping), 1 (Shared NVSwitch), 2 (vGPU mdev).
+
+# [list] Systemd services expected active per mode.
+# Override if your installation uses different service names.
+gpu_validation_nvswitch_validation_mode0_fm_services:
+  - nvidia-fabricmanager
+  - nvlsm
+gpu_validation_nvswitch_validation_mode1_fm_services:
+  - nvidia-fabricmanager
+  - nvlsm
+  - fm-partition-restore
+gpu_validation_nvswitch_validation_mode2_fm_services:
+  - nvidia-fabricmanager
+  - nvlsm
+  - nvidia-vgpu-mgr
+
+# [int] Mode 2 only: expected number of mdev instances (from mdevctl list).
+# Set to 0 to skip the assertion. Typically equals gpu_validation_nvswitch_validation_expected_gpu_count.
+gpu_validation_nvswitch_validation_mode2_expected_mdev_count: 0
+
+# [string] GPU PCI vendor ID
+gpu_validation_nvswitch_validation_gpu_vendor_id: "10de"
+# [string] GPU PCI product ID — HGX B300 GB110 SXM6 AC
+gpu_validation_nvswitch_validation_gpu_product_id: "3182"
+# [int] Total number of GPUs expected on the compute host
+gpu_validation_nvswitch_validation_expected_gpu_count: 8
+# [bool] Check that all GPUs reach the expected Fabric State
+gpu_validation_nvswitch_validation_fabric_state_check: true
+# [string] Expected nvidia-smi Fabric State value
+gpu_validation_nvswitch_validation_fabric_state_expected: "Completed"
+# [int] Expected number of activated NVSwitch partitions; set after first fmpm -l run
+gpu_validation_nvswitch_validation_expected_partition_count: 0
+# [int] Number of GPUs per passthrough VM flavor
+gpu_validation_nvswitch_validation_gpus_per_vm: 1
+# [string] Nova resource class for the GPU passthrough flavor
+gpu_validation_nvswitch_validation_passthrough_resource_class: "CUSTOM_HGX-B300-GPU"
+# [string] Expected libvirt device type for passthrough GPUs in virsh dumpxml
+gpu_validation_nvswitch_validation_expected_device_type: "hostdev"
+# [bool] Check that FABRIC_MODE in fabricmanager.cfg matches gpu_validation_nvswitch_validation_fabric_mode
+gpu_validation_nvswitch_validation_check_fabric_mode_cfg: false
+# [string] Path to the FM configuration file
+gpu_validation_nvswitch_validation_fabric_mode_cfg_file: /usr/share/nvidia/nvswitch/fabricmanager.cfg

gpu-validation/tasks/nvswitch.yaml

@@ -0,0 +1,83 @@
+---
+# host-side checks: FM services, fabric state, partition activation.
+
+- name: Collect FM service status
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+  loop: "{{ gpu_validation_nvswitch_validation_fm_services }}"
+  register: _fm_service_status
+  failed_when: false
+
+- name: Set FM service status facts
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_fm_service_results: "{{ _fm_service_status.results
+      | map(attribute='item') | zip(_fm_service_status.results
+         | map(attribute='status') | map(attribute='ActiveState'))
+      | list }}"
+
+- name: Show FM service states
+  ansible.builtin.debug:
+    msg: "{{ item[0] }}: {{ item[1] }}"
+  loop: "{{ gpu_validation_nvswitch_fm_service_results }}"
+
+- name: Collect Fabric State for all GPUs
+  become: true
+  ansible.builtin.shell: |
+    set -o pipefail
+    nvidia-smi -q | grep -A2 'Fabric' | grep 'State' | awk '{print $NF}'
+  args:
+    executable: /bin/bash
+  register: _fabric_states
+  changed_when: false
+  failed_when: false
+
+- name: Show fabric states
+  ansible.builtin.debug:
+    msg: "GPU Fabric States: {{ _fabric_states.stdout_lines }}"
+
+- name: Set fabric state facts
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_fabric_states: "{{ _fabric_states.stdout_lines | map('trim') | list }}"
+    gpu_validation_nvswitch_fabric_completed_count: >-
+      {{ _fabric_states.stdout_lines
+         | map('trim')
+         | select('equalto', gpu_validation_nvswitch_validation_fabric_state_expected)
+         | list | length }}
+
+- name: Collect fmpm partition list
+  become: true
+  ansible.builtin.command:
+    cmd: fmpm -l
+  register: _fmpm_list
+  changed_when: false
+  failed_when: false
+
+- name: Show fmpm partition list
+  ansible.builtin.debug:
+    msg: "{{ _fmpm_list.stdout | default('fmpm not available or no output') }}"
+
+- name: Set partition facts
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_fmpm_output: "{{ _fmpm_list.stdout | default('') }}"
+    gpu_validation_nvswitch_activated_partition_count: >-
+      {{ (_fmpm_list.stdout | default(''))
+         .splitlines()
+         | select('search', '\\bactivated\\b')
+         | list | length }}
+
+- name: Collect NVLink topology matrix
+  become: true
+  ansible.builtin.command:
+    cmd: nvidia-smi topo -m
+  register: _nvlink_topo
+  changed_when: false
+  failed_when: false
+
+- name: Show NVLink topology matrix
+  ansible.builtin.debug:
+    msg: "{{ _nvlink_topo.stdout | default('nvidia-smi topo not available') }}"
+
+- name: Set topology fact
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_topo_output: "{{ _nvlink_topo.stdout | default('') }}"

gpu-validation/tasks/nvswitch_assertions.yaml

@@ -0,0 +1,41 @@
+---
+# assertions: FM services active, fabric state, partition count.
+
+- name: Assert FM services are active
+  ansible.builtin.assert:
+    that:
+      - item[1] == 'active'
+    fail_msg: "Service {{ item[0] }} is not active (state: {{ item[1] }})"
+    success_msg: "Service {{ item[0] }} is active"
+  loop: "{{ gpu_validation_nvswitch_fm_service_results }}"
+
+- name: Assert all GPUs reached Fabric State Completed
+  ansible.builtin.assert:
+    that:
+      - gpu_validation_nvswitch_fabric_states | length == gpu_validation_nvswitch_validation_expected_gpu_count | int
+      - gpu_validation_nvswitch_fabric_completed_count | int == gpu_validation_nvswitch_validation_expected_gpu_count | int
+    fail_msg: >-
+      Discovered {{ gpu_validation_nvswitch_fabric_states | length }} GPU(s), expected {{ gpu_validation_nvswitch_validation_expected_gpu_count }}.
+      Only {{ gpu_validation_nvswitch_fabric_completed_count }} of
+      {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached
+      Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }}.
+      States seen: {{ gpu_validation_nvswitch_fabric_states }}
+    success_msg: >-
+      All {{ gpu_validation_nvswitch_validation_expected_gpu_count }} GPUs reached
+      Fabric State: {{ gpu_validation_nvswitch_validation_fabric_state_expected }}
+  when: gpu_validation_nvswitch_validation_fabric_state_check | bool
+
+- name: Assert expected number of activated partitions
+  ansible.builtin.assert:
+    that:
+      - gpu_validation_nvswitch_activated_partition_count | int == gpu_validation_nvswitch_validation_expected_partition_count | int
+    fail_msg: >-
+      fmpm -l shows {{ gpu_validation_nvswitch_activated_partition_count }} activated partitions,
+      expected {{ gpu_validation_nvswitch_validation_expected_partition_count }}.
+      Full fmpm output:
+      {{ gpu_validation_nvswitch_fmpm_output }}
+    success_msg: >-
+      All {{ gpu_validation_nvswitch_validation_expected_partition_count }} NVSwitch partitions
+      are activated
+  when:
+    - gpu_validation_nvswitch_validation_expected_partition_count | int > 0

gpu-validation/tasks/nvswitch_isolation.yaml

@@ -0,0 +1,43 @@
+---
+# In-VM isolation checks: GPU count, NVLink topology per VM.
+# Runs on the [gpu_vms] group (the passthrough-flavor VMs).
+
+- name: Count GPUs visible inside VM
+  ansible.builtin.command:
+    cmd: nvidia-smi --query-gpu=name --format=csv,noheader
+  register: _vm_gpus
+  changed_when: false
+  failed_when: false
+
+- name: Show GPU list inside VM
+  ansible.builtin.debug:
+    msg: "GPUs in VM {{ inventory_hostname }}: {{ _vm_gpus.stdout_lines }}"
+
+- name: Set GPU count fact
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_vm_gpu_count: "{{ _vm_gpus.stdout_lines | length }}"
+
+- name: Collect in-VM NVLink topology matrix
+  ansible.builtin.command:
+    cmd: nvidia-smi topo -m
+  register: _vm_topo
+  changed_when: false
+  failed_when: false
+
+- name: Show in-VM NVLink topology matrix
+  ansible.builtin.debug:
+    msg: "{{ _vm_topo.stdout | default('nvidia-smi topo not available') }}"
+
+- name: Set in-VM topology facts
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_vm_topo_output: "{{ _vm_topo.stdout | default('') }}"
+    gpu_validation_nvswitch_vm_nvlink_lines: >-
+      {{ (_vm_topo.stdout | default(''))
+         .splitlines()
+         | select('search', '\\bNV[0-9]')
+         | list }}
+    gpu_validation_nvswitch_vm_topo_gpu_count: >-
+      {{ (_vm_topo.stdout | default(''))
+         .splitlines()
+         | select('match', '^GPU\\d')
+         | list | length | int }}

gpu-validation/tasks/nvswitch_isolation_assertions.yaml

@@ -0,0 +1,44 @@
+---
+# assertions: GPU count per VM matches flavor, no cross-partition NVLink.
+
+- name: Assert GPU count matches passthrough flavor
+  ansible.builtin.assert:
+    that:
+      - gpu_validation_nvswitch_vm_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int
+    fail_msg: >-
+      VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s),
+      expected {{ gpu_validation_nvswitch_validation_gpus_per_vm }} per the passthrough flavor.
+      GPUs seen: {{ _vm_gpus.stdout_lines }}
+    success_msg: >-
+      VM {{ inventory_hostname }} has {{ gpu_validation_nvswitch_vm_gpu_count }} GPU(s) as expected
+
+- name: Assert NVLink entries are present (GPUs within partition are connected)
+  ansible.builtin.assert:
+    that:
+      - gpu_validation_nvswitch_vm_nvlink_lines | length > 0
+    fail_msg: >-
+      No NVLink entries found in nvidia-smi topo -m output for
+      VM {{ inventory_hostname }}.  GPUs within the same partition
+      should show NVLink connectivity.
+      Topology output:
+      {{ gpu_validation_nvswitch_vm_topo_output }}
+    success_msg: >-
+      NVLink connectivity confirmed within partition for
+      VM {{ inventory_hostname }}
+  when:
+    - gpu_validation_nvswitch_validation_gpus_per_vm | int > 1
+
+- name: Assert topo GPU row count matches flavor (no extra GPUs visible from topo matrix)
+  ansible.builtin.assert:
+    that:
+      - gpu_validation_nvswitch_vm_topo_gpu_count | int == gpu_validation_nvswitch_validation_gpus_per_vm | int
+    fail_msg: >-
+      nvidia-smi topo -m shows {{ gpu_validation_nvswitch_vm_topo_gpu_count }} GPU row(s) in
+      VM {{ inventory_hostname }} but flavor specifies
+      {{ gpu_validation_nvswitch_validation_gpus_per_vm }}.
+      A mismatch may indicate cross-partition GPU visibility.
+      Topology output:
+      {{ gpu_validation_nvswitch_vm_topo_output }}
+    success_msg: >-
+      nvidia-smi topo -m GPU row count ({{ gpu_validation_nvswitch_vm_topo_gpu_count }}) matches flavor
+      — no cross-partition GPU leakage detected in VM {{ inventory_hostname }}

gpu-validation/tasks/nvswitch_mode0.yaml

@@ -0,0 +1,89 @@
+---
+# Mode 0 (FABRIC_MODE=0) host-side checks.
+# NVSwitch fabric is open/all-to-all — fmpm is NOT called.
+# FM + NVLSM are required to initialize the fabric; no partition tables are programmed.
+# Isolation is software-only (CUDA + IOMMU). Suitable for trusted tenants only.
+
+- name: Collect FABRIC_MODE from fabricmanager.cfg
+  become: true
+  ansible.builtin.command:
+    cmd: "grep -E '^FABRIC_MODE' {{ gpu_validation_nvswitch_validation_fabric_mode_cfg_file }}"
+  register: _fabric_mode_cfg
+  changed_when: false
+  failed_when: false
+  when: gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool
+
+- name: Show FABRIC_MODE config value
+  ansible.builtin.debug:
+    msg: "fabricmanager.cfg: {{ _fabric_mode_cfg.stdout | default('not found') }}"
+  when: gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool
+
+- name: Set FABRIC_MODE config fact
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_fabric_mode_cfg_value: >-
+      {{ (_fabric_mode_cfg.stdout | default('') | regex_search('FABRIC_MODE\s*=\s*(\d+)', '\\1') | default([]) | first | default('-1')) | int }}
+  when: gpu_validation_nvswitch_validation_check_fabric_mode_cfg | bool
+
+- name: Collect FM service status (mode 0)
+  become: true
+  ansible.builtin.systemd:
+    name: "{{ item }}"
+  loop: "{{ gpu_validation_nvswitch_validation_mode0_fm_services }}"
+  register: _fm_service_status
+  failed_when: false
+
+- name: Set FM service status facts
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_fm_service_results: "{{ _fm_service_status.results
+      | map(attribute='item') | zip(
+        _fm_service_status.results
+        | map(attribute='status') | map(attribute='ActiveState'))
+      | list }}"
+
+- name: Show FM service states
+  ansible.builtin.debug:
+    msg: "{{ item[0] }}: {{ item[1] }}"
+  loop: "{{ gpu_validation_nvswitch_fm_service_results }}"
+
+- name: Collect Fabric State for all GPUs
+  become: true
+  ansible.builtin.shell: |
+    set -o pipefail
+    nvidia-smi -q | grep -A2 'Fabric' | grep 'State' | awk '{print $NF}'
+  args:
+    executable: /bin/bash
+  register: _fabric_states
+  changed_when: false
+  failed_when: false
+
+- name: Show fabric states
+  ansible.builtin.debug:
+    msg: "GPU Fabric States: {{ _fabric_states.stdout_lines }}"
+
+- name: Set fabric state facts
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_fabric_states: "{{ _fabric_states.stdout_lines | map('trim') | list }}"
+    gpu_validation_nvswitch_fabric_completed_count: >-
+      {{ _fabric_states.stdout_lines
+         | map('trim')
+         | select('equalto', gpu_validation_nvswitch_validation_fabric_state_expected)
+         | list | length }}
+
+- name: Collect NVLink topology matrix (open fabric — informational only)
+  become: true
+  ansible.builtin.command:
+    cmd: nvidia-smi topo -m
+  register: _nvlink_topo
+  changed_when: false
+  failed_when: false
+
+- name: Show NVLink topology matrix
+  ansible.builtin.debug:
+    msg: |
+      NOTE: Mode 0 NVSwitch fabric is open/all-to-all. All GPUs will show NVLink
+      connectivity to each other. Cross-VM NVLink is NOT blocked at the switch level.
+      {{ _nvlink_topo.stdout | default('nvidia-smi topo not available') }}
+
+- name: Set topology fact
+  ansible.builtin.set_fact:
+    gpu_validation_nvswitch_topo_output: "{{ _nvlink_topo.stdout | default('') }}"