Harden verification OpenStack CLI calls to use JSON output

collection/stages/roles/verification/tasks/check_cinder_csi.yml

@@ -43,13 +43,20 @@
 - name: Get volume az for workers (if any)
   ansible.builtin.shell: |
     set -o pipefail
-    openstack volume show $(openstack volume list -c ID -c Name -f value | grep "{{ item }}" | cut -d' ' -f2) -c availability_zone -f value
+    vol_id=$(openstack volume list -f json | jq -r --arg w "{{ item }}" '
+      [.[] | select((.Name // .name // "") | contains($w)) | (.ID // .id)] | first // empty
+    ')
+    if [ -z "${vol_id}" ]; then
+      exit 0
+    fi
+    openstack volume show "${vol_id}" -f json | jq -r '.availability_zone // empty'
   environment:
     OS_CLOUD: "{{ user_cloud }}"
   loop: "{{ worker_names }}"
   register: temporary_output
   failed_when: false
   changed_when: false
+  when: edge_nova_az is not defined
 
 - name: Convert output on a list
   ansible.builtin.set_fact:
@@ -97,8 +104,7 @@
     existing_pvc_ids: "{{ existing_pvcs | json_query('resources[].spec.volumeName') | list }}"
 
 - name: Get cinder volumes for PVCs
-  ansible.builtin.shell: |
-    openstack volume show "{{ item }}" -c availability_zone -f value
+  ansible.builtin.command: openstack volume show "{{ item }}" -f json
   environment:
     OS_CLOUD: "{{ user_cloud }}"
   loop: "{{ existing_pvc_ids }}"
@@ -107,7 +113,7 @@
 
 - name: Convert output on a list
   ansible.builtin.set_fact:
-    existing_pvc_azs: "{{ existing_pvc_azs + [item.stdout] }}"
+    existing_pvc_azs: "{{ existing_pvc_azs + [(item.stdout | from_json).availability_zone] }}"
   loop: "{{ temporary_output.results | list }}"
 
 - name: Check - expected cinder AZs on PVCs

collection/stages/roles/verification/tasks/check_lb_svc.yml

@@ -52,36 +52,45 @@
         - not internal_lb
         - lb_ingress_ip is match(ipv4_regex)
       block:
-        - name: Get the LB internal IP on openstack when the lb is using a FIP
-          ansible.builtin.shell: |
-            openstack floating ip show {{ lb_ingress_ip }} -c fixed_ip_address -f value
+        # NOTE: `openstack floating ip show … -f <any>` is broken on the
+        # shiftstackclient OSC ("Invalid formatter provided"). Use list+filter.
+        - name: Get the LB FIP details on openstack when the lb is using a FIP
+          ansible.builtin.command: openstack floating ip list --floating-ip-address {{ lb_ingress_ip }} -f json
           environment:
             OS_CLOUD: "{{ user_cloud }}"
-          register: lb_internal_ip_output
+          register: lb_fip_list
           changed_when: false
 
+        - name: Check the FIP "{{ lb_ingress_ip }}" exists on OpenStack
+          ansible.builtin.assert:
+            that:
+              - (lb_fip_list.stdout | from_json | length) > 0
+            fail_msg: Could not find floating IP "{{ lb_ingress_ip }}" in OSP
+
         - name: Define the internal IP
           ansible.builtin.set_fact:
-            lb_internal_ip: "{{ internal_lb | ternary(lb_ingress_ip, lb_internal_ip_output.stdout) }}"
+            lb_internal_ip: "{{ internal_lb | ternary(lb_ingress_ip, (lb_fip_list.stdout | from_json | first)['Fixed IP Address']) }}"
 
-        - name: Get the Octavia provider type for the LB
-          ansible.builtin.shell: |
-            set -o pipefail
-            openstack loadbalancer list -c vip_address -c provider -f value | grep {{ lb_internal_ip }} | awk '{print $2}'
+        - name: List load balancers on OpenStack
+          ansible.builtin.command: openstack loadbalancer list -f json
           environment:
             OS_CLOUD: "{{ user_cloud }}"
-          register: svc_lb_type
+          register: svc_lb_list
           changed_when: false
 
+        - name: Select the LB matching VIP "{{ lb_internal_ip }}"
+          ansible.builtin.set_fact:
+            svc_lb_matches: "{{ svc_lb_list.stdout | from_json | selectattr('vip_address', 'defined') | selectattr('vip_address', 'equalto', lb_internal_ip) | list }}"
+
         - name: Check the LB exists with the internal IP (VIP address) "{{ lb_internal_ip }}"
           ansible.builtin.assert:
-            that: svc_lb_type.stdout | length > 0
+            that: svc_lb_matches | length > 0
             fail_msg: Could not find a LB in OSP with the vip_address "{{ lb_internal_ip }}"
             success_msg: A LB with the vip_address "{{ lb_internal_ip }}" has been found in OSP
 
         - name: Detecte LB Octavia provider
           ansible.builtin.debug:
-            msg: "Detected LB Octavia provider: {{ svc_lb_type.stdout }}"
+            msg: "Detected LB Octavia provider: {{ svc_lb_matches[0].provider }}"
 
     - name: Get pod names
       kubernetes.core.k8s_info:
@@ -142,27 +151,42 @@
       delay: 15
       until: oc_project.resources|length == 0
 
-    - name: Check the LB with "{{ lb_ingress_ip }}" ingress IP has been removed from OSP
-      ansible.builtin.shell: |
-        set -o pipefail
-        openstack loadbalancer list -f value | grep {{ lb_ingress_ip }}
+    - name: List load balancers after project deletion
+      ansible.builtin.command: openstack loadbalancer list -f json
       environment:
         OS_CLOUD: "{{ user_cloud }}"
-      register: svc_lb
-      failed_when: svc_lb.rc == 0
+      register: svc_lb_after
       changed_when: false
       when:
         - lb_internal_ip is defined
         - lb_internal_ip | length > 0
 
-    - name: Check the fip "{{ lb_ingress_ip }}" has been removed from OSP
-      ansible.builtin.shell: |
-        openstack floating ip show {{ lb_ingress_ip }} -f value
+    - name: Check the LB with "{{ lb_ingress_ip }}" ingress IP has been removed from OSP
+      ansible.builtin.assert:
+        that:
+          - (svc_lb_after.stdout | from_json | selectattr('vip_address', 'defined') | selectattr('vip_address', 'equalto', lb_ingress_ip) | list | length) == 0
+          - (svc_lb_after.stdout | from_json | selectattr('vip_address', 'defined') | selectattr('vip_address', 'equalto', lb_internal_ip) | list | length) == 0
+        fail_msg: Found a LB in OSP still associated with vip_address "{{ lb_ingress_ip }}" or "{{ lb_internal_ip }}"
+      when:
+        - lb_internal_ip is defined
+        - lb_internal_ip | length > 0
+
+    # Prefer list over show: floating ip show -f * fails on shiftstackclient OSC.
+    - name: Check whether the fip "{{ lb_ingress_ip }}" still exists on OSP
+      ansible.builtin.command: openstack floating ip list --floating-ip-address {{ lb_ingress_ip }} -f json
       environment:
         OS_CLOUD: "{{ user_cloud }}"
-      register: fip
-      failed_when: fip.rc == 0
+      register: fip_list
       changed_when: false
       when:
         - lb_internal_ip is defined
         - lb_internal_ip | length > 0
+
+    - name: Check the fip "{{ lb_ingress_ip }}" has been removed from OSP
+      ansible.builtin.assert:
+        that:
+          - (fip_list.stdout | from_json | length) == 0
+        fail_msg: FIP "{{ lb_ingress_ip }}" is still present in OSP
+      when:
+        - lb_internal_ip is defined
+        - lb_internal_ip | length > 0

collection/stages/roles/verification/tasks/check_prometheus_pvcs.yml

@@ -32,9 +32,7 @@
     monitoring_pvc_ids: "{{ monitoring_pvcs | json_query('resources[].spec.volumeName') | list }}"
 
 - name: Check cinder volume for PVC exist
-  ansible.builtin.shell: |
-    set -o pipefail
-    openstack volume list -f csv | grep  {{ item }}
+  ansible.builtin.command: openstack volume show "{{ item }}" -f json
   environment:
     OS_CLOUD: "{{ user_cloud }}"
   changed_when: false

collection/stages/roles/verification/tasks/check_registry.yml

@@ -111,7 +111,14 @@
     - name: Get the images objects in swift for the {{ swift_check_ns }} namespace from the containers
       ansible.builtin.shell: |
         set -o pipefail
-        openstack container list -c Name -f value | xargs -I% openstack object list % -c Name -f value | grep {{ swift_check_ns }}.{{ swift_check_image }}
+        target="{{ swift_check_ns }}.{{ swift_check_image }}"
+        openstack container list -f json \
+          | jq -r '.[] | (.Name // .name // empty)' \
+          | while read -r container; do
+              [ -z "${container}" ] && continue
+              openstack object list "${container}" -f json \
+                | jq -r --arg t "${target}" '.[] | (.Name // .name // empty) | select(contains($t))'
+            done
       environment:
         OS_CLOUD: "{{ user_cloud }}"
       register: image_registry_objects
@@ -123,7 +130,14 @@
     - name: List objects in case of failure from the containers
       ansible.builtin.shell: |
         set -o pipefail
-        openstack container list -c Name -f value | xargs -I% openstack object list % -c Name -f value
+        openstack container list -f json \
+          | jq -r '.[] | (.Name // .name // empty)' \
+          | while read -r container; do
+              [ -z "${container}" ] && continue
+              echo "=== container: ${container} ==="
+              openstack object list "${container}" -f json \
+                | jq -r '.[] | (.Name // .name // empty)'
+            done
       environment:
         OS_CLOUD: "{{ user_cloud }}"
       changed_when: false

collection/stages/roles/verification/tasks/main.yml

@@ -119,7 +119,7 @@
         - not _skip_health  # CSI is disabled post-adoption
 
     - name: Check if manila is present on the OSP installation
-      ansible.builtin.command: openstack catalog show manila -c name
+      ansible.builtin.command: openstack catalog show manila -f json
       environment:
         OS_CLOUD: "{{ user_cloud }}"
       register: manila_enabled