Added APIC OOB connectivity checks for CSCwu91693

aci-preupgrade-validation-script.py

@@ -6797,6 +6797,88 @@ def infravlan_overlap_access_policy_check(tversion, **kwargs):
     return Result(result=result, msg=msg, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+@check_wrapper(check_title="APIC OOB Connectivity check")
+def apic_oob_connectivity_check(cversion, tversion, **kwargs):
+    result = PASS
+    headers = ["Node ID", "OOB IP", "Port", "Status"]
+    recommended_action = "Restore OOB management connectivity between all APICs and ensure the required HTTPS ports are reachable across the OOB network."
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-oob-connectivity'
+
+    def get_apic_oob_connectivity(apic_id_ip, port):
+        data = []
+        has_error = False
+
+        for apic in apic_id_ip:
+            attrs = apic['topSystem']['attributes']
+            node_id = attrs.get('id', '')
+
+            if attrs.get('oobMgmtAddr', '0.0.0.0') != '0.0.0.0':
+                ip = attrs.get('oobMgmtAddr')
+            elif attrs.get('oobMgmtAddr6', '::') not in ('', '::', '0:0:0:0:0:0:0:0'):
+                ip = attrs.get('oobMgmtAddr6')
+            else:
+                continue
+
+            try:
+                ip_formatted = '[{}]'.format(ip) if ':' in ip else ip
+                with open(os.devnull, 'wb') as devnull:
+                    if subprocess.call(
+                        ['curl', '--max-time', '5', '-k', '-s', '-o', os.devnull,
+                         'https://{}:{}'.format(ip_formatted, port)],
+                        stderr=devnull
+                    ) != 0:
+                        data.append([node_id, ip, port, "Unreachable"])
+            except Exception as e:
+                log.error("Exception checking OOB connectivity for node %s: %s", node_id, e)
+                data.append([node_id, ip, port, "Error"])
+                has_error = True
+                continue
+
+        return data, has_error
+
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING)
+
+    if tversion.older_than("6.0(2a)"):
+        return Result(result=NA, msg=VER_NOT_AFFECTED)
+
+    apic_id_ip = icurl('class', 'topSystem.json?query-target-filter=eq(topSystem.role,"controller")')
+    if not apic_id_ip:
+        return Result(result=NA, msg="No APIC controller nodes found.")
+
+    data = []
+    has_error = False
+
+    # Default port check: APIC bootx uses port 443 from 6.0(2)
+    default_data, default_error = get_apic_oob_connectivity(apic_id_ip, 443)
+    data.extend(default_data)
+    if default_error:
+        has_error = True
+
+    # Custom HTTPS port check: upgrade fanout uses commHttps port from 6.2(1)
+    if not cversion.older_than("6.2(1g)"):
+        port = 443
+        commHttps = icurl('class', 'commHttps.json')
+        if commHttps:
+            try:
+                port = int(commHttps[0]['commHttps']['attributes'].get('port', 443))
+            except (ValueError, KeyError):
+                log.warning("Could not read commHttps port")
+                return Result(result=ERROR, msg="Could not read https port id from commHttps MO.")
+
+        if port != 443:  # Port 443 already covered by default port check above
+            custom_data, custom_error = get_apic_oob_connectivity(apic_id_ip, port)
+            data.extend(custom_data)
+            if custom_error:
+                has_error = True
+
+    if has_error:
+        result = ERROR
+    elif data:
+        result = FAIL_UF
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
+
 # ---- Script Execution ----
 
 
@@ -6973,7 +7055,7 @@ class CheckManager:
         n9k_c93180yc_fx3_switch_memory_check,
         stale_dbgacEpgSummaryTask_check,
         infravlan_overlap_access_policy_check,
-
+        apic_oob_connectivity_check,
     ]
     ssh_checks = [
         # General

docs/docs/validations.md

@@ -207,6 +207,7 @@ Items                                           | Defect       | This Script
 [N9K-C93180YC-FX3 Switch Memory Less Than 32GB][d36] | CSCwm42741   | :white_check_mark: | :no_entry_sign:
 [Stale dbgacEpgSummaryTask Objects][d37]         | CSCwt69100   | :white_check_mark: | :no_entry_sign:
 [InfraVLAN Overlap in Access Policy VLAN Pools][d38] | CSCwt58626   | :white_check_mark: | :no_entry_sign:
+[APIC OOB Connectivity][d39]                      | CSCwu91693   | :white_check_mark: | :no_entry_sign:
 
 [d1]: #ep-announce-compatibility
 [d2]: #eventmgr-db-size-defect-susceptibility
@@ -246,6 +247,7 @@ Items                                           | Defect       | This Script
 [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb
 [d37]: #stale-dbgacepgsummarytask-objects
 [d38]: #infravlan-overlap-access-policy-check
+[d39]: #apic-oob-connectivity
 
 ## General Check Details
 
@@ -2858,6 +2860,14 @@ Due to the bug [CSCwt58626][77] , If Apic upgrade planned for target versions 6.
 
 To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does not overlap with any configured block, or select a non-impacted fixed version. After upgrading to a fixed version this fault and Restriction have been removed.
 
+### APIC OOB Connectivity
+
+Starting from 6.0(2), APIC firmware upgrades are triggered via an HTTPS POST request (bootx) sent to each peer APIC over its out-of-band (OOB) management interface. Due to [CSCwu91693][78], if OOB connectivity to a peer APIC is unavailable at the time this trigger is sent, that APIC does not receive it and silently fails to start the upgrade, while the remaining reachable APICs proceed normally. This results in a partially upgraded cluster with no explicit error raised at the time of failure.
+
+This check verifies OOB reachability between APICs on the port(s) actually used for the upgrade trigger. The default port 443 is validated on all versions from 6.0(2) onward, since it is always used unless a custom HTTPS port is configured. From 6.2(1) onward, the upgrade trigger also honors a custom HTTPS port if one is configured via the `commHttps` policy; this custom port is validated only when the current version is 6.2(1) or later, and only when the configured port differs from 443, which is already covered by the default check.
+
+For each applicable port, the script gets OOB management IP of every APIC in the cluster, then attempts an HTTPS connection to each peer on that port with a 5-second timeout. If any APIC is found unreachable on a port used for the upgrade trigger, the check fails.
+
 [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script
 [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html
 [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html
@@ -2935,3 +2945,4 @@ To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does
 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100
 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698
 [77]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt58626
+[78]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwu91693

tests/checks/apic_oob_connectivity_check/commHttps_custom_port.json

@@ -0,0 +1,9 @@
+[
+    {
+        "commHttps": {
+            "attributes": {
+                "port": "8443"
+            }
+        }
+    }
+]

tests/checks/apic_oob_connectivity_check/commHttps_default_port.json

@@ -0,0 +1,9 @@
+[
+    {
+        "commHttps": {
+            "attributes": {
+                "port": "443"
+            }
+        }
+    }
+]

tests/checks/apic_oob_connectivity_check/commHttps_invalid_port.json

@@ -0,0 +1 @@
+[{"commHttps": {"attributes": {"port": "invalid"}}}]