diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index af8ddb3..c94c3e9 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6797,6 +6797,88 @@ def infravlan_overlap_access_policy_check(tversion, **kwargs): return Result(result=result, msg=msg, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) +@check_wrapper(check_title="APIC OOB Connectivity check") +def apic_oob_connectivity_check(cversion, tversion, **kwargs): + result = PASS + headers = ["Node ID", "OOB IP", "Port", "Status"] + recommended_action = "Restore OOB management connectivity between all APICs and ensure the required HTTPS ports are reachable across the OOB network." + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-oob-connectivity' + + def get_apic_oob_connectivity(apic_id_ip, port): + data = [] + has_error = False + + for apic in apic_id_ip: + attrs = apic['topSystem']['attributes'] + node_id = attrs.get('id', '') + + if attrs.get('oobMgmtAddr', '0.0.0.0') != '0.0.0.0': + ip = attrs.get('oobMgmtAddr') + elif attrs.get('oobMgmtAddr6', '::') not in ('', '::', '0:0:0:0:0:0:0:0'): + ip = attrs.get('oobMgmtAddr6') + else: + continue + + try: + ip_formatted = '[{}]'.format(ip) if ':' in ip else ip + with open(os.devnull, 'wb') as devnull: + if subprocess.call( + ['curl', '--max-time', '5', '-k', '-s', '-o', os.devnull, + 'https://{}:{}'.format(ip_formatted, port)], + stderr=devnull + ) != 0: + data.append([node_id, ip, port, "Unreachable"]) + except Exception as e: + log.error("Exception checking OOB connectivity for node %s: %s", node_id, e) + data.append([node_id, ip, port, "Error"]) + has_error = True + continue + + return data, has_error + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + if tversion.older_than("6.0(2a)"): + return Result(result=NA, msg=VER_NOT_AFFECTED) + + apic_id_ip = icurl('class', 'topSystem.json?query-target-filter=eq(topSystem.role,"controller")') + if not apic_id_ip: + return Result(result=NA, msg="No APIC controller nodes found.") + + data = [] + has_error = False + + # Default port check: APIC bootx uses port 443 from 6.0(2) + default_data, default_error = get_apic_oob_connectivity(apic_id_ip, 443) + data.extend(default_data) + if default_error: + has_error = True + + # Custom HTTPS port check: upgrade fanout uses commHttps port from 6.2(1) + if not cversion.older_than("6.2(1g)"): + port = 443 + commHttps = icurl('class', 'commHttps.json') + if commHttps: + try: + port = int(commHttps[0]['commHttps']['attributes'].get('port', 443)) + except (ValueError, KeyError): + log.warning("Could not read commHttps port") + return Result(result=ERROR, msg="Could not read https port id from commHttps MO.") + + if port != 443: # Port 443 already covered by default port check above + custom_data, custom_error = get_apic_oob_connectivity(apic_id_ip, port) + data.extend(custom_data) + if custom_error: + has_error = True + + if has_error: + result = ERROR + elif data: + result = FAIL_UF + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6973,7 +7055,7 @@ class CheckManager: n9k_c93180yc_fx3_switch_memory_check, stale_dbgacEpgSummaryTask_check, infravlan_overlap_access_policy_check, - + apic_oob_connectivity_check, ] ssh_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 6695fc9..ddc2793 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -207,6 +207,7 @@ Items | Defect | This Script [N9K-C93180YC-FX3 Switch Memory Less Than 32GB][d36] | CSCwm42741 | :white_check_mark: | :no_entry_sign: [Stale dbgacEpgSummaryTask Objects][d37] | CSCwt69100 | :white_check_mark: | :no_entry_sign: [InfraVLAN Overlap in Access Policy VLAN Pools][d38] | CSCwt58626 | :white_check_mark: | :no_entry_sign: +[APIC OOB Connectivity][d39] | CSCwu91693 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -246,6 +247,7 @@ Items | Defect | This Script [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb [d37]: #stale-dbgacepgsummarytask-objects [d38]: #infravlan-overlap-access-policy-check +[d39]: #apic-oob-connectivity ## General Check Details @@ -2858,6 +2860,14 @@ Due to the bug [CSCwt58626][77] , If Apic upgrade planned for target versions 6. To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does not overlap with any configured block, or select a non-impacted fixed version. After upgrading to a fixed version this fault and Restriction have been removed. +### APIC OOB Connectivity + +Starting from 6.0(2), APIC firmware upgrades are triggered via an HTTPS POST request (bootx) sent to each peer APIC over its out-of-band (OOB) management interface. Due to [CSCwu91693][78], if OOB connectivity to a peer APIC is unavailable at the time this trigger is sent, that APIC does not receive it and silently fails to start the upgrade, while the remaining reachable APICs proceed normally. This results in a partially upgraded cluster with no explicit error raised at the time of failure. + +This check verifies OOB reachability between APICs on the port(s) actually used for the upgrade trigger. The default port 443 is validated on all versions from 6.0(2) onward, since it is always used unless a custom HTTPS port is configured. From 6.2(1) onward, the upgrade trigger also honors a custom HTTPS port if one is configured via the `commHttps` policy; this custom port is validated only when the current version is 6.2(1) or later, and only when the configured port differs from 443, which is already covered by the default check. + +For each applicable port, the script gets OOB management IP of every APIC in the cluster, then attempts an HTTPS connection to each peer on that port with a 5-second timeout. If any APIC is found unreachable on a port used for the upgrade trigger, the check fails. + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2935,3 +2945,4 @@ To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 [77]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt58626 +[78]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwu91693 diff --git a/tests/checks/apic_oob_connectivity_check/commHttps_custom_port.json b/tests/checks/apic_oob_connectivity_check/commHttps_custom_port.json new file mode 100644 index 0000000..1ce3911 --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/commHttps_custom_port.json @@ -0,0 +1,9 @@ +[ + { + "commHttps": { + "attributes": { + "port": "8443" + } + } + } +] diff --git a/tests/checks/apic_oob_connectivity_check/commHttps_default_port.json b/tests/checks/apic_oob_connectivity_check/commHttps_default_port.json new file mode 100644 index 0000000..5c99dc5 --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/commHttps_default_port.json @@ -0,0 +1,9 @@ +[ + { + "commHttps": { + "attributes": { + "port": "443" + } + } + } +] diff --git a/tests/checks/apic_oob_connectivity_check/commHttps_invalid_port.json b/tests/checks/apic_oob_connectivity_check/commHttps_invalid_port.json new file mode 100644 index 0000000..ca1c3f2 --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/commHttps_invalid_port.json @@ -0,0 +1 @@ +[{"commHttps": {"attributes": {"port": "invalid"}}}] diff --git a/tests/checks/apic_oob_connectivity_check/test_apic_oob_connectivity_check.py b/tests/checks/apic_oob_connectivity_check/test_apic_oob_connectivity_check.py new file mode 100644 index 0000000..38f297d --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/test_apic_oob_connectivity_check.py @@ -0,0 +1,249 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "apic_oob_connectivity_check" + +# icurl queries +topSystem = 'topSystem.json?query-target-filter=eq(topSystem.role,"controller")' +commHttps = "commHttps.json" + +# headers returned only when the check reaches its final return (early MANUAL/NA/ERROR returns have no headers) +HEADERS = ["Node ID", "OOB IP", "Port", "Status"] + + +@pytest.mark.parametrize( + "icurl_outputs, cversion, tversion, curl_exit_codes, expected_result, expected_headers, expected_data", + [ + # tversion not provided -> MANUAL + ( + {topSystem: [], commHttps: []}, + "6.0(2a)", + None, + [], + script.MANUAL, + [], + [], + ), + # tversion < 6.0(2a) -> NA (version not affected) + ( + {topSystem: [], commHttps: []}, + "5.2(7f)", + "5.2(7f)", + [], + script.NA, + [], + [], + ), + # tversion = 6.0(1h), immediately below the 6.0(2a) gate -> NA + ( + {topSystem: [], commHttps: []}, + "6.0(1h)", + "6.0(1h)", + [], + script.NA, + [], + [], + ), + # tversion >= 6.0(2a), no controller nodes found -> NA + ( + {topSystem: [], commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [], + script.NA, + [], + [], + ), + # tversion >= 6.0(2a), all APICs have no OOB configured -> PASS + ( + {topSystem: read_data(dir, "topSystem_no_oob.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [], + script.PASS, + HEADERS, + [], + ), + # tversion >= 6.0(2a), all APICs reachable on port 443 -> PASS + ( + {topSystem: read_data(dir, "topSystem_3apics_oob.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [0, 0, 0], + script.PASS, + HEADERS, + [], + ), + # cversion >= 6.2(1g), default port 443, all APICs reachable -> PASS + # (custom port check skipped since commHttps port == 443) + ( + { + topSystem: read_data(dir, "topSystem_3apics_oob.json"), + commHttps: read_data(dir, "commHttps_default_port.json"), + }, + "6.2(1g)", + "6.2(2a)", + [0, 0, 0], + script.PASS, + HEADERS, + [], + ), + # cversion >= 6.2(1g), custom port 8443, all APICs reachable on both ports -> PASS + # (default port 443: [0,0,0], custom port 8443: [0,0,0]) + ( + { + topSystem: read_data(dir, "topSystem_3apics_oob.json"), + commHttps: read_data(dir, "commHttps_custom_port.json"), + }, + "6.2(1g)", + "6.2(2a)", + [0, 0, 0, 0, 0, 0], + script.PASS, + HEADERS, + [], + ), + # cversion = 6.2(1f), immediately below the 6.2(1g) gate, custom port configured -> PASS + # (custom port check must be skipped; only 3 curl calls for port 443, not 6) + ( + { + topSystem: read_data(dir, "topSystem_3apics_oob.json"), + commHttps: read_data(dir, "commHttps_custom_port.json"), + }, + "6.2(1f)", + "6.2(2a)", + [0, 0, 0], + script.PASS, + HEADERS, + [], + ), + # tversion >= 6.0(2a), one APIC unreachable on port 443 (exit 28) -> FAIL_UF + ( + {topSystem: read_data(dir, "topSystem_3apics_oob.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [0, 28, 0], + script.FAIL_UF, + HEADERS, + [["2", "10.30.10.191", 443, "Unreachable"]], + ), + # cversion >= 6.2(1g), one APIC unreachable on default port 443 (exit 7) -> FAIL_UF + ( + { + topSystem: read_data(dir, "topSystem_3apics_oob.json"), + commHttps: read_data(dir, "commHttps_default_port.json"), + }, + "6.2(1g)", + "6.2(2a)", + [0, 7, 0], + script.FAIL_UF, + HEADERS, + [["2", "10.30.10.191", 443, "Unreachable"]], + ), + # cversion >= 6.2(1g), custom port 8443, all unreachable on custom port -> FAIL_UF + # (default port 443 all pass: [0,0,0], custom port 8443 all fail: [28,28,28]) + ( + { + topSystem: read_data(dir, "topSystem_3apics_oob.json"), + commHttps: read_data(dir, "commHttps_custom_port.json"), + }, + "6.2(1g)", + "6.2(2a)", + [0, 0, 0, 28, 28, 28], + script.FAIL_UF, + HEADERS, + [ + ["1", "10.30.10.189", 8443, "Unreachable"], + ["2", "10.30.10.191", 8443, "Unreachable"], + ["3", "10.30.10.193", 8443, "Unreachable"], + ], + ), + # cversion >= 6.2(1g), commHttps returns invalid port value -> ERROR + ( + { + topSystem: read_data(dir, "topSystem_3apics_oob.json"), + commHttps: read_data(dir, "commHttps_invalid_port.json"), + }, + "6.2(1g)", + "6.2(2a)", + [0, 0, 0], + script.ERROR, + [], + [], + ), + # IPv6 OOB, all APICs reachable -> PASS + ( + {topSystem: read_data(dir, "topSystem_3apics_oob_ipv6.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [0, 0, 0], + script.PASS, + HEADERS, + [], + ), + # IPv6 OOB, one APIC unreachable (exit 28) -> FAIL_UF + ( + {topSystem: read_data(dir, "topSystem_3apics_oob_ipv6.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [0, 28, 0], + script.FAIL_UF, + HEADERS, + [["2", "2001:db8::2", 443, "Unreachable"]], + ), + # Both IPv4 and IPv6 configured, all reachable -> PASS (IPv4 should be used) + ( + {topSystem: read_data(dir, "topSystem_3apics_oob_both.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [0, 0, 0], + script.PASS, + HEADERS, + [], + ), + # Both IPv4 and IPv6 configured, one unreachable -> FAIL_UF (IPv4 should be used) + ( + {topSystem: read_data(dir, "topSystem_3apics_oob_both.json"), commHttps: []}, + "6.0(2a)", + "6.0(3a)", + [0, 28, 0], + script.FAIL_UF, + HEADERS, + [["2", "10.30.10.191", 443, "Unreachable"]], + ), + ], +) +def test_logic(run_check, mock_icurl, monkeypatch, cversion, tversion, curl_exit_codes, expected_result, expected_headers, expected_data): + idx = [0] + + def mock_subprocess_call(cmd, stderr=None): + # Verify that IPv6 addresses in the curl URL are wrapped in square brackets + import re + url = cmd[-1] + match = re.search(r'https://([^/]+):', url) + if match: + host = match.group(1) + # If host contains a colon (IPv6) it must be wrapped in brackets + if ':' in host: + assert host.startswith('[') and host.endswith(']'), ( + "IPv6 address in curl URL must be wrapped in square brackets, got: {}".format(url) + ) + code = curl_exit_codes[idx[0]] if idx[0] < len(curl_exit_codes) else 0 + idx[0] += 1 + return code + + monkeypatch.setattr(script.subprocess, "call", mock_subprocess_call) + result = run_check( + cversion=script.AciVersion(cversion), + tversion=script.AciVersion(tversion) if tversion else None, + ) + assert result.result == expected_result + assert result.headers == expected_headers + assert result.data == expected_data diff --git a/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob.json b/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob.json new file mode 100644 index 0000000..38f86d2 --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob.json @@ -0,0 +1,32 @@ +[ + { + "topSystem": { + "attributes": { + "id": "1", + "role": "controller", + "oobMgmtAddr": "10.30.10.189", + "oobMgmtAddr6": "::" + } + } + }, + { + "topSystem": { + "attributes": { + "id": "2", + "role": "controller", + "oobMgmtAddr": "10.30.10.191", + "oobMgmtAddr6": "::" + } + } + }, + { + "topSystem": { + "attributes": { + "id": "3", + "role": "controller", + "oobMgmtAddr": "10.30.10.193", + "oobMgmtAddr6": "::" + } + } + } +] diff --git a/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_both.json b/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_both.json new file mode 100644 index 0000000..14dded9 --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_both.json @@ -0,0 +1,32 @@ +[ + { + "topSystem": { + "attributes": { + "id": "1", + "role": "controller", + "oobMgmtAddr": "10.30.10.189", + "oobMgmtAddr6": "2001:db8::1" + } + } + }, + { + "topSystem": { + "attributes": { + "id": "2", + "role": "controller", + "oobMgmtAddr": "10.30.10.191", + "oobMgmtAddr6": "2001:db8::2" + } + } + }, + { + "topSystem": { + "attributes": { + "id": "3", + "role": "controller", + "oobMgmtAddr": "10.30.10.193", + "oobMgmtAddr6": "2001:db8::3" + } + } + } +] diff --git a/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_ipv6.json b/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_ipv6.json new file mode 100644 index 0000000..5c4e8ea --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/topSystem_3apics_oob_ipv6.json @@ -0,0 +1,32 @@ +[ + { + "topSystem": { + "attributes": { + "id": "1", + "role": "controller", + "oobMgmtAddr": "0.0.0.0", + "oobMgmtAddr6": "2001:db8::1" + } + } + }, + { + "topSystem": { + "attributes": { + "id": "2", + "role": "controller", + "oobMgmtAddr": "0.0.0.0", + "oobMgmtAddr6": "2001:db8::2" + } + } + }, + { + "topSystem": { + "attributes": { + "id": "3", + "role": "controller", + "oobMgmtAddr": "0.0.0.0", + "oobMgmtAddr6": "2001:db8::3" + } + } + } +] diff --git a/tests/checks/apic_oob_connectivity_check/topSystem_no_oob.json b/tests/checks/apic_oob_connectivity_check/topSystem_no_oob.json new file mode 100644 index 0000000..aa586ef --- /dev/null +++ b/tests/checks/apic_oob_connectivity_check/topSystem_no_oob.json @@ -0,0 +1,12 @@ +[ + { + "topSystem": { + "attributes": { + "id": "1", + "role": "controller", + "oobMgmtAddr": "0.0.0.0", + "oobMgmtAddr6": "::" + } + } + } +]