From a64331261c3d21b475ddbf9223fe636f4aa45aec Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 21 May 2026 11:36:53 -0400 Subject: [PATCH 01/44] mark as v4.2.0-dev --- aci-preupgrade-validation-script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index b420eeae..d5a4400a 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -38,7 +38,7 @@ import os import re -SCRIPT_VERSION = "v4.1.0" +SCRIPT_VERSION = "v4.2.0-dev" DEFAULT_TIMEOUT = 600 # sec # result constants DONE = 'DONE' From 916ca44ce1bf17a25ceaba392c5cb8cd9ff4ac66 Mon Sep 17 00:00:00 2001 From: dhaselva Date: Thu, 21 May 2026 21:37:01 +0530 Subject: [PATCH 02/44] Unsupported hardware check (#364) * Add arg to define thread limit - to throttle concurrent API calls when required (#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * Unsupported hardware check * fix: Add cversion check for post_upgrade_cb_check (#377) * added validation for CFD CSCwp64296 (#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass * Addressed the review comments * Added validation for CSCwd40071 (#332) * Added validation for CSCwd40071 * Addressed the comments * Added cversion for the check * Removed empty spaces * logic change. removed 0.0.0.0/0 and made pytest changes * logic modified and validation.md file updated * Addressed the review comments * Added validation for CSCws84232 (#334) * Feat:added svccoreCtrlr excessive entries check * updated the formated json in svccore_negative.json * Removed other PR checks * Fix: svccoreNode mo check is updated * Fix:update the threshold to 240 * refactor:moved to general check * refactor:updated recommended action * refactor:optimise the check for get the count alone * refactor:added try catch exception for error handling * update the threshold value for 240 * refactor:query-target was added * refactor: removed unwanted spaces * refactor:removed unwanted spaces in validation.md * resolve conflict while rebasing * resolve conflict while rebasing * updated pytest script file * updated validation.md * addressed review comments * updated validation.md --------- Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> * Removed gen1_switch_compatibility_check --------- Co-authored-by: GM Co-authored-by: Gabriel Co-authored-by: tkishida Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb Co-authored-by: Harinadh-Saladi Co-authored-by: sanjanch Co-authored-by: asraf-khan Co-authored-by: Thatleft Co-authored-by: Detlef Sass Co-authored-by: veesenth_cisco --- aci-preupgrade-validation-script.py | 84 +++++++++-- docs/docs/validations.md | 17 ++- .../fabricNode_no_gen1.json | 30 ---- .../fabricNode_with_gen1.json | 44 ------ .../test_gen1_switch_compatibility_check.py | 42 ------ .../eqptExtCh_supported_only.json | 10 ++ .../eqptExtCh_with_unsupported.json | 18 +++ .../eqptLC_supported_only.json | 10 ++ .../eqptLC_with_unformatted_dn.json | 10 ++ .../eqptLC_with_unsupported.json | 18 +++ .../eqptSupC_supported_only.json | 10 ++ .../eqptSupC_with_unsupported.json | 18 +++ .../fabricNode_supported_only.json | 20 +++ .../fabricNode_with_unsupported_hardware.json | 38 +++++ .../test_supported_hardware_check.py | 133 ++++++++++++++++++ 15 files changed, 370 insertions(+), 132 deletions(-) delete mode 100644 tests/checks/gen1_switch_compatibility_check/fabricNode_no_gen1.json delete mode 100644 tests/checks/gen1_switch_compatibility_check/fabricNode_with_gen1.json delete mode 100644 tests/checks/gen1_switch_compatibility_check/test_gen1_switch_compatibility_check.py create mode 100644 tests/checks/supported_hardware_check/eqptExtCh_supported_only.json create mode 100644 tests/checks/supported_hardware_check/eqptExtCh_with_unsupported.json create mode 100644 tests/checks/supported_hardware_check/eqptLC_supported_only.json create mode 100644 tests/checks/supported_hardware_check/eqptLC_with_unformatted_dn.json create mode 100644 tests/checks/supported_hardware_check/eqptLC_with_unsupported.json create mode 100644 tests/checks/supported_hardware_check/eqptSupC_supported_only.json create mode 100644 tests/checks/supported_hardware_check/eqptSupC_with_unsupported.json create mode 100644 tests/checks/supported_hardware_check/fabricNode_supported_only.json create mode 100644 tests/checks/supported_hardware_check/fabricNode_with_unsupported_hardware.json create mode 100644 tests/checks/supported_hardware_check/test_supported_hardware_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index d5a4400a..a951bfe4 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3866,27 +3866,83 @@ def target_version_compatibility_check(cversion, tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) -@check_wrapper(check_title="Gen 1 switch compatibility") -def gen1_switch_compatibility_check(tversion, fabric_nodes, **kwargs): +@check_wrapper(check_title="Supported hardware compatibility") +def supported_hardware_check(tversion, fabric_nodes, **kwargs): result = FAIL_UF - headers = ["Target Version", "Node ID", "Model", "Warning"] + headers = ["Target Version", "Node ID", "Model", "Type", "Warning"] + data = [] + unformatted_headers = ["Target Version", "DN", "Model", "Type", "Warning"] + unformatted_data = [] gen1_models = ["N9K-C9336PQ", "N9K-X9736PQ", "N9K-C9504-FM", "N9K-C9508-FM", "N9K-C9516-FM", "N9K-C9372PX-E", "N9K-C9372TX-E", "N9K-C9332PQ", "N9K-C9372PX", "N9K-C9372TX", "N9K-C9396PX", "N9K-C9396TX", "N9K-C93128TX"] - data = [] + unsupported_6_0_1_switch_models = ["N9K-C93120TX"] + unsupported_6_1_1_switch_models = ["N9K-C93180LC-EX"] + unsupported_5_0_1_exp_module_models = ["N9K-M12PQ", "N9K-M6PQ", "N9K-M6PQ-E"] + unsupported_6_1_1_fex_models = ["N2K-C2332TQ-10GT", "N2K-C2348TQ-10GE", "N2K-C2232PP-10GE", "N2K-C2232TM-E-10GE", "N2K-C2348TQ-10G-E"] + unsupported_6_1_1_sup_models = ["N9K-SUP-A", "N9K-SUP-B"] recommended_action = 'Select supported target version or upgrade hardware' - doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#compatibility-switch-hardware-gen1' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#supported-hardware-compatibility' - if not tversion: - return Result(result=MANUAL, msg=TVER_MISSING) - if tversion.newer_than("5.0(1a)"): + if not tversion.older_than("5.0(1a)"): for node in fabric_nodes: - if node['fabricNode']['attributes']['model'] in gen1_models: - data.append([str(tversion), node['fabricNode']['attributes']['id'], - node['fabricNode']['attributes']['model'], 'Not supported on 5.x+']) - if not data: + model = node['fabricNode']['attributes']['model'] + if model in gen1_models: + data.append([str(tversion), node['fabricNode']['attributes']['id'], model, 'Switch', 'Not supported on 5.x+']) + + eqptLCs = icurl('class', 'eqptLC.json') + for eqptLC in eqptLCs: + model = eqptLC['eqptLC']['attributes']['model'] + if model in unsupported_5_0_1_exp_module_models: + dn = re.search(node_regex, eqptLC['eqptLC']['attributes']['dn']) + if dn: + data.append([str(tversion), dn.group('node'), model, 'Expansion Module', 'Not supported on 5.x+']) + else: + unformatted_data.append([str(tversion), eqptLC['eqptLC']['attributes']['dn'], model, 'Expansion Module', 'Not supported on 5.x+']) + + if not tversion.older_than("6.0(1a)"): + for node in fabric_nodes: + model = node['fabricNode']['attributes']['model'] + if model in unsupported_6_0_1_switch_models: + data.append([str(tversion), node['fabricNode']['attributes']['id'], model, 'Switch', 'Deprecated from 6.0(1)+']) + + if not tversion.older_than("6.1(1f)"): + for node in fabric_nodes: + model = node['fabricNode']['attributes']['model'] + if model in unsupported_6_1_1_switch_models: + data.append([str(tversion), node['fabricNode']['attributes']['id'], model, 'Switch', 'Deprecated from 6.1(1)+']) + + eqptExtChs = icurl('class', 'eqptExtCh.json') + for eqptExtCh in eqptExtChs: + model = eqptExtCh['eqptExtCh']['attributes']['model'] + if model in unsupported_6_1_1_fex_models: + dn = re.search(node_regex, eqptExtCh['eqptExtCh']['attributes']['dn']) + if dn: + data.append([str(tversion), dn.group('node'), model, 'FEX', 'Deprecated from 6.1(1)+']) + else: + unformatted_data.append([str(tversion), eqptExtCh['eqptExtCh']['attributes']['dn'], model, 'FEX', 'Deprecated from 6.1(1)+']) + + eqptSupCs = icurl('class', 'eqptSupC.json') + for eqptSupC in eqptSupCs: + model = eqptSupC['eqptSupC']['attributes']['model'] + if model in unsupported_6_1_1_sup_models: + dn = re.search(node_regex, eqptSupC['eqptSupC']['attributes']['dn']) + if dn: + data.append([str(tversion), dn.group('node'), model, 'Supervisor', 'Deprecated from 6.1(1)+']) + else: + unformatted_data.append([str(tversion), eqptSupC['eqptSupC']['attributes']['dn'], model, 'Supervisor', 'Deprecated from 6.1(1)+']) + + if not data and not unformatted_data: result = PASS - return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + return Result( + result=result, + headers=headers, + data=data, + unformatted_headers=unformatted_headers, + unformatted_data=unformatted_data, + recommended_action=recommended_action, + doc_url=doc_url, + ) @check_wrapper(check_title="Contract Port 22 Defect") @@ -6417,7 +6473,7 @@ class CheckManager: api_checks = [ # General Checks target_version_compatibility_check, - gen1_switch_compatibility_check, + supported_hardware_check, r_leaf_compatibility_check, cimc_compatibilty_check, apic_cluster_health_check, diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 64d59554..82f22119 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -37,7 +37,8 @@ Items | This Script [Fabric Link Redundancy][g17] | :white_check_mark: | :no_entry_sign: [APIC Database Size][g18] | :white_check_mark: | :no_entry_sign: [APIC downgrade compatibility when crossing 6.2 release][g19]| :white_check_mark: | :no_entry_sign: -[Svccore Excessive Data Check][g20] | :white_check_mark: | :no_entry_sign: +[Supported Hardware Compatibility][g20] | :white_check_mark: | :no_entry_sign: +[Svccore Excessive Data Check][g21] | :white_check_mark: | :no_entry_sign: [g1]: #compatibility-target-aci-version [g2]: #compatibility-cimc-version @@ -58,7 +59,8 @@ Items | This Script [g17]: #fabric-link-redundancy [g18]: #apic-database-size [g19]: #apic-downgrade-compatibility-when-crossing-62-release -[g20]: #svccore-excessive-data-check +[g20]: #supported-hardware-compatibility +[g21]: #svccore-excessive-data-check ### Fault Checks Items | Faults | This Script | APIC built-in @@ -267,6 +269,17 @@ The script checks the presence of generation one switches when the upgrade is cr Or you can check the [Release Note 15.0(1) of ACI switches][3] to see the list of generation one switches, typically the one without any suffix such as N9K-C9372PX, that are no longer supported from 15.0(1) release. +### Supported Hardware Compatibility + +The script checks the presence of deprecated hardware in the fabric. + +The list of supported and unsupported hardware is populated from the Release Notes across all ACI releases. This means the check covers hardware compatibility changes introduced in any version, not just the most recent release. As new release notes are published and hardware is deprecated, this list is updated accordingly. + +Refer the [Release Note 15.0(1) of ACI switches][3] to see the list of unsuporrted hardware for your desired target versions. Prior upgrading to target version, replace the unsupported hardware elements in your fabric with other supported hardware. + +Contact cisco TAC for further assistance. + + ### Compatibility (Remote Leaf Switch) The script checks the requirement to use remote leaf switches on the target version. diff --git a/tests/checks/gen1_switch_compatibility_check/fabricNode_no_gen1.json b/tests/checks/gen1_switch_compatibility_check/fabricNode_no_gen1.json deleted file mode 100644 index d266e8af..00000000 --- a/tests/checks/gen1_switch_compatibility_check/fabricNode_no_gen1.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "fabricNode": { - "attributes": { - "adSt": "on", - "dn": "topology/pod-2/node-201", - "fabricSt": "active", - "id": "201", - "model": "N9K-C93180YC-FX", - "name": "RL201", - "nodeType": "remote-leaf-wan", - "role": "leaf" - } - } - }, - { - "fabricNode": { - "attributes": { - "adSt": "on", - "dn": "topology/pod-2/node-202", - "fabricSt": "active", - "id": "202", - "model": "N9K-C93180YC-FX", - "name": "RL202", - "nodeType": "remote-leaf-wan", - "role": "leaf" - } - } - } -] diff --git a/tests/checks/gen1_switch_compatibility_check/fabricNode_with_gen1.json b/tests/checks/gen1_switch_compatibility_check/fabricNode_with_gen1.json deleted file mode 100644 index e01273b6..00000000 --- a/tests/checks/gen1_switch_compatibility_check/fabricNode_with_gen1.json +++ /dev/null @@ -1,44 +0,0 @@ -[ - { - "fabricNode": { - "attributes": { - "adSt": "on", - "dn": "topology/pod-1/node-101", - "fabricSt": "active", - "id": "101", - "model": "N9K-C9372TX-E", - "name": "Leaf-101", - "nodeType": "unspecified", - "role": "leaf" - } - } - }, - { - "fabricNode": { - "attributes": { - "adSt": "on", - "dn": "topology/pod-1/node-102", - "fabricSt": "active", - "id": "102", - "model": "N9K-C9372TX-E", - "name": "Leaf-102", - "nodeType": "unspecified", - "role": "leaf" - } - } - }, - { - "fabricNode": { - "attributes": { - "adSt": "on", - "dn": "topology/pod-1/node-1001", - "fabricSt": "active", - "id": "1001", - "model": "N9K-C9332PQ", - "name": "Spine-1001", - "nodeType": "unspecified", - "role": "spine" - } - } - } -] diff --git a/tests/checks/gen1_switch_compatibility_check/test_gen1_switch_compatibility_check.py b/tests/checks/gen1_switch_compatibility_check/test_gen1_switch_compatibility_check.py deleted file mode 100644 index 4f076c56..00000000 --- a/tests/checks/gen1_switch_compatibility_check/test_gen1_switch_compatibility_check.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import pytest -import logging -import importlib -from helpers.utils import read_data - -log = logging.getLogger(__name__) -dir = os.path.dirname(os.path.abspath(__file__)) - -script = importlib.import_module("aci-preupgrade-validation-script") -AciVersion = script.AciVersion - -test_function = "gen1_switch_compatibility_check" - - -@pytest.mark.parametrize( - "tversion, fabric_nodes, expected_result, expected_data", - [ - # FAIL - gen1 HW does not support t_ver - ( - "5.2(3b)", - read_data(dir, "fabricNode_with_gen1.json"), - script.FAIL_UF, - [ - ["5.2(3b)", "101", "N9K-C9372TX-E", "Not supported on 5.x+"], - ["5.2(3b)", "102", "N9K-C9372TX-E", "Not supported on 5.x+"], - ["5.2(3b)", "1001", "N9K-C9332PQ", "Not supported on 5.x+"], - ], - ), - # PASS - gen1 HW supports t_ver - ("4.2(7r)", read_data(dir, "fabricNode_with_gen1.json"), script.PASS, []), - # PASS - no gen1 hw found - ("5.2(3b)", read_data(dir, "fabricNode_no_gen1.json"), script.PASS, []), - ], -) -def test_logic(run_check, tversion, fabric_nodes, expected_result, expected_data): - result = run_check( - tversion=AciVersion(tversion) if tversion else None, - fabric_nodes=fabric_nodes, - ) - assert result.result == expected_result - assert result.data == expected_data diff --git a/tests/checks/supported_hardware_check/eqptExtCh_supported_only.json b/tests/checks/supported_hardware_check/eqptExtCh_supported_only.json new file mode 100644 index 00000000..88400f8d --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptExtCh_supported_only.json @@ -0,0 +1,10 @@ +[ + { + "eqptExtCh": { + "attributes": { + "dn": "topology/pod-1/node-101/sys/ch/supslot-1/extch-101", + "model": "N2K-C2348UPQ" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/eqptExtCh_with_unsupported.json b/tests/checks/supported_hardware_check/eqptExtCh_with_unsupported.json new file mode 100644 index 00000000..848f9070 --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptExtCh_with_unsupported.json @@ -0,0 +1,18 @@ +[ + { + "eqptExtCh": { + "attributes": { + "dn": "topology/pod-1/node-101/sys/ch/supslot-1/extch-101", + "model": "N2K-C2232PP-10GE" + } + } + }, + { + "eqptExtCh": { + "attributes": { + "dn": "topology/pod-1/node-101/sys/ch/supslot-1/extch-102", + "model": "N2K-C2348UPQ" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/eqptLC_supported_only.json b/tests/checks/supported_hardware_check/eqptLC_supported_only.json new file mode 100644 index 00000000..9d27af9a --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptLC_supported_only.json @@ -0,0 +1,10 @@ +[ + { + "eqptLC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/lcslot-1/lc", + "model": "N9K-X9732C-EX" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/eqptLC_with_unformatted_dn.json b/tests/checks/supported_hardware_check/eqptLC_with_unformatted_dn.json new file mode 100644 index 00000000..7520fe4e --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptLC_with_unformatted_dn.json @@ -0,0 +1,10 @@ +[ + { + "eqptLC": { + "attributes": { + "dn": "sys/ch/lcslot-1/lc", + "model": "N9K-M6PQ" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/eqptLC_with_unsupported.json b/tests/checks/supported_hardware_check/eqptLC_with_unsupported.json new file mode 100644 index 00000000..f3adc68b --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptLC_with_unsupported.json @@ -0,0 +1,18 @@ +[ + { + "eqptLC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/lcslot-1/lc", + "model": "N9K-M6PQ" + } + } + }, + { + "eqptLC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/lcslot-2/lc", + "model": "N9K-X9732C-EX" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/eqptSupC_supported_only.json b/tests/checks/supported_hardware_check/eqptSupC_supported_only.json new file mode 100644 index 00000000..c8d387a3 --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptSupC_supported_only.json @@ -0,0 +1,10 @@ +[ + { + "eqptSupC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/supslot-1/sup", + "model": "N9K-SUP-C" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/eqptSupC_with_unsupported.json b/tests/checks/supported_hardware_check/eqptSupC_with_unsupported.json new file mode 100644 index 00000000..da9f67aa --- /dev/null +++ b/tests/checks/supported_hardware_check/eqptSupC_with_unsupported.json @@ -0,0 +1,18 @@ +[ + { + "eqptSupC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/supslot-1/sup", + "model": "N9K-SUP-B" + } + } + }, + { + "eqptSupC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/supslot-2/sup", + "model": "N9K-SUP-C" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/fabricNode_supported_only.json b/tests/checks/supported_hardware_check/fabricNode_supported_only.json new file mode 100644 index 00000000..3b6cb1a5 --- /dev/null +++ b/tests/checks/supported_hardware_check/fabricNode_supported_only.json @@ -0,0 +1,20 @@ +[ + { + "fabricNode": { + "attributes": { + "id": "101", + "model": "N9K-C93180YC-FX", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "id": "1001", + "model": "N9K-C9504", + "role": "spine" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/fabricNode_with_unsupported_hardware.json b/tests/checks/supported_hardware_check/fabricNode_with_unsupported_hardware.json new file mode 100644 index 00000000..6345c323 --- /dev/null +++ b/tests/checks/supported_hardware_check/fabricNode_with_unsupported_hardware.json @@ -0,0 +1,38 @@ +[ + { + "fabricNode": { + "attributes": { + "id": "101", + "model": "N9K-C9372TX-E", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "id": "102", + "model": "N9K-C93180LC-EX", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "id": "103", + "model": "N9K-C93180YC-FX", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "id": "104", + "model": "N9K-C9332PQ", + "role": "leaf" + } + } + } +] diff --git a/tests/checks/supported_hardware_check/test_supported_hardware_check.py b/tests/checks/supported_hardware_check/test_supported_hardware_check.py new file mode 100644 index 00000000..ddf1452b --- /dev/null +++ b/tests/checks/supported_hardware_check/test_supported_hardware_check.py @@ -0,0 +1,133 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) +test_function = "supported_hardware_check" +# icurl queries +eqptLC = "eqptLC.json" +eqptExtCh = "eqptExtCh.json" +eqptSupC = "eqptSupC.json" + +@pytest.mark.parametrize( + "icurl_outputs, tversion, fabric_nodes, expected_result, expected_data, expected_unformatted_data", + [ + # FAIL - unsupported Gen1 and 6.1(1)+ deprecated hardware found + ( + { + eqptLC: read_data(dir, "eqptLC_with_unsupported.json"), + eqptExtCh: read_data(dir, "eqptExtCh_with_unsupported.json"), + eqptSupC: read_data(dir, "eqptSupC_with_unsupported.json"), + }, + "6.1(1f)", + read_data(dir, "fabricNode_with_unsupported_hardware.json"), + script.FAIL_UF, + [ + ["6.1(1f)", "101", "N9K-C9372TX-E", "Switch", "Not supported on 5.x+"], + ["6.1(1f)", "104", "N9K-C9332PQ", "Switch", "Not supported on 5.x+"], + ["6.1(1f)", "1001", "N9K-M6PQ", "Expansion Module", "Not supported on 5.x+"], + ["6.1(1f)", "102", "N9K-C93180LC-EX", "Switch", "Deprecated from 6.1(1)+"], + ["6.1(1f)", "101", "N2K-C2232PP-10GE", "FEX", "Deprecated from 6.1(1)+"], + ["6.1(1f)", "1001", "N9K-SUP-B", "Supervisor", "Deprecated from 6.1(1)+"], + ], + [], + ), + # PASS - no unsupported hardware found + ( + { + eqptLC: read_data(dir, "eqptLC_supported_only.json"), + eqptExtCh: read_data(dir, "eqptExtCh_supported_only.json"), + eqptSupC: read_data(dir, "eqptSupC_supported_only.json"), + }, + "6.1(1f)", + read_data(dir, "fabricNode_supported_only.json"), + script.PASS, + [], + [], + ), + # FAIL - pre 6.1(1f): only Gen1 hit should be reported + ( + { + eqptLC: read_data(dir, "eqptLC_supported_only.json"), + }, + "6.1(1a)", + read_data(dir, "fabricNode_with_unsupported_hardware.json"), + script.FAIL_UF, + [["6.1(1a)", "101", "N9K-C9372TX-E", "Switch", "Not supported on 5.x+"], + ["6.1(1a)", "104", "N9K-C9332PQ", "Switch", "Not supported on 5.x+"]], + [], + ), + # PASS - pre 5.x: unsupported hardware checks should not trigger + ( + { + eqptLC: read_data(dir, "eqptLC_with_unsupported.json"), + eqptExtCh: read_data(dir, "eqptExtCh_with_unsupported.json"), + eqptSupC: read_data(dir, "eqptSupC_with_unsupported.json"), + }, + "4.2(7r)", + read_data(dir, "fabricNode_with_unsupported_hardware.json"), + script.PASS, + [], + [], + ), + # FAIL - 6.0(1)+ unsupported switch model + ( + { + eqptLC: read_data(dir, "eqptLC_supported_only.json"), + }, + "6.0(1a)", + [ + { + "fabricNode": { + "attributes": { + "id": "201", + "model": "N9K-C93120TX", + } + } + } + ], + script.FAIL_UF, + [["6.0(1a)", "201", "N9K-C93120TX", "Switch", "Deprecated from 6.0(1)+"]], + [], + ), + # PASS - empty fabric nodes and supported inventory + ( + { + eqptLC: read_data(dir, "eqptLC_supported_only.json"), + eqptExtCh: read_data(dir, "eqptExtCh_supported_only.json"), + eqptSupC: read_data(dir, "eqptSupC_supported_only.json"), + }, + "6.1(1f)", + [], + script.PASS, + [], + [], + ), + # FAIL - expansion module with unformatted DN (no topology/pod-X/node-Y prefix); + # the entry should appear in unformatted_data with the raw DN, not in data + ( + { + eqptLC: read_data(dir, "eqptLC_with_unformatted_dn.json"), + }, + "5.0(1a)", + [], + script.FAIL_UF, + [], + [["5.0(1a)", "sys/ch/lcslot-1/lc", "N9K-M6PQ", "Expansion Module", "Not supported on 5.x+"]], + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, fabric_nodes, expected_result, expected_data, expected_unformatted_data): + result = run_check( + tversion=script.AciVersion(tversion) if tversion else None, + fabric_nodes=fabric_nodes, + ) + + assert result.result == expected_result + assert result.data == expected_data + assert result.unformatted_data == expected_unformatted_data From 0d28b8085006ef7ab3215be6704844ec2f2e16d3 Mon Sep 17 00:00:00 2001 From: Harinadh-Saladi Date: Thu, 21 May 2026 21:52:03 +0530 Subject: [PATCH 03/44] Added code for CSCwo96334:/tmp directory disk space for snapshot storage (#368) * Add arg to define thread limit - to throttle concurrent API calls when required (#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * Added code for CSCwo96334:/tmp directory disk space for snapshot storage * Updated fix version * Added clarification comment for /tmp snapshot handling * fix: Add cversion check for post_upgrade_cb_check (#377) * added validation for CFD CSCwp64296 (#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass * Corrected the typo for snapshot --------- Co-authored-by: GM Co-authored-by: Gabriel Co-authored-by: tkishida Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb Co-authored-by: sanjanch Co-authored-by: asraf-khan Co-authored-by: Thatleft Co-authored-by: Detlef Sass --- aci-preupgrade-validation-script.py | 28 ++++-- .../Fault_combination.json | 90 +++++++++++++++++++ .../Fault_exists_not_raised.json | 60 +++++++++++++ .../Fault_raised.json | 90 +++++++++++++++++++ .../test_apic_disk_space_faults_check.py | 72 ++++++++++++++- 5 files changed, 329 insertions(+), 11 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index a951bfe4..da1b4b9e 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3010,17 +3010,19 @@ def scalability_faults_check(**kwargs): @check_wrapper(check_title="APIC Disk Space Usage (F1527, F1528, F1529 equipment-full)") -def apic_disk_space_faults_check(cversion, **kwargs): +def apic_disk_space_faults_check(cversion, tversion, **kwargs): result = FAIL_UF headers = ['Fault', 'Pod', 'Node', 'Mount Point', 'Current Usage %', 'Recommended Action'] data = [] unformatted_headers = ['Fault', 'Fault DN', 'Recommended Action'] unformatted_data = [] doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-disk-space-usage" + # we are checking /tmp utilization because high usage can lead to snaphshot corruption during an upgrade. After the fix version, snapshot storage location moved to /data. recommended_action = { '/firmware': 'Remove unneeded images', '/techsupport': 'Remove unneeded techsupports/cores', - '/data/log': 'Remove unneeded logs in var/log/dme/log' + '/data/log': 'Remove unneeded logs in var/log/dme/log', + '/tmp': 'Remove unneeded logs in /tmp directory' } default_action = 'Contact Cisco TAC.' if cversion.same_as('4.0(1h)') or cversion.older_than('3.2(6i)'): @@ -3029,6 +3031,8 @@ def apic_disk_space_faults_check(cversion, **kwargs): dn_regex = node_regex + r'/.+p-\[(?P.+)\]-f' desc_regex = r'is (?P\d{2,3}%) full' + tmp_faults_skip_versions = ["6.0(9f)", "6.1(4h)", "6.2(1g)"] + tmp_faults_skipped = False # Track if we skip /tmp faults for CSCwo96334 versions faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F1527"),eq(faultInst.code,"F1528"),eq(faultInst.code,"F1529"))') for faultInst in faultInsts: @@ -3038,14 +3042,24 @@ def apic_disk_space_faults_check(cversion, **kwargs): fc = faultInst['faultInst']['attributes']['code'] dn = re.search(dn_regex, faultInst['faultInst']['attributes']['dn']) desc = re.search(desc_regex, faultInst['faultInst']['attributes']['descr']) - if dn and desc: - data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), + if dn: + mountpoint = dn.group('mountpoint') + # CSCwo96334: Skip /tmp faults when target is >= 6.1(4h) or any unaffected versions + if mountpoint == '/tmp' and (not tversion.older_than("6.1(4h)") or any(tversion.same_as(version) for version in tmp_faults_skip_versions)): + tmp_faults_skipped = True + continue + if desc: + data.append([fc, dn.group('pod'), dn.group('node'), dn.group('mountpoint'), desc.group('usage'), recommended_action.get(dn.group('mountpoint'), default_action)]) - else: - unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], default_action]) + else: + unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], default_action]) if not data and not unformatted_data: - result = PASS + # If we only found /tmp faults that were skipped (CSCwo96334 fixed target versions), return NA + if tmp_faults_skipped: + result = NA + else: + result = PASS return Result( result=result, headers=headers, diff --git a/tests/checks/apic_disk_space_faults_check/Fault_combination.json b/tests/checks/apic_disk_space_faults_check/Fault_combination.json index b6312cb9..7d1453eb 100644 --- a/tests/checks/apic_disk_space_faults_check/Fault_combination.json +++ b/tests/checks/apic_disk_space_faults_check/Fault_combination.json @@ -88,5 +88,95 @@ "type": "operational" } } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1529", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "critical", + "prevSeverity": "critical", + "rule": "eqpt-storage-full-critical", + "severity": "critical", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1528", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528", + "domain": "infra", + "highestSeverity": "major", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "major", + "prevSeverity": "major", + "rule": "eqpt-storage-full-major", + "severity": "major", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "available (Old: 1501496, New: 240908), capUtilized (Old: 79, New: 82), inodesFree (Old: 12148991, New: 12148990), inodesUsed (Old: 721, New: 722), used (Old: 595656, New: 1856244)", + "childAction": "", + "code": "F1527", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /data/log on Node 1 with hostname fab3-apic1 mounted at /data/log is 82% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/data/log]-f-[tmpfs]/fault-F1527", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-full-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } } ] \ No newline at end of file diff --git a/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json b/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json index 76ee8b5e..79b77fe8 100644 --- a/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json +++ b/tests/checks/apic_disk_space_faults_check/Fault_exists_not_raised.json @@ -87,5 +87,65 @@ "type": "operational" } } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1529", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "retaining", + "occur": "1", + "origSeverity": "critical", + "prevSeverity": "critical", + "rule": "eqpt-storage-full-critical", + "severity": "cleared", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1528", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528", + "domain": "infra", + "highestSeverity": "major", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "soaking-clearing", + "occur": "1", + "origSeverity": "major", + "prevSeverity": "major", + "rule": "eqpt-storage-full-major", + "severity": "major", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } } ] \ No newline at end of file diff --git a/tests/checks/apic_disk_space_faults_check/Fault_raised.json b/tests/checks/apic_disk_space_faults_check/Fault_raised.json index 37284baf..58747a2b 100644 --- a/tests/checks/apic_disk_space_faults_check/Fault_raised.json +++ b/tests/checks/apic_disk_space_faults_check/Fault_raised.json @@ -87,5 +87,95 @@ "type": "operational" } } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1529", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 100% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1529", + "domain": "infra", + "highestSeverity": "critical", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "critical", + "prevSeverity": "critical", + "rule": "eqpt-storage-full-critical", + "severity": "critical", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1528", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 89% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1528", + "domain": "infra", + "highestSeverity": "major", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "major", + "prevSeverity": "major", + "rule": "eqpt-storage-full-major", + "severity": "major", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } + }, + + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-full", + "changeSet": "inodesFree (Old: 12167881, New: 12167880), inodesUsed (Old: 64, New: 65)", + "childAction": "", + "code": "F1527", + "created": "2026-03-13T11:46:02.307+00:00", + "delegated": "no", + "descr": "Storage unit /tmp on node 1 with hostname fab3-apic1 mounted at /tmp is 82% full", + "dn": "topology/pod-1/node-1/sys/ch/p-[/tmp]-f-[tmpfs]/fault-F1527", + "domain": "infra", + "highestSeverity": "warning", + "lastTransition": "2026-03-13T11:48:02.867+00:00", + "lc": "raised", + "occur": "1", + "origSeverity": "warning", + "prevSeverity": "warning", + "rule": "eqpt-storage-full-warning", + "severity": "warning", + "status": "", + "subject": "equipment-full", + "title": "", + "type": "operational" + } + } } ] \ No newline at end of file diff --git a/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py b/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py index 40ff8b1f..a593ab4f 100644 --- a/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py +++ b/tests/checks/apic_disk_space_faults_check/test_apic_disk_space_faults_check.py @@ -16,30 +16,36 @@ @pytest.mark.parametrize( - "icurl_outputs, cversion, expected_result, expected_data", + "icurl_outputs, cversion, tversion, expected_result, expected_data", [ # PASS - No raised faults ( {faultInst: []}, "4.2(1h)", + "4.2(1h)", script.PASS, [], ), - # FAIL - Raised faults with /firmware,/techsupport,/data/log mount points + # FAIL - Raised faults with /firmware,/techsupport,/data/log, /tmp mount points ( {faultInst: read_data(dir, "Fault_raised.json")}, "4.2(1h)", + "4.2(1h)", script.FAIL_UF, [ ["F1528", "1", "1", "/data/log", "89%", "Remove unneeded logs in var/log/dme/log"], ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], ["F1528", "1", "1", "/techsupport", "89%", "Remove unneeded techsupports/cores"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/tmp", "82%", "Remove unneeded logs in /tmp directory"], ], ), # PASS - Faults exist but not raised nor soaking (cleared) ( {faultInst: read_data(dir, "Fault_exists_not_raised.json")}, "4.2(1h)", + "4.2(1h)", script.PASS, [], ), @@ -47,16 +53,69 @@ ( {faultInst: read_data(dir, "Fault_combination.json")}, "4.2(1h)", + "4.2(1h)", + script.FAIL_UF, + [ + ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], + ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], + ], + ), + # FAIL - /tmp included when tversion is below 6.1(4h) + ( + {faultInst: read_data(dir, "Fault_combination.json")}, + "4.2(1h)", + "6.1(2f)", + script.FAIL_UF, + [ + ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], + ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], + ], + ), + # FAIL - /tmp included when tversion is not one of CSCwo96334 fixed target versions + ( + {faultInst: read_data(dir, "Fault_combination.json")}, + "4.2(1h)", + "6.1(2g)", script.FAIL_UF, [ ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1529", "1", "1", "/tmp", "100%", "Remove unneeded logs in /tmp directory"], + ["F1528", "1", "1", "/tmp", "89%", "Remove unneeded logs in /tmp directory"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], ], ), + # FAIL - /tmp skipped when tversion is one of CSCwo96334 fixed target versions + ( + {faultInst: read_data(dir, "Fault_combination.json")}, + "4.2(1h)", + "6.0(9f)", + script.FAIL_UF, + [ + ["F1529", "1", "1", "/data/log", "94%", "Remove unneeded logs in var/log/dme/log"], + ["F1528", "1", "1", "/firmware", "89%", "Remove unneeded images"], + ["F1527", "1", "1", "/data/log", "82%", "Remove unneeded logs in var/log/dme/log"], + ], + ), + # NA - only /tmp faults and tversion is one of CSCwo96334 fixed target versions + ( + {faultInst: read_data(dir, "Fault_combination.json")[3:5]}, + "4.2(1h)", + "6.1(4h)", + script.NA, + [], + ), # FAIL - Raised faults with unknown mount point (unformatted data) ( {faultInst: read_data(dir, "Fault_unformatted_data.json")}, "4.2(1h)", + "4.2(1h)", script.FAIL_UF, [ ["F1528", "1", "1", "/unknown", "88%", "Contact Cisco TAC."], @@ -66,6 +125,7 @@ ( {faultInst: read_data(dir, "Fault_unformatted_data.json")}, "4.0(1h)", + "4.0(1h)", script.FAIL_UF, [ ["F1528", "1", "1", "/unknown", "88%", "Contact Cisco TAC. A typical issue is CSCvn13119."], @@ -75,12 +135,16 @@ ( {faultInst: []}, "4.0(1h)", + "4.0(1h)", script.PASS, [], ), ], ) -def test_logic(run_check, mock_icurl, cversion, expected_result, expected_data): - result = run_check(cversion=script.AciVersion(cversion)) +def test_logic(run_check, mock_icurl, cversion, tversion, expected_result, expected_data): + result = run_check( + cversion=script.AciVersion(cversion), + tversion=script.AciVersion(tversion), + ) assert result.result == expected_result assert result.data == expected_data From 70dc2d69b70af3a5bac2fcf90cd17bdc2bfadc16 Mon Sep 17 00:00:00 2001 From: muthu-ku Date: Thu, 11 Jun 2026 19:08:09 +0530 Subject: [PATCH 04/44] Added version check for CSCwd37387 (#378) * Add arg to define thread limit - to throttle concurrent API calls when required (#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * fix: Add cversion check for post_upgrade_cb_check (#377) * added validation for CFD CSCwp64296 (#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass * added version c heck for CSCwd37387 * updated document link * updated validation.md * updated validation.md document * Added validation for CSCwd40071 (#332) * Added validation for CSCwd40071 * Addressed the comments * Added cversion for the check * Removed empty spaces * logic change. removed 0.0.0.0/0 and made pytest changes * logic modified and validation.md file updated * Added validation for CSCws84232 (#334) * Feat:added svccoreCtrlr excessive entries check * updated the formated json in svccore_negative.json * Removed other PR checks * Fix: svccoreNode mo check is updated * Fix:update the threshold to 240 * refactor:moved to general check * refactor:updated recommended action * refactor:optimise the check for get the count alone * refactor:added try catch exception for error handling * update the threshold value for 240 * refactor:query-target was added * refactor: removed unwanted spaces * refactor:removed unwanted spaces in validation.md * resolve conflict while rebasing * resolve conflict while rebasing * updated pytest script file * updated validation.md * addressed review comments * updated validation.md --------- Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> * addressed review comments * removed additional whutespace --------- Co-authored-by: GM Co-authored-by: Gabriel Co-authored-by: tkishida Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb Co-authored-by: Harinadh-Saladi Co-authored-by: sanjanch Co-authored-by: asraf-khan Co-authored-by: Thatleft Co-authored-by: Detlef Sass Co-authored-by: veesenth_cisco --- aci-preupgrade-validation-script.py | 5 +++-- docs/docs/validations.md | 9 ++++---- .../test_llfc_susceptibility_check.py | 21 +++++++++++++++++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index da1b4b9e..488889a5 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3985,7 +3985,7 @@ def llfc_susceptibility_check(cversion, tversion, vpc_node_ids, **kwargs): data = [] sx_affected = t_affected = False recommended_action = 'Manually change Peer devices Transmit(send) Flow Control to off prior to switch Upgrade' - doc_url = 'https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#link-level-flow-control' if not tversion: return Result(result=MANUAL, msg=TVER_MISSING) @@ -3998,7 +3998,8 @@ def llfc_susceptibility_check(cversion, tversion, vpc_node_ids, **kwargs): sx_affected = True # Check for Copper 1000base-T, CSCvj67507 fixed by CSCwd37387 - if cversion.older_than("4.1(1i)") and tversion.newer_than("4.1(1h)") and tversion.older_than("5.2(7f)"): + if ((cversion.older_than("4.1(1i)") or cversion.same_as("4.1(1i)")) and tversion.older_than("5.2(7f)") and tversion.newer_than("4.0(1h")) or (cversion.older_than("5.2(7f)") and tversion.newer_than("5.2(7f)") + or (tversion.older_than("5.2(7f)") and cversion.same_as(tversion))): t_affected = True if sx_affected or t_affected: diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 82f22119..5080f338 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2406,11 +2406,9 @@ To avoid this issue, change the `collectorLocation` type to `none` through the A ### Link Level Flow Control -Due to the defect CSCvo27498, after upgrade of first ACI leaf switch in a VPC pair to newer 15.x version from older 13.x version, downstream VPC might be down due to `vpc port channel mis-config due to vpc links in the 2 switches connected to different partners` even though they are connected to same device. +Due to the defect CSCvo27498[70], after upgrade of first ACI leaf switch in a VPC pair to newer 15.2(7f) version, some of vpc member port on upgrading device may go down. By default Link level Flow control is off in ACI but in older code, the ACI software was incorrectly signalling far end device to enable transmit flow control. If far end device transmit(send) flow control in auto or desirable mode, it will enable transmit flow control. -By default Link level Flow control is off in ACI but in older code, the ACI software was incorrectly signalling far end device to enable transmit flow control. if far end device transmit(send) flow control in auto or desirable mode, it will enable transmit flow control. - -After the first switch in VPC pair is upgraded to newer 15.x code, the incorrect flow control signalling is fixed. But due to mismatched software versions in ACI during upgrade, the far end device port-channel member interfaces will end up with mismatched send flow control. When this happens. they could send a different LACP operational key causing the ACI leaf to interpret that it is connected to different partners. +After the first switch in VPC pair is upgraded to 15.2(7f) the incorrect flow control signalling is fixed. But due to mismatched software versions in ACI during upgrade, the far end device port-channel member interfaces will end up with mismatched send flow control. When this happens. they could send a different LACP operational key causing the ACI leaf to interpret that it is connected to different partners. The script checks if the version is susceptible to the default along with the specific 1G SFPs that are affected by the defect. @@ -2867,4 +2865,5 @@ This check will verify the count of the `svccoreCtrlr` Managed Object and raise [66]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwr66848 [67]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwh80837 [68]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwd40071 -[69]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws84232 \ No newline at end of file +[69]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws84232 +[70]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498 \ No newline at end of file diff --git a/tests/checks/llfc_susceptibility_check/test_llfc_susceptibility_check.py b/tests/checks/llfc_susceptibility_check/test_llfc_susceptibility_check.py index eb90593d..2bb046fb 100644 --- a/tests/checks/llfc_susceptibility_check/test_llfc_susceptibility_check.py +++ b/tests/checks/llfc_susceptibility_check/test_llfc_susceptibility_check.py @@ -18,6 +18,27 @@ @pytest.mark.parametrize( "icurl_outputs, cversion, tversion, vpc_node_ids, expected_result", [ + ( + {ethpmFcots: read_data(dir, "ethpmFcot.json")}, + "5.2(5c)", + "6.1(5e)", + ["101", "103", "204", "206"], + script.MANUAL, + ), + ( + {ethpmFcots: read_data(dir, "ethpmFcot.json")}, + "4.1(1i)", + "5.2(2a)", + ["101", "103", "204", "206"], + script.MANUAL, + ), + ( + {ethpmFcots: read_data(dir, "ethpmFcot.json")}, + "5.2(7f)", + "6.1(5c)", + ["101", "103", "204", "206"], + script.PASS, + ), ( {ethpmFcots: read_data(dir, "ethpmFcot.json")}, "4.2(1b)", From 0de713ce1089213b9f9f18c857938d549ea31564 Mon Sep 17 00:00:00 2001 From: dhaselva Date: Wed, 17 Jun 2026 19:05:27 +0530 Subject: [PATCH 05/44] Check for F0467 bgpProt-policy-already-existing. CSCwt78235 (#373) * Add arg to define thread limit - to throttle concurrent API calls when required (#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * Check for F0467 bgpProt-policy-already-existing * Incorporated the review comments * Addressed review comments * fix: Add cversion check for post_upgrade_cb_check (#377) * added validation for CFD CSCwp64296 (#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass * Added version check instead of generic check * Modified the target version check * Removed whitepsaces * Modified the content in validation file * Updated the pytest files * Added validation for CSCwd40071 (#332) * Added validation for CSCwd40071 * Addressed the comments * Added cversion for the check * Removed empty spaces * logic change. removed 0.0.0.0/0 and made pytest changes * logic modified and validation.md file updated * Added validation for CSCws84232 (#334) * Feat:added svccoreCtrlr excessive entries check * updated the formated json in svccore_negative.json * Removed other PR checks * Fix: svccoreNode mo check is updated * Fix:update the threshold to 240 * refactor:moved to general check * refactor:updated recommended action * refactor:optimise the check for get the count alone * refactor:added try catch exception for error handling * update the threshold value for 240 * refactor:query-target was added * refactor: removed unwanted spaces * refactor:removed unwanted spaces in validation.md * resolve conflict while rebasing * resolve conflict while rebasing * updated pytest script file * updated validation.md * addressed review comments * updated validation.md --------- Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> * modified the logic suggested by Gabe * Modified the bgp proto content * Removed unwanted whitespace * Removed the duplicate entry from line line 6555 * Added the tversion check * Addressed the review comments * added the cco image version in pytest file --------- Co-authored-by: GM Co-authored-by: Gabriel Co-authored-by: tkishida Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb Co-authored-by: Harinadh-Saladi Co-authored-by: sanjanch Co-authored-by: asraf-khan Co-authored-by: Thatleft Co-authored-by: Detlef Sass Co-authored-by: veesenth_cisco --- aci-preupgrade-validation-script.py | 51 +++++- docs/docs/validations.md | 12 +- .../faultDelegate_NEG.json | 1 + .../faultDelegate_POS.json | 20 ++ ...bgp_timer_policy_already_existing_check.py | 173 ++++++++++++++++++ 5 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_NEG.json create mode 100644 tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_POS.json create mode 100644 tests/checks/bgp_timer_policy_already_existing_check/test_bgp_timer_policy_already_existing_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 488889a5..e6ee2e48 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3898,6 +3898,9 @@ def supported_hardware_check(tversion, fabric_nodes, **kwargs): recommended_action = 'Select supported target version or upgrade hardware' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#supported-hardware-compatibility' + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + if not tversion.older_than("5.0(1a)"): for node in fabric_nodes: model = node['fabricNode']['attributes']['model'] @@ -6411,7 +6414,52 @@ def svccore_excessive_data_check(**kwargs): return Result(result=ERROR, msg="Error occurred while fetching svccore object counts: {}".format(str(e)), doc_url=doc_url) -# ---- Script Execution ---- +@check_wrapper(check_title='BGP Timer Policy Already Existing (F0467 bgpProt-policy-already-existing)') +def bgpProto_timer_policy_already_existing_check(tversion, cversion, **kwargs): + result = FAIL_O + headers = ['Fault', 'Tenant', 'L3Out', 'changeSet'] + data = [] + unformatted_headers = ['Fault', 'Affected', 'changeSet'] + unformatted_data = [] + recommended_action = 'Remove the fault by keeping Single bgp timer policy per vrf for different l3out.' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#bgpProto-timer-policy-already-existing' + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + cversion_check = (cversion.newer_than("6.2(1g)") or ( + cversion.major1 == "6" and cversion.major2 == "1" and cversion.newer_than("6.1(5e)")) and cversion.same_as(tversion)) + + tversion_check = (tversion.newer_than("6.2(1g)") or ( + tversion.major1 == "6" and tversion.major2 == "1" and tversion.newer_than("6.1(5e)"))) + + if cversion_check or tversion_check: + result=MANUAL + + affected_regex = r'uni/tn-(?P[^/]+)/out-(?P[^\]]+)' + filter = 'faultDelegate.json?query-target-filter=and(eq(faultDelegate.code,"F0467"),wcard(faultDelegate.changeSet,"bgpProt-policy-already-existing"))' + fault_delegates = icurl('class', filter) + + for fault_delegate in fault_delegates: + attributes = fault_delegate['faultDelegate']['attributes'] + fault_code = attributes.get('code', '') + affected = attributes.get('affected', '') + change_set = attributes.get('changeSet', '') + affected_array = re.search(affected_regex, affected) + if affected_array: + data.append([fault_code, affected_array.group('tenant'), affected_array.group('l3out'), change_set]) + else: + unformatted_data.append([fault_code, affected, change_set]) + + if not data and not unformatted_data: + result = PASS + elif result == MANUAL: + return Result(result=result, msg="Clear the fault code F0467 for bgp timer policy", headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) + + return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) + + +# ---- Script Execution ----. def parse_args(args): @@ -6582,6 +6630,7 @@ class CheckManager: rogue_ep_coop_exception_mac_check, n9k_c9408_model_lem_count_check, inband_management_policy_misconfig_check, + bgpProto_timer_policy_already_existing_check, ] ssh_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 5080f338..807c5abd 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -203,6 +203,7 @@ Items | Defect | This Script [N9K-C9408 with more than 5 N9K-X9400-16W LEMs][d31] | CSCws82819 | :white_check_mark: | :no_entry_sign: [Multi-Pod Modular Spine Bootscript File][d32] | CSCwr66848 | :white_check_mark: | :no_entry_sign: [Inband Management Policy Misconfiguration][d33]| CSCwd40071 | :white_check_mark: | :no_entry_sign: +[BgpProto timer policy already existing][d34] | CSCwt78235 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -237,6 +238,7 @@ Items | Defect | This Script [d31]: #n9k-c9408-with-more-than-5-n9k-x9400-16w-lems [d32]: #multi-pod-modular-spine-bootscript-file [d33]: #inband-management-policy-misconfiguration +[d34]: #bgpProto-timer-policy-already-existing ## General Check Details @@ -2790,12 +2792,17 @@ Due to excessive `svccoreCtrlr` or `svccoreNode` managed objects, Apic gui stuck The svccoreCtrlr and svccoreNode objects represent core files related to Apic and Leaf/Spines process respectively. -Due to [CSCws84232][67], the APIC GUI may become unresponsive after login, with dashboards stuck in a continuous “Loading…”state. +Due to [CSCws84232][69], the APIC GUI may become unresponsive after login, with dashboards stuck in a continuous “Loading…”state. Administrators may be unable to access or operate the APIC GUI, potentially impacting day-to-day management or upgrade. This check will verify the count of the `svccoreCtrlr` Managed Object and raise and alarm with the bug if object count found more than 240. Remove the content or objects of `svccoreCtrlr` or `svccoreNode`. Contact Cisco TAC or upgrade to a release containing the fix for CSCws84232 before proceeding with an upgrade. +### BgpProto Timer Policy Already Existing + +This bug [CSCwt78235][71] validates `F0467` faults where `changeSet` contains 'bgpProt-policy-already-existing'. The fault indicates conflicting BGP protocol timer policy under an L3Outs deployed in same vrf under same node. If this fault is not resolved, l3out will not be programmed properly in the leaf after the clean reboot or the upgrade. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2866,4 +2873,5 @@ This check will verify the count of the `svccoreCtrlr` Managed Object and raise [67]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwh80837 [68]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwd40071 [69]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws84232 -[70]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498 \ No newline at end of file +[70]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498 +[71]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt78235 diff --git a/tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_NEG.json b/tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_NEG.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_NEG.json @@ -0,0 +1 @@ +[] diff --git a/tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_POS.json b/tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_POS.json new file mode 100644 index 00000000..3aa62e08 --- /dev/null +++ b/tests/checks/bgp_timer_policy_already_existing_check/faultDelegate_POS.json @@ -0,0 +1,20 @@ +[ + { + "faultDelegate": { + "attributes": { + "affected": "resPolCont/rtdOutCont/rtdOutDef-[uni/tn-common/out-L3outY]/nwissues", + "code": "F0467", + "changeSet": "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no" + } + } + }, + { + "faultDelegate": { + "attributes": { + "affected": "resPolCont/rtdOutCont/rtdOutDef-[uni/tn-prod/out-L3outA]/nwissues", + "code": "F0467", + "changeSet": "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no" + } + } + } +] diff --git a/tests/checks/bgp_timer_policy_already_existing_check/test_bgp_timer_policy_already_existing_check.py b/tests/checks/bgp_timer_policy_already_existing_check/test_bgp_timer_policy_already_existing_check.py new file mode 100644 index 00000000..ee6997f8 --- /dev/null +++ b/tests/checks/bgp_timer_policy_already_existing_check/test_bgp_timer_policy_already_existing_check.py @@ -0,0 +1,173 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) +test_function = "bgpProto_timer_policy_already_existing_check" +# icurl queries +faultDelegates = 'faultDelegate.json?query-target-filter=and(eq(faultDelegate.code,"F0467"),wcard(faultDelegate.changeSet,"bgpProt-policy-already-existing"))' + +@pytest.mark.parametrize( + "icurl_outputs, tversion, cversion, expected_result, expected_data, expected_msg", + [ + # target release beyond affected range with existing fault: manual clearance required + ( + {faultDelegates: read_data(dir, "faultDelegate_POS.json")}, + "6.2(2e)", + "6.1(1f)", + script.MANUAL, + [ + [ + "F0467", + "common", + "L3outY", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + [ + "F0467", + "prod", + "L3outA", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + ], + "Clear the fault code F0467 for bgp timer policy", + ), + # target release beyond affected range on 6.1 train with existing fault: manual clearance required + ( + {faultDelegates: read_data(dir, "faultDelegate_POS.json")}, + "6.1(6a)", + "6.1(1f)", + script.MANUAL, + [ + [ + "F0467", + "common", + "L3outY", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + [ + "F0467", + "prod", + "L3outA", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + ], + "Clear the fault code F0467 for bgp timer policy", + ), + # boundary version is still affected for strict newer_than check + ( + {faultDelegates: read_data(dir, "faultDelegate_POS.json")}, + "6.2(1g)", + "6.1(1f)", + script.FAIL_O, + [ + [ + "F0467", + "common", + "L3outY", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + [ + "F0467", + "prod", + "L3outA", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + ], + None, + ), + # 6.1 boundary version is still affected for strict newer_than check + ( + {faultDelegates: read_data(dir, "faultDelegate_POS.json")}, + "6.1(5e)", + "6.1(1f)", + script.FAIL_O, + [ + [ + "F0467", + "common", + "L3outY", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + [ + "F0467", + "prod", + "L3outA", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + ], + None, + ), + # target release affected on 6.1 train (< 6.1(5e)) + ( + {faultDelegates: read_data(dir, "faultDelegate_POS.json")}, + "6.1(4h)", + "6.1(1f)", + script.FAIL_O, + [ + [ + "F0467", + "common", + "L3outY", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + [ + "F0467", + "prod", + "L3outA", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + ], + None, + ), + # current and target versions both beyond affected range: manual clearance required + ( + {faultDelegates: read_data(dir, "faultDelegate_POS.json")}, + "6.2(2e)", + "6.2(2e)", + script.MANUAL, + [ + [ + "F0467", + "common", + "L3outY", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + [ + "F0467", + "prod", + "L3outA", + "configQual:bgpProt-policy-already-existing, configSt:failed-to-apply, temporaryError:no", + ], + ], + "Clear the fault code F0467 for bgp timer policy", + ), + ( + {faultDelegates: read_data(dir, "faultDelegate_NEG.json")}, + "6.1(5e)", + "6.1(1f)", + script.PASS, + [], + None, + ), + # target release beyond affected range with no fault entries + ( + {faultDelegates: read_data(dir, "faultDelegate_NEG.json")}, + "6.2(2e)", + "6.1(1f)", + script.PASS, + [], + None, + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, cversion, expected_result, expected_data, expected_msg): + result = run_check(tversion=script.AciVersion(tversion), cversion=script.AciVersion(cversion)) + assert result.result == expected_result + assert result.data == expected_data + if expected_msg is not None: + assert result.msg == expected_msg \ No newline at end of file From 5b547128952a32c224f1428bfc29348f068648d1 Mon Sep 17 00:00:00 2001 From: psureshb Date: Wed, 17 Jun 2026 19:57:38 +0530 Subject: [PATCH 06/44] Update inband management policy misconfiguration check to include IPv6 address and gateway validation. Adjusted query filter in the validation script and updated test cases to reflect changes in address configurations. (#395) --- aci-preupgrade-validation-script.py | 2 +- .../mgmtRsInBStNode_invalid_addr_and_gw_config.json | 2 ++ .../mgmtRsInBStNode_invalid_address_config.json | 2 ++ .../mgmtRsInBStNode_invalid_gateway_config.json | 2 ++ .../test_inband_management_policy_misconfig_check.py | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index e6ee2e48..ae196700 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6376,7 +6376,7 @@ def inband_management_policy_misconfig_check(cversion, tversion, **kwargs): doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#inband-management-policy-misconfiguration" if (cversion.older_than("5.2(8d)")) and (tversion.newer_than("6.0(4c)") or tversion.same_as("6.0(4c)")): - mgmtRsInBStNodes = icurl('class', 'mgmtRsInBStNode.json?query-target-filter=or(eq(mgmtRsInBStNode.addr,"0.0.0.0"),eq(mgmtRsInBStNode.gw,"0.0.0.0"))') + mgmtRsInBStNodes = icurl('class', 'mgmtRsInBStNode.json?query-target-filter=and(or(eq(mgmtRsInBStNode.addr,"0.0.0.0"),eq(mgmtRsInBStNode.gw,"0.0.0.0")),or(eq(mgmtRsInBStNode.v6Addr,"::"),eq(mgmtRsInBStNode.v6Gw,"::")))') for mgmtRsInBStNode in mgmtRsInBStNodes: attrs = mgmtRsInBStNode["mgmtRsInBStNode"]["attributes"] addr = attrs['addr'] diff --git a/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_addr_and_gw_config.json b/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_addr_and_gw_config.json index 565e715b..9960fcf4 100644 --- a/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_addr_and_gw_config.json +++ b/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_addr_and_gw_config.json @@ -7,6 +7,8 @@ "configurationMode": "static", "dn": "uni/tn-mgmt/mgmtp-default/inb-inb/rsinBStNode-[topology/pod-1/node-103]", "gw": "0.0.0.0", + "v6Addr": "::", + "v6Gw": "::", "modTs": "2024-12-20T07:45:21.454+00:00", "rType": "mo", "rn": "rsinBStNode-[topology/pod-1/node-103]", diff --git a/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_address_config.json b/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_address_config.json index ee47c09e..3ee8a500 100644 --- a/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_address_config.json +++ b/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_address_config.json @@ -7,6 +7,8 @@ "configurationMode": "static", "dn": "uni/tn-mgmt/mgmtp-default/inb-inb/rsinBStNode-[topology/pod-1/node-103]", "gw": "191.1.1.1", + "v6Addr": "::", + "v6Gw": "::", "modTs": "2024-12-20T07:45:21.454+00:00", "rType": "mo", "rn": "rsinBStNode-[topology/pod-1/node-103]", diff --git a/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_gateway_config.json b/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_gateway_config.json index 9fc4c0c7..6e8cb053 100644 --- a/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_gateway_config.json +++ b/tests/checks/inband_management_policy_misconfig_check/mgmtRsInBStNode_invalid_gateway_config.json @@ -7,6 +7,8 @@ "configurationMode": "static", "dn": "uni/tn-mgmt/mgmtp-default/inb-inb/rsinBStNode-[topology/pod-1/node-103]", "gw": "0.0.0.0", + "v6Addr": "::", + "v6Gw": "::", "modTs": "2024-12-20T07:45:21.454+00:00", "rType": "mo", "rn": "rsinBStNode-[topology/pod-1/node-103]", diff --git a/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py b/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py index 29739aab..656cb4e4 100644 --- a/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py +++ b/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py @@ -8,7 +8,7 @@ log = logging.getLogger(__name__) dir = os.path.dirname(os.path.abspath(__file__)) test_function = "inband_management_policy_misconfig_check" -mgmtRsInBStNode = 'mgmtRsInBStNode.json?query-target-filter=or(eq(mgmtRsInBStNode.addr,"0.0.0.0"),eq(mgmtRsInBStNode.gw,"0.0.0.0"))' +mgmtRsInBStNode = 'mgmtRsInBStNode.json?query-target-filter=and(or(eq(mgmtRsInBStNode.addr,"0.0.0.0"),eq(mgmtRsInBStNode.gw,"0.0.0.0")),or(eq(mgmtRsInBStNode.v6Addr,"::"),eq(mgmtRsInBStNode.v6Gw,"::")))' @pytest.mark.parametrize( "icurl_outputs, cversion, tversion, expected_result, expected_data", From 5fad901194492b8088457c834866bee8b2c06bc2 Mon Sep 17 00:00:00 2001 From: Priyanka-Patil14 Date: Wed, 24 Jun 2026 23:37:11 +0530 Subject: [PATCH 07/44] Added WRED with affected Leaf/LC/FM model check (#379) * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * print cleanup * Added WRED with affected Leaf/LC/FM model check * Addressed PR review comments * Addressed PR review comments * Addressed PR review comments and updated the docs * Upated the message * Addressed review comments * Refactored result variable in wred_affected_model_check * Updated wred_affected_model_check * Addressed PR review comments * Remove the sort method * Addressed PR review comments * Fix non-ASCII char in test comment * Fixed duplicate entries * Address PR review comments --------- Co-authored-by: GM Co-authored-by: Gabriel --- aci-preupgrade-validation-script.py | 56 +++++++- docs/docs/validations.md | 15 +++ .../eqptFC_affected.json | 10 ++ .../eqptFC_duplicate.json | 18 +++ .../eqptFC_empty.json | 1 + .../eqptFC_mixed.json | 18 +++ .../fabricNode_spine.json | 13 ++ .../qosCong_mixed.json | 16 +++ .../qosCong_tail_drop.json | 9 ++ .../qosCong_wred.json | 9 ++ .../test_wred_affected_model_check.py | 121 ++++++++++++++++++ 11 files changed, 285 insertions(+), 1 deletion(-) create mode 100644 tests/checks/wred_affected_model_check/eqptFC_affected.json create mode 100644 tests/checks/wred_affected_model_check/eqptFC_duplicate.json create mode 100644 tests/checks/wred_affected_model_check/eqptFC_empty.json create mode 100644 tests/checks/wred_affected_model_check/eqptFC_mixed.json create mode 100644 tests/checks/wred_affected_model_check/fabricNode_spine.json create mode 100644 tests/checks/wred_affected_model_check/qosCong_mixed.json create mode 100644 tests/checks/wred_affected_model_check/qosCong_tail_drop.json create mode 100644 tests/checks/wred_affected_model_check/qosCong_wred.json create mode 100644 tests/checks/wred_affected_model_check/test_wred_affected_model_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index ae196700..78110c13 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6459,7 +6459,60 @@ def bgpProto_timer_policy_already_existing_check(tversion, cversion, **kwargs): return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) -# ---- Script Execution ----. +@check_wrapper(check_title="WRED with Affected FM Models") +def wred_affected_model_check(tversion, fabric_nodes, **kwargs): + result = PASS + headers = ["Node ID", "Node Name", "Model"] + data = [] + recommended_action = "Disable WRED in fabric or upgrade to a release newer than 6.1(5e) or 6.2(2d)." + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#wred-with-affected-fm-models" + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + version_affected = ( + (tversion.major1 == "6" and tversion.major2 == "1" and (tversion.older_than("6.1(5e)") or tversion.same_as("6.1(5e)"))) + or (tversion.major1 == "6" and tversion.major2 == "2" and tversion.older_than("6.2(2e)")) + ) + if not version_affected: + return Result(result=NA, msg=VER_NOT_AFFECTED) + + affected_models = {"N9K-C9504-FM-E", "N9K-C9508-FM-E", "N9K-C9516-FM-E"} + + node_name_map = { + node["fabricNode"]["attributes"]["id"]: node["fabricNode"]["attributes"]["name"] + for node in fabric_nodes + } + + for cong in icurl("class", "qosCong.json"): + if cong.get("qosCong", {}).get("attributes", {}).get("algo") == "wred": + break + else: + return Result(result=PASS, msg="WRED not enabled.") + + unique_list = {} + for obj in icurl("class", "eqptFC.json"): + attr = obj["eqptFC"]["attributes"] + model = attr.get("model", "") + if model not in affected_models: + continue + dn = attr.get("dn", "") + if not dn.startswith("topology/"): + continue + dn_match = re.search(node_regex, dn) + if not dn_match: + continue + node_id = dn_match.group("node") + unique_list[(node_id, model)] = [node_id, node_name_map.get(node_id, ""), model] + data = list(unique_list.values()) + + if data: + return Result(result=FAIL_O, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + return Result(result=NA, msg="No affected Fabric module found.") + + +# ---- Script Execution ---- def parse_args(args): @@ -6631,6 +6684,7 @@ class CheckManager: n9k_c9408_model_lem_count_check, inband_management_policy_misconfig_check, bgpProto_timer_policy_already_existing_check, + wred_affected_model_check, ] ssh_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 807c5abd..bd0de6d7 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -204,6 +204,7 @@ Items | Defect | This Script [Multi-Pod Modular Spine Bootscript File][d32] | CSCwr66848 | :white_check_mark: | :no_entry_sign: [Inband Management Policy Misconfiguration][d33]| CSCwd40071 | :white_check_mark: | :no_entry_sign: [BgpProto timer policy already existing][d34] | CSCwt78235 | :white_check_mark: | :no_entry_sign: +[WRED with Affected FM Models][d35] | CSCwt50713 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -239,6 +240,7 @@ Items | Defect | This Script [d32]: #multi-pod-modular-spine-bootscript-file [d33]: #inband-management-policy-misconfiguration [d34]: #bgpProto-timer-policy-already-existing +[d35]: #wred-with-affected-fm-models ## General Check Details @@ -2798,6 +2800,18 @@ Administrators may be unable to access or operate the APIC GUI, potentially impa This check will verify the count of the `svccoreCtrlr` Managed Object and raise and alarm with the bug if object count found more than 240. Remove the content or objects of `svccoreCtrlr` or `svccoreNode`. Contact Cisco TAC or upgrade to a release containing the fix for CSCws84232 before proceeding with an upgrade. +### WRED with Affected FM Models + +Due to [CSCwt50713][72], when WRED (Weighted Random Early Detection) is enabled and specific Fabric Module (FM) hardware models are present in the fabric, the spine switch may crash after moving to an affected ACI release in the 6.1(x) or 6.2(x) range. The crash is specifically triggered by running a tech-support collection or QoS-related commands on the affected spine. + +Affected versions: +version <= 6.1(5e) or version < 6.2(2e). + +Affected hardware models: N9K-C9504-FM-E, N9K-C9508-FM-E, N9K-C9516-FM-E. + +To avoid this issue, disable WRED on the affected nodes or upgrade to a release newer than 6.1(5e) in the 6.1(x) train or 6.2(2e) or later in the 6.2(x) train. + + ### BgpProto Timer Policy Already Existing This bug [CSCwt78235][71] validates `F0467` faults where `changeSet` contains 'bgpProt-policy-already-existing'. The fault indicates conflicting BGP protocol timer policy under an L3Outs deployed in same vrf under same node. If this fault is not resolved, l3out will not be programmed properly in the leaf after the clean reboot or the upgrade. @@ -2875,3 +2889,4 @@ This bug [CSCwt78235][71] validates `F0467` faults where `changeSet` contains 'b [69]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCws84232 [70]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498 [71]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt78235 +[72]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt50713 diff --git a/tests/checks/wred_affected_model_check/eqptFC_affected.json b/tests/checks/wred_affected_model_check/eqptFC_affected.json new file mode 100644 index 00000000..4e87550d --- /dev/null +++ b/tests/checks/wred_affected_model_check/eqptFC_affected.json @@ -0,0 +1,10 @@ +[ + { + "eqptFC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/fcslot-1/fc", + "model": "N9K-C9508-FM-E" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/eqptFC_duplicate.json b/tests/checks/wred_affected_model_check/eqptFC_duplicate.json new file mode 100644 index 00000000..d1210502 --- /dev/null +++ b/tests/checks/wred_affected_model_check/eqptFC_duplicate.json @@ -0,0 +1,18 @@ +[ + { + "eqptFC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/fcslot-2/fc", + "model": "N9K-C9508-FM-E" + } + } + }, + { + "eqptFC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/fcslot-6/fc", + "model": "N9K-C9508-FM-E" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/eqptFC_empty.json b/tests/checks/wred_affected_model_check/eqptFC_empty.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/tests/checks/wred_affected_model_check/eqptFC_empty.json @@ -0,0 +1 @@ +[] diff --git a/tests/checks/wred_affected_model_check/eqptFC_mixed.json b/tests/checks/wred_affected_model_check/eqptFC_mixed.json new file mode 100644 index 00000000..0e346b00 --- /dev/null +++ b/tests/checks/wred_affected_model_check/eqptFC_mixed.json @@ -0,0 +1,18 @@ +[ + { + "eqptFC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/fcslot-1/fc", + "model": "N9K-C9508-FM-E" + } + } + }, + { + "eqptFC": { + "attributes": { + "dn": "topology/pod-1/node-1001/sys/ch/fcslot-2/fc", + "model": "N9K-C9504-FM-G" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/fabricNode_spine.json b/tests/checks/wred_affected_model_check/fabricNode_spine.json new file mode 100644 index 00000000..c70eb8f8 --- /dev/null +++ b/tests/checks/wred_affected_model_check/fabricNode_spine.json @@ -0,0 +1,13 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-1001", + "id": "1001", + "name": "spine1001", + "role": "spine", + "model": "N9K-C9504" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/qosCong_mixed.json b/tests/checks/wred_affected_model_check/qosCong_mixed.json new file mode 100644 index 00000000..cb3a9003 --- /dev/null +++ b/tests/checks/wred_affected_model_check/qosCong_mixed.json @@ -0,0 +1,16 @@ +[ + { + "qosCong": { + "attributes": { + "algo": "tail-drop" + } + } + }, + { + "qosCong": { + "attributes": { + "algo": "wred" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/qosCong_tail_drop.json b/tests/checks/wred_affected_model_check/qosCong_tail_drop.json new file mode 100644 index 00000000..dd814ec9 --- /dev/null +++ b/tests/checks/wred_affected_model_check/qosCong_tail_drop.json @@ -0,0 +1,9 @@ +[ + { + "qosCong": { + "attributes": { + "algo": "tail-drop" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/qosCong_wred.json b/tests/checks/wred_affected_model_check/qosCong_wred.json new file mode 100644 index 00000000..1c297c85 --- /dev/null +++ b/tests/checks/wred_affected_model_check/qosCong_wred.json @@ -0,0 +1,9 @@ +[ + { + "qosCong": { + "attributes": { + "algo": "wred" + } + } + } +] diff --git a/tests/checks/wred_affected_model_check/test_wred_affected_model_check.py b/tests/checks/wred_affected_model_check/test_wred_affected_model_check.py new file mode 100644 index 00000000..f3610a93 --- /dev/null +++ b/tests/checks/wred_affected_model_check/test_wred_affected_model_check.py @@ -0,0 +1,121 @@ +import os +import pytest +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "wred_affected_model_check" + +# icurl queries +qosCong_api = "qosCong.json" +eqptFC_api = "eqptFC.json" + + +@pytest.mark.parametrize( + "tversion, fabric_nodes, icurl_outputs, expected_result, expected_data", + [ + # Case 1: Target version not supplied. Expected: MANUAL. + ( + None, + read_data(dir, "fabricNode_spine.json"), + {}, + script.MANUAL, + [], + ), + # Case 2: Target version 6.2(2e) is the first fixed release and not in the affected range. + # Version gate fails. Expected: NA without any API calls. + ( + "6.2(2e)", + read_data(dir, "fabricNode_spine.json"), + {}, + script.NA, + [], + ), + # Case 2: All 3 gates triggered via an affected FM on a spine node. + # Version 6.2(1g) is in affected range, WRED is enabled, FM model N9K-C9508-FM-E is affected. + # Expected: FAIL_O with node 1001 reported. + ( + "6.2(1g)", + read_data(dir, "fabricNode_spine.json"), + { + qosCong_api: read_data(dir, "qosCong_wred.json"), + eqptFC_api: read_data(dir, "eqptFC_affected.json"), + }, + script.FAIL_O, + [["1001", "spine1001", "N9K-C9508-FM-E"]], + ), + # Case 3: Version is affected but no affected FM hardware found. + # WRED is enabled so the script proceeds to the FM check, which finds nothing. + # Hardware gate fails. Expected: NA - issue is model-specific. + ( + "6.1(5e)", + read_data(dir, "fabricNode_spine.json"), + { + qosCong_api: read_data(dir, "qosCong_wred.json"), + eqptFC_api: read_data(dir, "eqptFC_empty.json"), + }, + script.NA, + [], + ), + # Case 4: Version is affected and FM is affected, but WRED is not enabled (tail-drop). + # WRED gate fails. Expected: PASS - confirms all 3 gates must be true simultaneously. + ( + "6.1(5e)", + read_data(dir, "fabricNode_spine.json"), + { + qosCong_api: read_data(dir, "qosCong_tail_drop.json"), + eqptFC_api: read_data(dir, "eqptFC_affected.json"), + }, + script.PASS, + [], + ), + # Case 5: Multiple FM objects - one affected (N9K-C9508-FM-E), one unaffected (N9K-C9504-FM-G). + # WRED is enabled. Only the affected FM should be reported. + # Expected: FAIL_O with only the affected FM row reported. + ( + "6.1(5e)", + read_data(dir, "fabricNode_spine.json"), + { + qosCong_api: read_data(dir, "qosCong_wred.json"), + eqptFC_api: read_data(dir, "eqptFC_mixed.json"), + }, + script.FAIL_O, + [["1001", "spine1001", "N9K-C9508-FM-E"]], + ), + # Case 6: Version is affected, WRED is enabled, but no affected FM models found. + # FM gate fails. Expected: NA. + ( + "6.2(1g)", + read_data(dir, "fabricNode_spine.json"), + { + qosCong_api: read_data(dir, "qosCong_wred.json"), + eqptFC_api: read_data(dir, "eqptFC_empty.json"), + }, + script.NA, + [], + ), + # Case 7: Same node has two FM slots with the same affected model (duplicate eqptFC objects). + # Deduplication by (node_id, model) must result in only one row. + # Expected: FAIL_O with a single row for node 1001. + ( + "6.2(1g)", + read_data(dir, "fabricNode_spine.json"), + { + qosCong_api: read_data(dir, "qosCong_wred.json"), + eqptFC_api: read_data(dir, "eqptFC_duplicate.json"), + }, + script.FAIL_O, + [["1001", "spine1001", "N9K-C9508-FM-E"]], + ), + ], +) +def test_logic(run_check, mock_icurl, tversion, fabric_nodes, expected_result, expected_data): + result = run_check( + tversion=script.AciVersion(tversion) if tversion else None, + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result + assert result.data == expected_data From dbdfe2fd4104e870ce1905e1521041f22caec90b Mon Sep 17 00:00:00 2001 From: muthu-ku Date: Tue, 30 Jun 2026 18:17:24 +0530 Subject: [PATCH 08/44] Added validation check for CSCwo74485 (#396) * added validation check for CSCwo74485 * updated warning message * updated script * updated the warning message * updated doc link * updated the comments --- aci-preupgrade-validation-script.py | 22 +++++++- docs/docs/validations.md | 3 + .../compatRsSuppHw_605_M4L4.json | 18 ++++++ .../compatRsSuppHw_615_M4L4.json | 18 ++++++ .../eqptCh_m4l4_mixed_models.json | 55 +++++++++++++++++++ .../eqptCh_m4l4_model_new_cimc.json | 29 ++++++++++ .../eqptCh_m4l4_model_old_cimc.json | 29 ++++++++++ .../test_cimc_compatibilty_check.py | 53 +++++++++++++++++- 8 files changed, 223 insertions(+), 4 deletions(-) create mode 100644 tests/checks/cimc_compatibilty_check/compatRsSuppHw_605_M4L4.json create mode 100644 tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M4L4.json create mode 100644 tests/checks/cimc_compatibilty_check/eqptCh_m4l4_mixed_models.json create mode 100644 tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_new_cimc.json create mode 100644 tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_old_cimc.json diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 78110c13..a3ca8539 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3655,13 +3655,15 @@ def vpc_paired_switches_check(vpc_node_ids, fabric_nodes, **kwargs): @check_wrapper(check_title="APIC CIMC Compatibility") -def cimc_compatibilty_check(tversion, **kwargs): +def cimc_compatibilty_check(tversion, cversion, **kwargs): result = FAIL_UF headers = ["Node ID", "Model", "Current CIMC version", "Catalog Recommended CIMC Version", "Warning"] data = [] recommended_action = 'Check Release note of APIC Model/version for latest recommendations.' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#compatibility-cimc-version' + m4l4_model_affected_version_found = False + apic_obj = icurl('class', 'eqptCh.json?query-target-filter=wcard(eqptCh.descr,"APIC")') if apic_obj and tversion: try: @@ -3670,6 +3672,21 @@ def cimc_compatibilty_check(tversion, **kwargs): apic_model = eqptCh['eqptCh']['attributes']['descr'] model = "apic" + apic_model.split('-')[2].lower() current_cimc = eqptCh['eqptCh']['attributes']['cimcVersion'] + + #defect CSCwo74485 cimc compatibility check for M4/L4 model. + if model in ("apicm4", "apicl4") and cversion: + is_affected_apic_version = ( + (cversion.major1 == "5" and cversion.major2 == "3") + or (cversion.major1 == "6" and cversion.major2 == "0" and cversion.older_than("6.0(9e)")) + or (cversion.major1 == "6" and cversion.major2 == "1" and cversion.older_than("6.1(4h)")) + ) + if is_affected_apic_version: + if not is_firstver_gt_secondver(current_cimc, "4.3(5)"): + m4l4_model_affected_version_found = True + nodeid = eqptCh['eqptCh']['attributes']['dn'].split('/')[2] + data.append([nodeid, apic_model, current_cimc, "-", "-"]) + continue + compat_lookup_dn = "uni/fabric/compcat-default/ctlrfw-apic-" + tversion.simple_version + \ "/rssuppHw-[uni/fabric/compcat-default/ctlrhw-" + model + "].json" compatMo = icurl('mo', compat_lookup_dn) @@ -3688,6 +3705,9 @@ def cimc_compatibilty_check(tversion, **kwargs): if not data: result = PASS + if m4l4_model_affected_version_found: + recommended_action = 'Intentionally Upgrade your APICs to a fixed target version [6.0(9e)+ or (6.1(4h)+] BEFORE upgrading CIMC to avoid hitting CSCwo74485.' + except KeyError: return Result(result=MANUAL, msg="eqptCh does not have cimcVersion parameter on this version", headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) else: diff --git a/docs/docs/validations.md b/docs/docs/validations.md index bd0de6d7..76e9c140 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -257,6 +257,8 @@ The script checks the minimum recommended CIMC version for the given APIC model As the `compatRsSuppHw` object recommendation is strictly tied to the target software image, it is possible that the [Release Note Documentation][4] for your model/target version has a different recommendation than what the software recommends. Always check the release note of your Target version and APIC model to ensure you are getting the latest recommendations. +Due to the defect CSCwo74485, APIC-SERVER-M4/L4 systems will fail to boot correctly after upgrading CIMC firmware to version 4.3.5 or later while on Non-fixed APIC releases 5.3.x/6.0.9d/6.1(3g) and below. Upgrade the APIC software first, then proceed with the CIMC upgrade for the releases 6.0.9e/ 6.1.4h and above, will avoid this issue. Follow the software advisory for this defect [CSCwo74485][73]. + !!! note Older versions of CIMC may required multi-step CIMC upgrades to get to the identified target version. Refer to the [Cisco UCS Rack Server Upgrade Matrix][22] for the latest documentation on which steps are required and support given your current and target CIMC versions. @@ -2890,3 +2892,4 @@ This bug [CSCwt78235][71] validates `F0467` faults where `changeSet` contains 'b [70]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498 [71]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt78235 [72]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt50713 +[73]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwo74485 \ No newline at end of file diff --git a/tests/checks/cimc_compatibilty_check/compatRsSuppHw_605_M4L4.json b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_605_M4L4.json new file mode 100644 index 00000000..a05bdec7 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_605_M4L4.json @@ -0,0 +1,18 @@ +[ + { + "compatRsSuppHw": { + "attributes": { + "cimcVersion": "4.0(2g)", + "dn": "uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl4]" + } + } + }, + { + "compatRsSuppHw": { + "attributes": { + "cimcVersion": "4.0(2g)", + "dn": "uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4]" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M4L4.json b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M4L4.json new file mode 100644 index 00000000..d8c7b772 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M4L4.json @@ -0,0 +1,18 @@ +[ + { + "compatRsSuppHw": { + "attributes": { + "cimcVersion": "4.0(2g)", + "dn": "uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl4]" + } + } + }, + { + "compatRsSuppHw": { + "attributes": { + "cimcVersion": "4.0(2g)", + "dn": "uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4]" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_mixed_models.json b/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_mixed_models.json new file mode 100644 index 00000000..24345c60 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_mixed_models.json @@ -0,0 +1,55 @@ +[ + + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "4.0(2f)", + "configRole": "unspecified", + "descr": "APIC-SERVER-M4", + "dn": "topology/pod-2/node-3/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-M4" + } + } + }, + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "4.0(2f)", + "configRole": "unspecified", + "descr": "APIC-SERVER-L4", + "dn": "topology/pod-2/node-4/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-L4" + } + } + }, + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "3.0(4l)", + "configRole": "unspecified", + "descr": "APIC-SERVER-L2", + "dn": "topology/pod-1/node-1/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-L2" + } + } + }, + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "3.0(4l)", + "configRole": "unspecified", + "descr": "APIC-SERVER-M1", + "dn": "topology/pod-2/node-5/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-M1" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_new_cimc.json b/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_new_cimc.json new file mode 100644 index 00000000..708b47e5 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_new_cimc.json @@ -0,0 +1,29 @@ +[ + + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "4.5(2f)", + "configRole": "unspecified", + "descr": "APIC-SERVER-M4", + "dn": "topology/pod-2/node-3/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-M4" + } + } + }, + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "4.5(2f)", + "configRole": "unspecified", + "descr": "APIC-SERVER-L4", + "dn": "topology/pod-2/node-4/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-L4" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_old_cimc.json b/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_old_cimc.json new file mode 100644 index 00000000..05396766 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/eqptCh_m4l4_model_old_cimc.json @@ -0,0 +1,29 @@ +[ + + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "4.0(2f)", + "configRole": "unspecified", + "descr": "APIC-SERVER-M4", + "dn": "topology/pod-2/node-3/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-M4" + } + } + }, + { + "eqptCh": { + "attributes": { + "bootSource": "bootflash", + "cimcVersion": "4.0(2f)", + "configRole": "unspecified", + "descr": "APIC-SERVER-L4", + "dn": "topology/pod-2/node-4/sys/ch", + "hybridMode": "no", + "model": "APIC-SERVER-L4" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py b/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py index cb587fb3..4db1b7ea 100644 --- a/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py +++ b/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py @@ -17,15 +17,59 @@ compatRsSuppHwL2_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl2].json' compatRsSuppHwM1_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm1].json' +compatRsSuppHwL4_605_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl4].json' +compatRsSuppHwM4_605_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4].json' +compatRsSuppHwL4_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl4].json' +compatRsSuppHwM4_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4].json' @pytest.mark.parametrize( - "icurl_outputs, tversion, expected_result", + "icurl_outputs, tversion, cversion, expected_result", [ + #m4/l4 model check and targeting affected version and cversion affected and cimc < 4.3.5 + ( + {eqptCh_api: read_data(dir, "eqptCh_m4l4_model_old_cimc.json"), + compatRsSuppHwL4_605_api: read_data(dir, "compatRsSuppHw_605_M4L4.json"), + compatRsSuppHwM4_605_api: read_data(dir, "compatRsSuppHw_605_M4L4.json")}, + "6.0(5h)", + "5.3(1d)", + script.FAIL_UF, + ), + #m4/l4 with other apic server model and check targeting affect version and cversion affected and cimc < 4.3.5 + ( + { + eqptCh_api: read_data(dir, "eqptCh_m4l4_mixed_models.json"), + compatRsSuppHwL4_605_api: read_data(dir, "compatRsSuppHw_605_M4L4.json"), + compatRsSuppHwM4_605_api: read_data(dir, "compatRsSuppHw_605_M4L4.json"), + compatRsSuppHwL2_api: read_data(dir, "compatRsSuppHw_605_L2.json"), + compatRsSuppHwM1_api: read_data(dir, "compatRsSuppHw_605_M1.json")}, + "6.0(5h)", + "5.3(1d)", + script.FAIL_UF, + ), + # current cimc > 3.4.5 (known issue) but APIC current version is not affected + ( + {eqptCh_api: read_data(dir, "eqptCh_m4l4_model_new_cimc.json"), + compatRsSuppHwL4_api: read_data(dir, "compatRsSuppHw_615_M4L4.json"), + compatRsSuppHwM4_api: read_data(dir, "compatRsSuppHw_615_M4L4.json")}, + "6.1(5e)", + "6.1(4h)", + script.PASS, + ), + #version affected and cimc version > 4.3.5 + ( + {eqptCh_api: read_data(dir, "eqptCh_m4l4_model_new_cimc.json"), + compatRsSuppHwL4_605_api: read_data(dir, "compatRsSuppHw_605_M4L4.json"), + compatRsSuppHwM4_605_api: read_data(dir, "compatRsSuppHw_605_M4L4.json")}, + "6.0(5h)", + "5.3(1d)", + script.PASS, + ), ( {eqptCh_api: read_data(dir, "eqptCh_reallyoldver.json"), compatRsSuppHwL2_api: read_data(dir, "compatRsSuppHw_605_L2.json"), compatRsSuppHwM1_api: read_data(dir, "compatRsSuppHw_605_M1.json")}, "6.0(5a)", + None, script.FAIL_UF, ), ( @@ -33,6 +77,7 @@ compatRsSuppHwL2_api: read_data(dir, "compatRsSuppHw_605_L2.json"), compatRsSuppHwM1_api: read_data(dir, "compatRsSuppHw_605_M1.json")}, "6.0(5a)", + None, script.FAIL_UF, ), ( @@ -40,6 +85,7 @@ compatRsSuppHwL2_api: read_data(dir, "compatRsSuppHw_605_L2.json"), compatRsSuppHwM1_api: read_data(dir, "compatRsSuppHw_605_M1.json")}, "6.0(5a)", + None, script.PASS, ), # Seen in QA testing where version + model does not have catalog entry @@ -48,10 +94,11 @@ compatRsSuppHwL2_api: read_data(dir, "compatRsSuppHw_605_L2.json"), compatRsSuppHwM1_api: read_data(dir, "compatRsSuppHw_empty.json")}, "6.0(5a)", + None, script.MANUAL, ), ], ) -def test_logic(run_check, mock_icurl, tversion, expected_result): - result = run_check(tversion=script.AciVersion(tversion)) +def test_logic(run_check, mock_icurl, tversion, cversion, expected_result): + result = run_check(tversion=script.AciVersion(tversion), cversion=script.AciVersion(cversion) if cversion is not None else None) assert result.result == expected_result From 14689de9eaab97381120425daa1b0c66050190b2 Mon Sep 17 00:00:00 2001 From: Priyanka-Patil14 Date: Wed, 1 Jul 2026 19:22:26 +0530 Subject: [PATCH 09/44] Resolved conflicts (#370) --- aci-preupgrade-validation-script.py | 61 ++++++++++++++ docs/docs/validations.md | 14 +++- .../fabricNode_N9K-C9508.json | 13 +++ .../fabricNode_one.json | 13 +++ .../fabricNode_two.json | 24 ++++++ .../procMemUsage_lt32gb.json | 11 +++ ...st_n9k_c93180yc_fx3_switch_memory_check.py | 82 +++++++++++++++++++ 7 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_N9K-C9508.json create mode 100644 tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_one.json create mode 100644 tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_two.json create mode 100644 tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb.json create mode 100644 tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index a3ca8539..667ec421 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6532,6 +6532,66 @@ def wred_affected_model_check(tversion, fabric_nodes, **kwargs): return Result(result=NA, msg="No affected Fabric module found.") +@check_wrapper(check_title='N9K-C93180YC-FX3 Switch Memory Less Than 32GB') +def n9k_c93180yc_fx3_switch_memory_check(fabric_nodes, **kwargs): + result = PASS + headers = ["NodeId", "Name", "Model", "Memory Detected (GB)"] + data = [] + recommended_action = 'Increase the switch memory to at least 32GB on affected N9K-C93180YC-FX3.' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#n9k-c93180yc-fx3-switch-memory-less-than-32gb' + min_memory_kb = 32 * 1000 * 1000 + msg = '' + + affected_nodes = [ + node for node in fabric_nodes + if node['fabricNode']['attributes']['model'] == 'N9K-C93180YC-FX3' + ] + + if not affected_nodes: + result = NA + msg = 'No N9K-C93180YC-FX3 switches found. Skipping.' + else: + node_ids = [node['fabricNode']['attributes']['id'] for node in affected_nodes] + node_filter = 'or({})'.format(','.join( + 'wcard(procMemUsage.dn,"node-{}/")'.format(nid) for nid in node_ids + )) + query = 'procMemUsage.json?query-target-filter=and({},wcard(procMemUsage.dn,"memusage-sup"),lt(procMemUsage.Total,"{}"))'.format( + node_filter, min_memory_kb + ) + proc_mem_mos = icurl('class', query) + + node_id_to_attrs = { + node['fabricNode']['attributes']['id']: node['fabricNode']['attributes'] + for node in affected_nodes + } + + for memory_mo in proc_mem_mos: + attrs = memory_mo['procMemUsage']['attributes'] + dn_match = re.search(node_regex, attrs['dn']) + if not dn_match: + continue + node_id = dn_match.group('node') + if node_id not in node_id_to_attrs: + continue + memory_in_gb = round(int(attrs['Total']) / 1000000, 2) + result = FAIL_O + data.append([ + node_id, + node_id_to_attrs[node_id]['name'], + node_id_to_attrs[node_id]['model'], + memory_in_gb, + ]) + + if data: + msg = ( + 'N9K-C93180YC-FX3 requires a minimum of 32GB RAM for proper operation in ACI mode. ' + 'One or more switches with less than 32GB of memory may experience service instability. ' + 'Upgrade the switch memory to at least 32GB.' + ) + + return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6705,6 +6765,7 @@ class CheckManager: inband_management_policy_misconfig_check, bgpProto_timer_policy_already_existing_check, wred_affected_model_check, + n9k_c93180yc_fx3_switch_memory_check, ] ssh_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 76e9c140..8c88d796 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -205,6 +205,7 @@ Items | Defect | This Script [Inband Management Policy Misconfiguration][d33]| CSCwd40071 | :white_check_mark: | :no_entry_sign: [BgpProto timer policy already existing][d34] | CSCwt78235 | :white_check_mark: | :no_entry_sign: [WRED with Affected FM Models][d35] | CSCwt50713 | :white_check_mark: | :no_entry_sign: +[N9K-C93180YC-FX3 Switch Memory Less Than 32GB][d36] | CSCwm42741 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -241,6 +242,7 @@ Items | Defect | This Script [d33]: #inband-management-policy-misconfiguration [d34]: #bgpProto-timer-policy-already-existing [d35]: #wred-with-affected-fm-models +[d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb ## General Check Details @@ -2819,6 +2821,15 @@ To avoid this issue, disable WRED on the affected nodes or upgrade to a release This bug [CSCwt78235][71] validates `F0467` faults where `changeSet` contains 'bgpProt-policy-already-existing'. The fault indicates conflicting BGP protocol timer policy under an L3Outs deployed in same vrf under same node. If this fault is not resolved, l3out will not be programmed properly in the leaf after the clean reboot or the upgrade. +### N9K-C93180YC-FX3 Switch Memory Less Than 32GB + +This check applies to N9K-C93180YC-FX3 switches only. It checks whether the switch has less than 32GB of memory. The minimum RAM requirement for the N9K-C93180YC-FX3 to operate properly in ACI mode is 32GB. This check is not version dependent and runs for all upgrade versions. + +[CSCwm42741][74] tracks this issue. N9K-C93180YC-FX3 switches running in ACI mode with less than 32GB of memory will not perform well and are at risk of service instability. With fix of CSCwm42741, a critical fault F4680 (`eqpt-low-memory-device`) is raised on affected switches. + +If any N9K-C93180YC-FX3 switch is flagged by this check, upgrade the switch memory to at least 32GB before proceeding with the upgrade. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2892,4 +2903,5 @@ This bug [CSCwt78235][71] validates `F0467` faults where `changeSet` contains 'b [70]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvo27498 [71]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt78235 [72]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt50713 -[73]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwo74485 \ No newline at end of file +[73]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwo74485 +[74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 \ No newline at end of file diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_N9K-C9508.json b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_N9K-C9508.json new file mode 100644 index 00000000..a82341a4 --- /dev/null +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_N9K-C9508.json @@ -0,0 +1,13 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-201", + "id": "201", + "name": "leaf201", + "model": "N9K-C9508", + "role": "leaf" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_one.json b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_one.json new file mode 100644 index 00000000..74a31ad3 --- /dev/null +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_one.json @@ -0,0 +1,13 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "id": "101", + "name": "leaf101", + "model": "N9K-C93180YC-FX3", + "role": "leaf" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_two.json b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_two.json new file mode 100644 index 00000000..fad02898 --- /dev/null +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/fabricNode_two.json @@ -0,0 +1,24 @@ +[ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-101", + "id": "101", + "name": "leaf101", + "model": "N9K-C93180YC-FX3", + "role": "leaf" + } + } + }, + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-102", + "id": "102", + "name": "leaf102", + "model": "N9K-C9364C", + "role": "leaf" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb.json b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb.json new file mode 100644 index 00000000..5f50d60d --- /dev/null +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb.json @@ -0,0 +1,11 @@ +[ + { + "procMemUsage": { + "attributes": { + "dn": "topology/pod-1/node-101/sys/procmem/memusage-sup", + "Modname": "sup", + "Total": "16000000" + } + } + } +] diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py new file mode 100644 index 00000000..45f46006 --- /dev/null +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py @@ -0,0 +1,82 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "n9k_c93180yc_fx3_switch_memory_check" + +# icurl queries - filtered by affected node IDs and memory threshold +proc_mem_query_node101 = 'procMemUsage.json?query-target-filter=and(or(wcard(procMemUsage.dn,"node-101/")),wcard(procMemUsage.dn,"memusage-sup"),lt(procMemUsage.Total,"32000000"))' + + +@pytest.mark.parametrize( + "fabric_nodes, icurl_outputs, expected_result, expected_msg, expected_data", + [ + # No nodes returned + ( + [], + {}, + script.NA, + 'No N9K-C93180YC-FX3 switches found. Skipping.', + [], + ), + # Non-N9K-C93180YC-FX3 node (N9K-C9508) + ( + read_data(dir, "fabricNode_N9K-C9508.json"), + {}, + script.NA, + 'No N9K-C93180YC-FX3 switches found. Skipping.', + [], + ), + # N9K-C93180YC-FX3 node with >=32GB memory - API returns empty (filtered by lt) + ( + read_data(dir, "fabricNode_one.json"), + { + proc_mem_query_node101: [], + }, + script.PASS, + '', + [], + ), + # Multiple nodes, only N9K-C93180YC-FX3 checked, all >=32GB - API returns empty + ( + read_data(dir, "fabricNode_two.json"), + { + proc_mem_query_node101: [], + }, + script.PASS, + '', + [], + ), + # N9K-C93180YC-FX3 node with <32GB memory (fail case) + ( + read_data(dir, "fabricNode_one.json"), + { + proc_mem_query_node101: read_data(dir, "procMemUsage_lt32gb.json"), + }, + script.FAIL_O, + ( + 'N9K-C93180YC-FX3 requires a minimum of 32GB RAM for proper operation in ACI mode. ' + 'One or more switches with less than 32GB of memory may experience service instability. ' + 'Upgrade the switch memory to at least 32GB.' + ), + [["101", "leaf101", "N9K-C93180YC-FX3", 16.0]], + ), + ], +) +def test_logic(run_check, mock_icurl, fabric_nodes, expected_result, expected_msg, expected_data): + result = run_check( + fabric_nodes=fabric_nodes, + ) + assert result.result == expected_result + assert result.msg == expected_msg + if result.data: + assert result.data == expected_data + else: + assert result.unformatted_data == expected_data \ No newline at end of file From ea8f62e725bed353f506b45e66a181749ad67433 Mon Sep 17 00:00:00 2001 From: Priyanka-Patil14 Date: Wed, 1 Jul 2026 19:45:39 +0530 Subject: [PATCH 10/44] Added pre-upgrade check for defect CSCwt69100 (#385) * Resolved Conflicts * Update validations.md * Update validations.md * Update aci-preupgrade-validation-script.py --------- Co-authored-by: GM --- aci-preupgrade-validation-script.py | 30 +++++++++++++++++++++++++++++ docs/docs/validations.md | 15 +++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 667ec421..8d47b755 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6592,6 +6592,34 @@ def n9k_c93180yc_fx3_switch_memory_check(fabric_nodes, **kwargs): return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) +@check_wrapper(check_title="Stale dbgacEpgSummaryTask Objects") +def stale_dbgacEpgSummaryTask_check(tversion, **kwargs): + result = PASS + headers = ["DN", "Start Time"] + data = [] + recommended_action = "Contact Cisco TAC for next steps. For more details, refer to the workaround in [CSCwt69100](https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100)." + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#stale-dbgacepgsummarytask-objects" + + if tversion and ((tversion.major1 == "6" and tversion.major2 == "1" and tversion.newer_than("6.1(5e)")) or tversion.newer_than("6.2(1g)")): + return Result(result=NA, msg=VER_NOT_AFFECTED, doc_url=doc_url) + + threshold = datetime.utcnow() - timedelta(hours=24) + for obj in icurl("class", 'dbgacEpgSummaryTask.json?query-target-filter=eq(dbgacEpgSummaryTask.operSt,"processing")'): + attr = obj["dbgacEpgSummaryTask"]["attributes"] + dn = attr.get("dn", "") + start_ts = attr.get("startTs", "") + try: + task_dt = datetime.strptime(start_ts[:19], "%Y-%m-%dT%H:%M:%S") + except ValueError: + continue + if task_dt < threshold: + data.append([dn, start_ts]) + + if data: + result = FAIL_UF + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6766,6 +6794,8 @@ class CheckManager: bgpProto_timer_policy_already_existing_check, wred_affected_model_check, n9k_c93180yc_fx3_switch_memory_check, + stale_dbgacEpgSummaryTask_check, + ] ssh_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 8c88d796..85588880 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -140,7 +140,6 @@ Items | Faults | This Script [AVE End-of-life][c23] | :white_check_mark: | :no_entry_sign: [Shared Service with vzAny Consumer][c24] | :white_check_mark: | :no_entry_sign: - [c1]: #vpc-paired-leaf-switches [c2]: #overlapping-vlan-pool [c3]: #vnid-mismatch @@ -206,6 +205,7 @@ Items | Defect | This Script [BgpProto timer policy already existing][d34] | CSCwt78235 | :white_check_mark: | :no_entry_sign: [WRED with Affected FM Models][d35] | CSCwt50713 | :white_check_mark: | :no_entry_sign: [N9K-C93180YC-FX3 Switch Memory Less Than 32GB][d36] | CSCwm42741 | :white_check_mark: | :no_entry_sign: +[Stale dbgacEpgSummaryTask Objects][d37] | CSCwt69100 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -243,6 +243,7 @@ Items | Defect | This Script [d34]: #bgpProto-timer-policy-already-existing [d35]: #wred-with-affected-fm-models [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb +[d37]: #stale-dbgacepgsummarytask-objects ## General Check Details @@ -2830,6 +2831,15 @@ This check applies to N9K-C93180YC-FX3 switches only. It checks whether the swit If any N9K-C93180YC-FX3 switch is flagged by this check, upgrade the switch memory to at least 32GB before proceeding with the upgrade. +### Stale dbgacEpgSummaryTask Objects + +Due to [CSCwt69100][75], a stale `dbgacEpgSummaryTask` object stuck in `processing` state with empty content can cause the policymgr process to crash on all APICs during an upgrade or process restart. + +Affected versions: 6.1(5e) and below, or 6.2(1g). + +Contact Cisco TAC for next steps. For more details, refer to the workaround in [CSCwt69100][75]. + + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2904,4 +2914,5 @@ If any N9K-C93180YC-FX3 switch is flagged by this check, upgrade the switch memo [71]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt78235 [72]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt50713 [73]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwo74485 -[74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 \ No newline at end of file +[74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 +[75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 From ffc1138cc3922f15f5a00da0807e2f0481716965 Mon Sep 17 00:00:00 2001 From: asraf-khan Date: Wed, 1 Jul 2026 19:53:52 +0530 Subject: [PATCH 11/44] Added new script validation for CSCwt38698 (#384) * Add arg to define thread limit - to throttle concurrent API calls when required (#355) * add `--max-threads` arg * fix bad descriptor errs/race conditions * update pytests * 353 the script incorrectly detects vpc and port channel interfaces as cscwh68103 invalid fabricpathep targets (#357) * specific testing for known failure conditions of cscwh68103 as to not catch valid scenarios * Update aci-preupgrade-validation-script.py mark version * print cleanup * fix: Add cversion check for post_upgrade_cb_check (#377) * added validation for CFD CSCwp64296 (#307) Added rogue ep/coop exception mac check for the CFD CSCwp64296 * Added pre-upgrade validation for N9K-C9408 with more than 6 N9K-X9400-16W LEM's for the bug CSCws82819 (#354) * Added a new check for the bug 'CSCws82819N9K-C9408 boot loop on 16.1.2f and later with 6 or more LEMs' * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) (#361) * New Validation for APIC Storage Inode Usage (F4388, F4389, F4390 equipment-full) * Add new exception handling of invalid query filter in `icurl` due to, for example, a non-supported fault code on older versions * Add validation for multipod_modular_spine_bootscript_check - CSCwr66848 (#365) * Add check for CFD - CSCwr66848 * Update pytest.yml to run on vX.Y.Z branches * adding of cli parameters for user and password (#335) * added cli parameter for user (-u) and password (-p) to be used in an eased way for fully automated execution of 'aci-preupgrade-validation-script.py' * test: Add pytest --------- Co-authored-by: Detlef Sass * Added validation for CSCwd40071 (#332) * Added validation for CSCwd40071 * Addressed the comments * Added cversion for the check * Removed empty spaces * logic change. removed 0.0.0.0/0 and made pytest changes * logic modified and validation.md file updated * Added check for CFD - CSCwt38698 * Removed serial Number from header * Updated check name and added post upgrade check * Updated review comments * Updated review comments * Updated review comments on combining check on existing function * Resolved conflicts * Updated as review comments * Updated as review comments * Updated as review comments * Updated as review comments * Updated as review comments --------- Co-authored-by: GM Co-authored-by: Gabriel Co-authored-by: tkishida Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> Co-authored-by: psureshb Co-authored-by: Harinadh-Saladi Co-authored-by: sanjanch Co-authored-by: Thatleft Co-authored-by: Detlef Sass --- aci-preupgrade-validation-script.py | 126 +++++++-- docs/docs/validations.md | 8 + .../eqptFlash_mixed_node.json | 12 + .../eqptFlash_multi_micron.json | 23 ++ .../eqptFlash_single_micron_noFault.json | 14 + .../eqptFlash_single_micron_withFault.json | 14 + .../switch_ssd_check/test_switch_ssd_check.py | 260 +++++++++++++++++- 7 files changed, 428 insertions(+), 29 deletions(-) create mode 100644 tests/checks/switch_ssd_check/eqptFlash_mixed_node.json create mode 100644 tests/checks/switch_ssd_check/eqptFlash_multi_micron.json create mode 100644 tests/checks/switch_ssd_check/eqptFlash_single_micron_noFault.json create mode 100644 tests/checks/switch_ssd_check/eqptFlash_single_micron_withFault.json diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 8d47b755..5e79f56e 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -2611,19 +2611,35 @@ def hw_program_fail_check(cversion, **kwargs): @check_wrapper(check_title="Switch SSD Health (F3073, F3074 equipment-flash-warning)") -def switch_ssd_check(**kwargs): +def switch_ssd_check(cversion, tversion, **kwargs): result = FAIL_O - headers = ["Fault", "Pod", "Node", "SSD Model", "% Threshold Crossed", "Recommended Action"] + headers = ["Fault", "Pod", "Node", "SSD Model", "% Threshold Crossed"] data = [] - unformatted_headers = ["Fault", "Fault DN", "% Threshold Crossed", "Recommended Action"] + unformatted_headers = ["Fault", "Fault DN", "% Threshold Crossed"] unformatted_data = [] thresh = {'F3073': '90%', 'F3074': '80%'} - recommended_action = { - 'F3073': 'Contact Cisco TAC for replacement procedure', - 'F3074': 'Monitor (no impact to upgrades)' - } + overall_ra = "" + micron_ra = ( + '\n\tRun the SSD Lifetime Validation script manually on all identified nodes before upgrading.\n' + '\tScript location: https://github.com/datacenter/aci-tac-scripts/tree/main/SSD%20Lifetime%20Validation\n' + ) + fault_ra = "Contact Cisco TAC for replacement procedure" + mixed_ra = ( + "Mixed SSD faults detected:" + "\n\tFor non-Micron SSDs (F3073/F3074 rows): Contact Cisco TAC for replacement procedure.\n" + "\tFor Micron SSD: Run the SSD Lifetime Validation script manually on all identified nodes before upgrading.\n" + "\tScript location: https://github.com/datacenter/aci-tac-scripts/tree/main/SSD%20Lifetime%20Validation\n" + ) + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-ssd-health" + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + affected = ['6.1(5e)', '6.2(1g)'] + cver_affected = any(cversion.same_as(v) for v in affected) + tver_affected = any(tversion.same_as(v) for v in affected) + cs_regex = r"model:(?P\w+)," faultInsts = icurl('class', 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F3073"),eq(faultInst.code,"F3074"))') @@ -2632,25 +2648,91 @@ def switch_ssd_check(**kwargs): dn_array = re.search(node_regex, faultInst['faultInst']['attributes']['dn']) cs_array = re.search(cs_regex, faultInst['faultInst']['attributes']['changeSet']) if dn_array and cs_array: + ssd_model = cs_array.group("model") data.append([fc, dn_array.group("pod"), dn_array.group("node"), cs_array.group("model"), - thresh.get(fc, ''), - recommended_action.get(fc, 'Resolve the fault')]) + thresh.get(fc, '')]) else: unformatted_data.append([fc, faultInst['faultInst']['attributes']['dn'], - thresh.get(fc, ''), - recommended_action.get(fc, 'Resolve the fault')]) - if not data and not unformatted_data: - result = PASS - return Result( - result=result, - headers=headers, - data=data, - unformatted_headers=unformatted_headers, - unformatted_data=unformatted_data, - doc_url=doc_url, - ) + thresh.get(fc, '')]) + has_fault_data = bool(data or unformatted_data) + + def collect_micron(classify): + eqptFlashs = icurl('class', 'eqptFlash.json?query-target-filter=eq(eqptFlash.vendor,"Micron")') + if not eqptFlashs: + return False, False + + micron_ssds_per_node = defaultdict(set) + micron_rows = [] + + for eqptFlash in eqptFlashs: + attr = eqptFlash['eqptFlash']['attributes'] + dn = re.search(node_regex, attr.get("dn", "")) + node_id = dn.group("node") + pod_id = dn.group("pod") + model = attr.get('model', '') + + micron_ssds_per_node[node_id].add(model) + micron_rows.append(['CSCwt38698 (False Fault Micron SSD defect)', + pod_id, + node_id, model, 'N/A']) + + if classify: + genuine_faults = [] + micron_false_faults = [] + + for fault_row in data: + node_id = fault_row[2] + ssd_model = fault_row[3] + + is_micron_fault = (node_id in micron_ssds_per_node and ssd_model in micron_ssds_per_node[node_id]) + + if not is_micron_fault: + genuine_faults.append(fault_row) + else: + for micron_row in micron_rows: + if micron_row[2] == node_id and micron_row[3] == ssd_model: + micron_false_faults.append(micron_row) + break + + del data[:] + del unformatted_data[:] + data.extend(genuine_faults) + data.extend(micron_false_faults) + return bool(micron_false_faults), bool(genuine_faults) + else: + data.extend(micron_rows) + return True, False + + if cver_affected: + has_micron_faults, has_genuine_fault = collect_micron(classify=True) + if has_micron_faults: + result = MANUAL + if has_genuine_fault: + overall_ra = mixed_ra + else: + overall_ra = micron_ra + elif has_fault_data: + result = FAIL_O + overall_ra = fault_ra + else: + result = PASS + elif tver_affected: + if has_fault_data: + result = FAIL_O + overall_ra = fault_ra + elif collect_micron(classify=False)[0]: + result = MANUAL + overall_ra = micron_ra + else: + result = PASS + else: + result = FAIL_O if has_fault_data else PASS + if has_fault_data: + overall_ra = fault_ra + + return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=overall_ra, doc_url=doc_url) # Connection Based Check @check_wrapper(check_title="APIC SSD Health") @@ -6967,4 +7049,4 @@ def main(_args=None): msg = "Abort due to unexpected error - {}".format(e) prints(msg) log.error(msg, exc_info=True) - sys.exit(1) + sys.exit(1) \ No newline at end of file diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 85588880..f7886811 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -810,6 +810,12 @@ See the [ACI Switch Node SSD Lifetime Explained technote][9] for more details. --- omit --- ``` +Due to [CSCwt38698][76], Micron SSDs present in the fabric may give false end-of-life failures after upgrading to 6.1(5e) or 6.2(1g). + +To confirm if this is genuine or false alarm, run the SSD Lifetime Validation script on all nodes with identified actual failure case. If the SSD lifetime is critically low after manually running the script, you have to follow the SSD replacement procedure outlined in the field notice to ensure that the node remains available after the upgrade. To avoid this false alarm you can choose non-impacted target version. + +- Script location: [SSD Lifetime Validation](https://github.com/datacenter/aci-tac-scripts/tree/main/SSD%20Lifetime%20Validation) + ### Config On APIC Connected Port @@ -2916,3 +2922,5 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [ [73]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwo74485 [74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 +[76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 + diff --git a/tests/checks/switch_ssd_check/eqptFlash_mixed_node.json b/tests/checks/switch_ssd_check/eqptFlash_mixed_node.json new file mode 100644 index 00000000..9440e4fd --- /dev/null +++ b/tests/checks/switch_ssd_check/eqptFlash_mixed_node.json @@ -0,0 +1,12 @@ +[ + { + "eqptFlash": { + "attributes": { + "dn": "topology/pod-1/node-205/sys/ch/supslot-2/sup/flash", + "model": "Micron_M600_MTFDDAT064MBF", + "vendor": "Micron", + "ser": "MSA20400892" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/switch_ssd_check/eqptFlash_multi_micron.json b/tests/checks/switch_ssd_check/eqptFlash_multi_micron.json new file mode 100644 index 00000000..c88aa8fa --- /dev/null +++ b/tests/checks/switch_ssd_check/eqptFlash_multi_micron.json @@ -0,0 +1,23 @@ +{ + "totalCount": "2", + "imdata": [ + { + "eqptFlash": { + "attributes": { + "dn": "topology/pod-1/node-205/sys/ch/supslot-1/sup/flash", + "vendor": "Micron", + "model": "Micron_M550_MTFDDAT256MAY" + } + } + }, + { + "eqptFlash": { + "attributes": { + "dn": "topology/pod-1/node-101/sys/ch/supslot-1/sup/flash", + "vendor": "Micron", + "model": "Micron_M600_MTFDDAT064MBF" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/checks/switch_ssd_check/eqptFlash_single_micron_noFault.json b/tests/checks/switch_ssd_check/eqptFlash_single_micron_noFault.json new file mode 100644 index 00000000..691f8ad5 --- /dev/null +++ b/tests/checks/switch_ssd_check/eqptFlash_single_micron_noFault.json @@ -0,0 +1,14 @@ +{ + "totalCount": "1", + "imdata": [ + { + "eqptFlash": { + "attributes": { + "dn": "topology/pod-1/node-103/sys/ch/supslot-1/sup/flash", + "vendor": "Micron", + "model": "MTFDDAK240MBB" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/checks/switch_ssd_check/eqptFlash_single_micron_withFault.json b/tests/checks/switch_ssd_check/eqptFlash_single_micron_withFault.json new file mode 100644 index 00000000..4f9d6492 --- /dev/null +++ b/tests/checks/switch_ssd_check/eqptFlash_single_micron_withFault.json @@ -0,0 +1,14 @@ +{ + "totalCount": "1", + "imdata": [ + { + "eqptFlash": { + "attributes": { + "dn": "topology/pod-1/node-205/sys/ch/supslot-1/sup/flash", + "vendor": "Micron", + "model": "Micron_M550_MTFDDAT256MAY" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/checks/switch_ssd_check/test_switch_ssd_check.py b/tests/checks/switch_ssd_check/test_switch_ssd_check.py index 1f7202f5..9c141247 100644 --- a/tests/checks/switch_ssd_check/test_switch_ssd_check.py +++ b/tests/checks/switch_ssd_check/test_switch_ssd_check.py @@ -14,18 +14,138 @@ # icurl queries faultInst = 'faultInst.json?query-target-filter=or(eq(faultInst.code,"F3073"),eq(faultInst.code,"F3074"))' +eqptFlash = 'eqptFlash.json?query-target-filter=eq(eqptFlash.vendor,"Micron")' @pytest.mark.parametrize( - "icurl_outputs, expected_result, expected_data", + "icurl_outputs, tversion, cversion, expected_result, expected_data", [ + # MANUAL - tversion missing (TVER_MISSING), no faults ( {faultInst: []}, + None, "6.0(2h)", + script.MANUAL, + [], + ), + # FAIL_O - genuine F3073/F3074 faults, version not affected + ( + {faultInst: read_data(dir, "faultInst.json")}, + "6.0(2h)", "6.0(1a)", + script.FAIL_O, + [ + [ + "F3073", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "90%", + ], + [ + "F3074", + "1", + "101", + "Micron_M600_MTFDDAT064MBF", + "80%", + ], + ], + ), + # PASS - no faults, version not affected (Micron block skipped) + ( + {faultInst: []}, + "6.0(2h)", "6.0(1a)", script.PASS, [], ), + # PASS - no faults, tversion affected 6.1(5e), no Micron drives ( - {faultInst: read_data(dir, "faultInst.json")}, + {faultInst: [], eqptFlash: []}, + "6.1(5e)", "6.0(2h)", + script.PASS, + [], + ), + # PASS - no faults, tversion affected 6.2(1g), no Micron drives + ( + {faultInst: [], eqptFlash: []}, + "6.2(1g)", "6.0(2h)", + script.PASS, + [], + ), + # PASS - no faults, cversion affected 6.1(5e), no Micron drives + ( + {faultInst: [], eqptFlash: []}, + "6.2(2a)", "6.1(5e)", + script.PASS, + [], + ), + # MANUAL - no faults, tversion affected 6.1(5e), single Micron drive + ( + {faultInst: [], eqptFlash: read_data(dir, "eqptFlash_single_micron_noFault.json")}, + "6.1(5e)", "6.0(2h)", + script.MANUAL, + [ + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "103", + "MTFDDAK240MBB", + "N/A", + ], + ], + ), + # MANUAL - no faults, multiple Micron drives, tversion affected + ( + {faultInst: [], eqptFlash: read_data(dir, "eqptFlash_multi_micron.json")}, + "6.1(5e)", "6.0(2h)", + script.MANUAL, + [ + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "N/A", + ], + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "101", + "Micron_M600_MTFDDAT064MBF", + "N/A", + ], + ], + ), + # MANUAL - false fault present + cversion affected + Micron drive + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: read_data(dir, "eqptFlash_multi_micron.json"), + }, + "6.2(2a)", "6.1(5e)", + script.MANUAL, + [ + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "N/A", + ], + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "101", + "Micron_M600_MTFDDAT064MBF", + "N/A", + ], + ], + ), + # FAIL_O - Genuine fault present + cversion affected + Micron drive + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: read_data(dir, "eqptFlash_single_micron_noFault.json"), + }, + "6.2(2a)", "6.1(5e)", script.FAIL_O, [ [ @@ -34,7 +154,6 @@ "205", "Micron_M550_MTFDDAT256MAY", "90%", - "Contact Cisco TAC for replacement procedure", ], [ "F3074", @@ -42,13 +161,140 @@ "101", "Micron_M600_MTFDDAT064MBF", "80%", - "Monitor (no impact to upgrades)", ], ], ), + # FAIL_O - fault present + tversion matched + Micron drive found + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: read_data(dir, "eqptFlash_single_micron_noFault.json"), + }, + "6.1(5e)", "6.0(2h)", + script.FAIL_O, + [ + [ + "F3073", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "90%", + ], + [ + "F3074", + "1", + "101", + "Micron_M600_MTFDDAT064MBF", + "80%", + ], + ], + ), + # MANUAL - false fault present + cversion matched + Micron drive found + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: read_data(dir, "eqptFlash_multi_micron.json"), + }, + "6.2(2a)", "6.1(5e)", + script.MANUAL, + [ + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "N/A", + ], + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "101", + "Micron_M600_MTFDDAT064MBF", + "N/A", + ], + ], + ), + # FAIL_O - fault present + cversion matched + Micron drive absent + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: [], + }, + "6.2(2a)", "6.1(5e)", + script.FAIL_O, + [ + [ + "F3073", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "90%", + ], + [ + "F3074", + "1", + "101", + "Micron_M600_MTFDDAT064MBF", + "80%", + ], + ], + ), + # MANUAL - Genuine + false fault present + cversion matched + Micron drive found + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: read_data(dir, "eqptFlash_single_micron_withFault.json"), + }, + "6.2(2a)", "6.1(5e)", + script.MANUAL, + [ + [ + 'F3074', + '1', + '101', + 'Micron_M600_MTFDDAT064MBF', + '80%', + ], + [ + "CSCwt38698 (False Fault Micron SSD defect)", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "N/A", + ], + ], + ), + # FAIL_O - Same node (205), different slots: genuine fault (Intel SSD) + ( + { + faultInst: read_data(dir, "faultInst.json"), + eqptFlash: read_data(dir, "eqptFlash_mixed_node.json"), + }, + "6.2(2a)", "6.1(5e)", + script.FAIL_O, + [ + [ + "F3073", + "1", + "205", + "Micron_M550_MTFDDAT256MAY", + "90%", + ], + [ + 'F3074', + '1', + '101', + 'Micron_M600_MTFDDAT064MBF', + '80%', + ], + ], + ), ], ) -def test_logic(run_check, mock_icurl, expected_result, expected_data): - result = run_check() +def test_logic(run_check, mock_icurl, tversion, cversion, expected_result, expected_data): + result = run_check( + tversion=script.AciVersion(tversion) if tversion else None, + cversion=script.AciVersion(cversion) if cversion else None + ) assert result.result == expected_result - assert result.data == expected_data + assert result.data == expected_data \ No newline at end of file From 152b77f14298ad5f6cda638555f83aa3c8da927c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 22 Jul 2026 10:22:20 -0400 Subject: [PATCH 12/44] Expand out-of-service port check to fabric ports (#292) Use explicit usage-mask equality filters to cover access and fabric blacklist states without matching unrelated mask values. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 6 ++- docs/docs/validations.md | 2 +- .../ethpmPhysIf-pos.json | 20 +++++++++- .../test_out_of_service_ports_check.py | 37 ++++++++++++++++--- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 5e79f56e..0ef3986b 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -5550,7 +5550,11 @@ def out_of_service_ports_check(**kwargs): doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#out-of-service-ports' ethpmPhysIf_api = 'ethpmPhysIf.json' - ethpmPhysIf_api += '?query-target-filter=and(eq(ethpmPhysIf.operSt,"2"),bw(ethpmPhysIf.usage,"32","34"))' + ethpmPhysIf_api += ( + '?query-target-filter=and(eq(ethpmPhysIf.operSt,"2"),' + 'or(eq(ethpmPhysIf.usage,"32"),eq(ethpmPhysIf.usage,"34"),' + 'eq(ethpmPhysIf.usage,"36"),eq(ethpmPhysIf.usage,"292")))' + ) ethpmPhysIf = icurl('class', ethpmPhysIf_api) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f7886811..68c72422 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2242,7 +2242,7 @@ This check will look for configured Pre-shared keys (PSK) within your APIC clust ### Out-of-Service Ports -Any Port that has been disabled via policy creates a `fabricRsOosPath` object and marks the ports usage as `blacklist`, or `blacklist,epg` if policy was applied to it. `fabricRsOosPath` objects can be found within the UI at the "Fabric" > "Disabled Interfaces and Decommissioned Switches" view. +Any access/downlink or fabric port that has been disabled via policy creates a `fabricRsOosPath` object. The check covers operationally up ports with the `blacklist`, `blacklist,epg`, `blacklist,fabric`, or `blacklist,fabric,fabric-ext` usage. `fabricRsOosPath` objects can be found within the UI at the "Fabric" > "Disabled Interfaces and Decommissioned Switches" view. While generally not recommended, there are policy bypass methods to bring up ports which are out-of-service via policy. The problem arises from the ports active state deviating from ports configured policy, and this fact generally remains undetected as policy was bypassed. If an event occurs which causes Switch Nodes to receive and reprogram policy from the APICs, the configured out-of-service policy will bring the out-of-service ports down, as expected. diff --git a/tests/checks/out_of_service_ports_check/ethpmPhysIf-pos.json b/tests/checks/out_of_service_ports_check/ethpmPhysIf-pos.json index 2ca30264..f3c3f784 100644 --- a/tests/checks/out_of_service_ports_check/ethpmPhysIf-pos.json +++ b/tests/checks/out_of_service_ports_check/ethpmPhysIf-pos.json @@ -4,7 +4,7 @@ "attributes": { "dn": "topology/pod-1/node-103/sys/phys-[eth1/9]/phys", "operSt": "up", - "usage": "blacklist,epg" + "usage": "blacklist" } } }, @@ -16,5 +16,23 @@ "usage": "blacklist,epg" } } + }, + { + "ethpmPhysIf": { + "attributes": { + "dn": "topology/pod-1/node-105/sys/phys-[eth1/49]/phys", + "operSt": "up", + "usage": "blacklist,fabric" + } + } + }, + { + "ethpmPhysIf": { + "attributes": { + "dn": "topology/pod-1/node-105/sys/phys-[eth1/50]/phys", + "operSt": "up", + "usage": "blacklist,fabric,fabric-ext" + } + } } ] \ No newline at end of file diff --git a/tests/checks/out_of_service_ports_check/test_out_of_service_ports_check.py b/tests/checks/out_of_service_ports_check/test_out_of_service_ports_check.py index 4296eeec..21b6d5ab 100644 --- a/tests/checks/out_of_service_ports_check/test_out_of_service_ports_check.py +++ b/tests/checks/out_of_service_ports_check/test_out_of_service_ports_check.py @@ -11,27 +11,52 @@ test_function = "out_of_service_ports_check" -# operst: '1' = 'up' -# usage: '32' = 'blacklist', '2' = 'epg'. '34'= 'blacklist,epg' +# operSt: '2' = 'up' +# usage: '32' = 'blacklist', '34' = 'blacklist,epg', +# '36' = 'blacklist,fabric', '292' = 'blacklist,fabric,fabric-ext' ethpmPhysIf_api = 'ethpmPhysIf.json' -ethpmPhysIf_api += '?query-target-filter=and(eq(ethpmPhysIf.operSt,"2"),bw(ethpmPhysIf.usage,"32","34"))' +ethpmPhysIf_api += ( + '?query-target-filter=and(eq(ethpmPhysIf.operSt,"2"),' + 'or(eq(ethpmPhysIf.usage,"32"),eq(ethpmPhysIf.usage,"34"),' + 'eq(ethpmPhysIf.usage,"36"),eq(ethpmPhysIf.usage,"292")))' +) @pytest.mark.parametrize( - "icurl_outputs, expected_result", + "icurl_outputs, expected_result, expected_usages", [ ( - # Two 'up' ports flagged with 'blacklist,epg' + # Four 'up' access and fabric ports with supported blacklist masks {ethpmPhysIf_api: read_data(dir, "ethpmPhysIf-pos.json")}, script.FAIL_O, + [ + "blacklist", + "blacklist,epg", + "blacklist,fabric", + "blacklist,fabric,fabric-ext", + ], ), ( # 0 ports returned {ethpmPhysIf_api: read_data(dir, "ethpmPhysIf-neg.json")}, script.PASS, + [], ) ], ) -def test_logic(run_check, mock_icurl, expected_result): +def test_logic(run_check, mock_icurl, expected_result, expected_usages): result = run_check() assert result.result == expected_result + assert [row[4] for row in result.data] == expected_usages + + +@pytest.mark.parametrize("usage_mask", ["32", "34", "36", "292"]) +def test_requested_usage_masks_are_queried(usage_mask): + usage_filter = 'eq(ethpmPhysIf.usage,"{}")'.format(usage_mask) + assert usage_filter in ethpmPhysIf_api + + +@pytest.mark.parametrize("usage_mask", ["31", "33", "35", "37", "291", "293"]) +def test_unrelated_usage_masks_are_not_queried(usage_mask): + usage_filter = 'eq(ethpmPhysIf.usage,"{}")'.format(usage_mask) + assert usage_filter not in ethpmPhysIf_api From 3c9ecea88514b090a4e16fe5373662c9251e3251 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 11:11:50 -0400 Subject: [PATCH 13/44] Fix APIC database collection false passes (#327) Report missing, empty, unavailable, or malformed mitmocounters data as an explicit error instead of passing incomplete collection. Preserve any oversized-class findings and add regression coverage for sorting and threshold behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 68 ++++++- .../test_apic_database_size_check.py | 187 ++++++++++++++++-- 2 files changed, 227 insertions(+), 28 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 5e79f56e..2436a530 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6102,6 +6102,7 @@ def apic_database_size_check(cversion, **kwargs): dme_svc_list = ['vmmmgr', 'policymgr', 'eventmgr', 'policydist'] unique_list = {} + collection_errors = [] apic_id_to_name = {} apic_node_mo = icurl('class', 'infraWiNode.json') for apic in apic_node_mo: @@ -6119,15 +6120,49 @@ def apic_database_size_check(cversion, **kwargs): for dme in dme_svc_list: for id in apic_id_to_name: apic_hostname = apic_id_to_name[id] - collect_stats_cmd = 'cat /debug/'+apic_hostname+'/'+dme+'/mitmocounters/mo | grep -v ALL | sort -rn -k3' - top_class_stats = run_cmd(collect_stats_cmd, splitlines=True) - - for svc_stats in top_class_stats[:4]: - if ":" in svc_stats: - class_name = svc_stats.split(":")[0].strip() - mo_count = svc_stats.split(":")[1].strip() - if int(mo_count) > 1000*1000*1.5: - unique_list[class_name] = {"id": id, "dme": dme, "checked_val": mo_count} + counter_file = '/debug/'+apic_hostname+'/'+dme+'/mitmocounters/mo' + collect_stats_cmd = 'cat ' + counter_file + ' 2>/dev/null' + try: + class_stats = run_cmd(collect_stats_cmd, splitlines=True) + except subprocess.CalledProcessError: + collection_errors.append([id, dme, 'Counter file is unavailable']) + continue + + parsed_class_stats = [] + malformed_stats = False + for stats in class_stats: + stats = stats.strip() + if not stats or 'ALL' in stats: + continue + if ':' not in stats: + malformed_stats = True + continue + class_name, mo_count = stats.split(':', 1) + class_name = class_name.strip() + if not class_name: + malformed_stats = True + continue + try: + mo_count = int(mo_count.strip()) + except ValueError: + malformed_stats = True + continue + parsed_class_stats.append((mo_count, class_name)) + + if malformed_stats: + collection_errors.append([id, dme, 'Counter data is malformed']) + if not parsed_class_stats and not malformed_stats: + collection_errors.append([id, dme, 'Counter file is missing or empty']) + continue + + top_class_stats = sorted(parsed_class_stats, reverse=True) + for mo_count, class_name in top_class_stats[:4]: + if mo_count > 1000*1000*1.5: + unique_list[class_name] = { + "id": id, + "dme": dme, + "checked_val": str(mo_count), + } else: headers = ["APIC ID", "DME", "Shard", "Size"] recommended_action = 'Contact Cisco TAC to investigate all flagged large DB sizes' @@ -6155,6 +6190,21 @@ def apic_database_size_check(cversion, **kwargs): checked_val = details['checked_val'] data.append([apic_id, dme, unique_key, checked_val]) + if collection_errors: + return Result( + result=ERROR, + msg='Unable to collect APIC database object counters', + headers=['APIC ID', 'DME', 'Collection Error'], + data=collection_errors, + unformatted_headers=headers, + unformatted_data=data, + recommended_action=( + 'Retry the check. Contact Cisco TAC to investigate any flagged ' + 'high object counts or persistent collection errors.' + ), + doc_url=doc_url, + ) + if data: result = FAIL_UF return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) diff --git a/tests/checks/apic_database_size_check/test_apic_database_size_check.py b/tests/checks/apic_database_size_check/test_apic_database_size_check.py index 8a651bc7..1aa2d943 100644 --- a/tests/checks/apic_database_size_check/test_apic_database_size_check.py +++ b/tests/checks/apic_database_size_check/test_apic_database_size_check.py @@ -13,25 +13,25 @@ apic_node_api = 'infraWiNode.json' -apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic1_vmm_cat = "cat /debug/apic1/vmmmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic1_evm_cat = "cat /debug/apic1/eventmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" - -apic2_pm_cat = "cat /debug/apic2/policymgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic2_pd_cat = "cat /debug/apic2/policydist/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic2_vmm_cat = "cat /debug/apic2/vmmmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic2_evm_cat = "cat /debug/apic2/eventmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" - -apic3_pm_cat = "cat /debug/apic3/policymgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic3_pd_cat = "cat /debug/apic3/policydist/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic3_vmm_cat = "cat /debug/apic3/vmmmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic3_evm_cat = "cat /debug/apic3/eventmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" - -apic4_pm_cat = "cat /debug/apic4/policymgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic4_pd_cat = "cat /debug/apic4/policydist/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic4_vmm_cat = "cat /debug/apic4/vmmmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" -apic4_evm_cat = "cat /debug/apic4/eventmgr/mitmocounters/mo | grep -v ALL | sort -rn -k3" +apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo 2>/dev/null" +apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo 2>/dev/null" +apic1_vmm_cat = "cat /debug/apic1/vmmmgr/mitmocounters/mo 2>/dev/null" +apic1_evm_cat = "cat /debug/apic1/eventmgr/mitmocounters/mo 2>/dev/null" + +apic2_pm_cat = "cat /debug/apic2/policymgr/mitmocounters/mo 2>/dev/null" +apic2_pd_cat = "cat /debug/apic2/policydist/mitmocounters/mo 2>/dev/null" +apic2_vmm_cat = "cat /debug/apic2/vmmmgr/mitmocounters/mo 2>/dev/null" +apic2_evm_cat = "cat /debug/apic2/eventmgr/mitmocounters/mo 2>/dev/null" + +apic3_pm_cat = "cat /debug/apic3/policymgr/mitmocounters/mo 2>/dev/null" +apic3_pd_cat = "cat /debug/apic3/policydist/mitmocounters/mo 2>/dev/null" +apic3_vmm_cat = "cat /debug/apic3/vmmmgr/mitmocounters/mo 2>/dev/null" +apic3_evm_cat = "cat /debug/apic3/eventmgr/mitmocounters/mo 2>/dev/null" + +apic4_pm_cat = "cat /debug/apic4/policymgr/mitmocounters/mo 2>/dev/null" +apic4_pd_cat = "cat /debug/apic4/policydist/mitmocounters/mo 2>/dev/null" +apic4_vmm_cat = "cat /debug/apic4/vmmmgr/mitmocounters/mo 2>/dev/null" +apic4_evm_cat = "cat /debug/apic4/eventmgr/mitmocounters/mo 2>/dev/null" apic1_acidiag = "acidiag dbsize --topshard --apic 1 -f json" apic2_acidiag = "acidiag dbsize --topshard --apic 2 -f json" @@ -334,3 +334,152 @@ def test_permission_logic(run_check, mock_icurl, mock_run_cmd, cversion, expecte cversion=script.AciVersion(cversion) if cversion else None ) assert result.result == expected_result + + +@pytest.mark.parametrize( + "failure_details,expected_error", + [ + ({"splitlines": True, "output": ""}, "Counter file is missing or empty"), + ({"CalledProcessError": True}, "Counter file is unavailable"), + ], +) +def test_missing_mitmocounters_returns_error( + run_check, + mock_icurl, + mock_run_cmd, + icurl_outputs, + cmd_outputs, + failure_details, + expected_error, +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + cmd_outputs.clear() + cmd_outputs.update({ + apic2_pm_cat: failure_details, + apic2_pd_cat: failure_details, + apic2_vmm_cat: failure_details, + apic2_evm_cat: failure_details, + }) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.ERROR + assert result.msg == "Unable to collect APIC database object counters" + assert result.headers == ["APIC ID", "DME", "Collection Error"] + assert len(result.data) == 4 + assert all(row[2] == expected_error for row in result.data) + + +def test_collection_error_preserves_oversized_classes( + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + cmd_outputs.clear() + cmd_outputs.update({ + apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, + apic2_pm_cat: {"CalledProcessError": True}, + apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, + apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, + }) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.ERROR + assert result.data == [["2", "policymgr", "Counter file is unavailable"]] + assert result.unformatted_headers == [ + "APIC ID", "DME", "Class Name", "Object Count" + ] + assert sorted(result.unformatted_data) == sorted([ + ["2", "vmmmgr", "compProv", "1800000"], + ["2", "vmmmgr", "compatCtlrFw", "1700000"], + ["2", "vmmmgr", "aaaIRbacRule", "1600000"], + ]) + assert "high object counts" in result.recommended_action + + +def test_object_counters_are_sorted_before_top_four_and_thresholded( + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs +): + unsorted_counters = """ +belowOne : 1 +atThreshold : 1500000 +belowTwo : 2 +highest : 1600000 +aboveThreshold : 1500001 +""" + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + cmd_outputs.clear() + cmd_outputs.update({ + apic2_vmm_cat: {"splitlines": True, "output": unsorted_counters}, + apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg}, + apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, + apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, + }) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.FAIL_UF + assert result.headers == ["APIC ID", "DME", "Class Name", "Object Count"] + assert sorted(result.data) == sorted([ + ["2", "vmmmgr", "highest", "1600000"], + ["2", "vmmmgr", "aboveThreshold", "1500001"], + ]) + + +def test_malformed_counter_preserves_oversized_classes( + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + cmd_outputs.clear() + cmd_outputs.update({ + apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, + apic2_pm_cat: {"splitlines": True, "output": "brokenClass :"}, + apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, + apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, + }) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.ERROR + assert result.data == [["2", "policymgr", "Counter data is malformed"]] + assert sorted(result.unformatted_data) == sorted([ + ["2", "vmmmgr", "compProv", "1800000"], + ["2", "vmmmgr", "compatCtlrFw", "1700000"], + ["2", "vmmmgr", "aaaIRbacRule", "1600000"], + ]) + + +def test_colonless_counter_data_returns_error( + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + cmd_outputs.clear() + cmd_outputs.update({ + apic2_vmm_cat: { + "splitlines": True, + "output": "validClass : 10\ntruncatedClass", + }, + apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg}, + apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, + apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, + }) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.ERROR + assert result.data == [["2", "vmmmgr", "Counter data is malformed"]] From 029d4d41594873f4a24f8738012946851b2f678f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 12:36:21 -0400 Subject: [PATCH 14/44] Retry transient APIC database counter reads (#327) Retry each legacy mitmocounters read up to three times and capture the final command error when collection remains unavailable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 40 ++++++- .../test_apic_database_size_check.py | 101 ++++++++++++++---- tests/checks/conftest.py | 6 +- 3 files changed, 121 insertions(+), 26 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 2436a530..418dfd88 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6101,6 +6101,8 @@ def apic_database_size_check(cversion, **kwargs): doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#apic-database-size' dme_svc_list = ['vmmmgr', 'policymgr', 'eventmgr', 'policydist'] + counter_read_attempts = 3 + counter_read_retry_delay = 1 unique_list = {} collection_errors = [] apic_id_to_name = {} @@ -6121,11 +6123,39 @@ def apic_database_size_check(cversion, **kwargs): for id in apic_id_to_name: apic_hostname = apic_id_to_name[id] counter_file = '/debug/'+apic_hostname+'/'+dme+'/mitmocounters/mo' - collect_stats_cmd = 'cat ' + counter_file + ' 2>/dev/null' - try: - class_stats = run_cmd(collect_stats_cmd, splitlines=True) - except subprocess.CalledProcessError: - collection_errors.append([id, dme, 'Counter file is unavailable']) + collect_stats_cmd = 'cat ' + counter_file + ' 2>&1' + class_stats = None + final_error = None + for attempt in range(1, counter_read_attempts + 1): + try: + class_stats = run_cmd(collect_stats_cmd, splitlines=True) + break + except subprocess.CalledProcessError as error: + error_output = error.output + if isinstance(error_output, bytes): + error_output = error_output.decode('utf-8', 'replace') + final_error = (error_output or str(error)).strip() + if attempt < counter_read_attempts: + log.warning( + 'Counter read failed for APIC %s %s ' + '(attempt %s/%s): %s', + id, + dme, + attempt, + counter_read_attempts, + final_error, + ) + time.sleep(counter_read_retry_delay) + + if class_stats is None: + collection_errors.append([ + id, + dme, + 'Counter file is unavailable after %s attempts: %s' % ( + counter_read_attempts, + final_error, + ), + ]) continue parsed_class_stats = [] diff --git a/tests/checks/apic_database_size_check/test_apic_database_size_check.py b/tests/checks/apic_database_size_check/test_apic_database_size_check.py index 1aa2d943..f1cbeb54 100644 --- a/tests/checks/apic_database_size_check/test_apic_database_size_check.py +++ b/tests/checks/apic_database_size_check/test_apic_database_size_check.py @@ -2,6 +2,7 @@ import pytest import logging import importlib +from subprocess import CalledProcessError from helpers.utils import read_data script = importlib.import_module("aci-preupgrade-validation-script") @@ -13,25 +14,25 @@ apic_node_api = 'infraWiNode.json' -apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo 2>/dev/null" -apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo 2>/dev/null" -apic1_vmm_cat = "cat /debug/apic1/vmmmgr/mitmocounters/mo 2>/dev/null" -apic1_evm_cat = "cat /debug/apic1/eventmgr/mitmocounters/mo 2>/dev/null" +apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo 2>&1" +apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo 2>&1" +apic1_vmm_cat = "cat /debug/apic1/vmmmgr/mitmocounters/mo 2>&1" +apic1_evm_cat = "cat /debug/apic1/eventmgr/mitmocounters/mo 2>&1" -apic2_pm_cat = "cat /debug/apic2/policymgr/mitmocounters/mo 2>/dev/null" -apic2_pd_cat = "cat /debug/apic2/policydist/mitmocounters/mo 2>/dev/null" -apic2_vmm_cat = "cat /debug/apic2/vmmmgr/mitmocounters/mo 2>/dev/null" -apic2_evm_cat = "cat /debug/apic2/eventmgr/mitmocounters/mo 2>/dev/null" +apic2_pm_cat = "cat /debug/apic2/policymgr/mitmocounters/mo 2>&1" +apic2_pd_cat = "cat /debug/apic2/policydist/mitmocounters/mo 2>&1" +apic2_vmm_cat = "cat /debug/apic2/vmmmgr/mitmocounters/mo 2>&1" +apic2_evm_cat = "cat /debug/apic2/eventmgr/mitmocounters/mo 2>&1" -apic3_pm_cat = "cat /debug/apic3/policymgr/mitmocounters/mo 2>/dev/null" -apic3_pd_cat = "cat /debug/apic3/policydist/mitmocounters/mo 2>/dev/null" -apic3_vmm_cat = "cat /debug/apic3/vmmmgr/mitmocounters/mo 2>/dev/null" -apic3_evm_cat = "cat /debug/apic3/eventmgr/mitmocounters/mo 2>/dev/null" +apic3_pm_cat = "cat /debug/apic3/policymgr/mitmocounters/mo 2>&1" +apic3_pd_cat = "cat /debug/apic3/policydist/mitmocounters/mo 2>&1" +apic3_vmm_cat = "cat /debug/apic3/vmmmgr/mitmocounters/mo 2>&1" +apic3_evm_cat = "cat /debug/apic3/eventmgr/mitmocounters/mo 2>&1" -apic4_pm_cat = "cat /debug/apic4/policymgr/mitmocounters/mo 2>/dev/null" -apic4_pd_cat = "cat /debug/apic4/policydist/mitmocounters/mo 2>/dev/null" -apic4_vmm_cat = "cat /debug/apic4/vmmmgr/mitmocounters/mo 2>/dev/null" -apic4_evm_cat = "cat /debug/apic4/eventmgr/mitmocounters/mo 2>/dev/null" +apic4_pm_cat = "cat /debug/apic4/policymgr/mitmocounters/mo 2>&1" +apic4_pd_cat = "cat /debug/apic4/policydist/mitmocounters/mo 2>&1" +apic4_vmm_cat = "cat /debug/apic4/vmmmgr/mitmocounters/mo 2>&1" +apic4_evm_cat = "cat /debug/apic4/eventmgr/mitmocounters/mo 2>&1" apic1_acidiag = "acidiag dbsize --topshard --apic 1 -f json" apic2_acidiag = "acidiag dbsize --topshard --apic 2 -f json" @@ -340,7 +341,17 @@ def test_permission_logic(run_check, mock_icurl, mock_run_cmd, cversion, expecte "failure_details,expected_error", [ ({"splitlines": True, "output": ""}, "Counter file is missing or empty"), - ({"CalledProcessError": True}, "Counter file is unavailable"), + ( + { + "CalledProcessError": True, + "returncode": 1, + "error_output": b"cat: file: No such file or directory\n", + }, + ( + "Counter file is unavailable after 3 attempts: " + "cat: file: No such file or directory" + ), + ), ], ) def test_missing_mitmocounters_returns_error( @@ -351,6 +362,7 @@ def test_missing_mitmocounters_returns_error( cmd_outputs, failure_details, expected_error, + monkeypatch, ): icurl_outputs.clear() icurl_outputs.update({ @@ -363,6 +375,7 @@ def test_missing_mitmocounters_returns_error( apic2_vmm_cat: failure_details, apic2_evm_cat: failure_details, }) + monkeypatch.setattr(script.time, "sleep", lambda _: None) result = run_check(cversion=script.AciVersion("6.0(8f)")) @@ -374,7 +387,7 @@ def test_missing_mitmocounters_returns_error( def test_collection_error_preserves_oversized_classes( - run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs, monkeypatch ): icurl_outputs.clear() icurl_outputs.update({ @@ -383,15 +396,26 @@ def test_collection_error_preserves_oversized_classes( cmd_outputs.clear() cmd_outputs.update({ apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, - apic2_pm_cat: {"CalledProcessError": True}, + apic2_pm_cat: { + "CalledProcessError": True, + "error_output": b"cat: file: No such file or directory\n", + }, apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, }) + monkeypatch.setattr(script.time, "sleep", lambda _: None) result = run_check(cversion=script.AciVersion("6.0(8f)")) assert result.result == script.ERROR - assert result.data == [["2", "policymgr", "Counter file is unavailable"]] + assert result.data == [[ + "2", + "policymgr", + ( + "Counter file is unavailable after 3 attempts: " + "cat: file: No such file or directory" + ), + ]] assert result.unformatted_headers == [ "APIC ID", "DME", "Class Name", "Object Count" ] @@ -403,6 +427,43 @@ def test_collection_error_preserves_oversized_classes( assert "high object counts" in result.recommended_action +def test_transient_counter_read_succeeds_on_retry( + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs, monkeypatch +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + successful_outputs = { + apic2_vmm_cat: mitcounters_neg, + apic2_pm_cat: mitcounters_neg, + apic2_evm_cat: mitcounters_neg, + apic2_pd_cat: mitcounters_neg, + } + call_counts = {} + sleep_calls = [] + + def transient_run_cmd(cmd, splitlines=False): + call_counts[cmd] = call_counts.get(cmd, 0) + 1 + if call_counts[cmd] == 1: + raise CalledProcessError( + 1, + cmd, + output=b"cat: file: No such file or directory\n", + ) + output = successful_outputs[cmd] + return output.splitlines() if splitlines else output + + monkeypatch.setattr(script, "run_cmd", transient_run_cmd) + monkeypatch.setattr(script.time, "sleep", sleep_calls.append) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.PASS + assert all(call_count == 2 for call_count in call_counts.values()) + assert sleep_calls.count(1) == 4 + + def test_object_counters_are_sorted_before_top_four_and_thresholded( run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs ): diff --git a/tests/checks/conftest.py b/tests/checks/conftest.py index 01cd8870..8f883dcf 100644 --- a/tests/checks/conftest.py +++ b/tests/checks/conftest.py @@ -121,7 +121,11 @@ def _mock_run_cmd(cmd, splitlines=False): log.error("Command `%s` not found in test data", cmd) return "" if details.get("CalledProcessError"): - raise CalledProcessError(127, cmd) + raise CalledProcessError( + details.get("returncode", 127), + cmd, + output=details.get("error_output"), + ) splitlines = details.get("splitlines", False) output = details.get("output") From 8f43dc4a9ef818a6edc59994d25fcb2ece54492f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 23 Jul 2026 15:50:39 -0400 Subject: [PATCH 15/44] Allow supported CIMC 4.2(3e) for APIC 6.1(5) (#338) Honor the APIC 6.1(5) release-note support exception for M5/M6 controllers while preserving the CSCwo74485 ordering gate and catalog failures for other versions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 7 +++- docs/docs/validations.md | 3 +- .../compatRsSuppHw_615_M5.json | 10 +++++ .../compatRsSuppHw_615_M6.json | 10 +++++ .../eqptCh_615_supported_423e.json | 42 +++++++++++++++++++ .../eqptCh_615_unsupported_423d.json | 22 ++++++++++ .../test_cimc_compatibilty_check.py | 35 ++++++++++++++++ 7 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M5.json create mode 100644 tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M6.json create mode 100644 tests/checks/cimc_compatibilty_check/eqptCh_615_supported_423e.json create mode 100644 tests/checks/cimc_compatibilty_check/eqptCh_615_unsupported_423d.json diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 5e79f56e..8b5af5cb 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3780,7 +3780,12 @@ def cimc_compatibilty_check(tversion, cversion, **kwargs): if compatMo and recommended_cimc: if not is_firstver_gt_secondver(current_cimc, "3.0(3a)"): warning = "Multi-step Upgrade may be required, check UCS CIMC Matrix." - if not is_firstver_gt_secondver(current_cimc, recommended_cimc): + release_note_supported = ( + tversion.simple_version == "6.1(5)" + and model in ("apicl3", "apicm3", "apicl4", "apicm4") + and current_cimc == "4.2(3e)" + ) + if not release_note_supported and not is_firstver_gt_secondver(current_cimc, recommended_cimc): nodeid = eqptCh['eqptCh']['attributes']['dn'].split('/')[2] data.append([nodeid, apic_model, current_cimc, recommended_cimc, warning]) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f7886811..e47f6b2e 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -260,6 +260,8 @@ The script checks the minimum recommended CIMC version for the given APIC model As the `compatRsSuppHw` object recommendation is strictly tied to the target software image, it is possible that the [Release Note Documentation][4] for your model/target version has a different recommendation than what the software recommends. Always check the release note of your Target version and APIC model to ensure you are getting the latest recommendations. +The APIC 6.1(5) release notes explicitly support CIMC 4.2(3e) on UCS C220/C240 M5 (APIC-L3/M3) and UCS C225 M6 (APIC-L4/M4). The check accepts that exact release-note-supported combination even when the image catalog recommends a newer CIMC version. + Due to the defect CSCwo74485, APIC-SERVER-M4/L4 systems will fail to boot correctly after upgrading CIMC firmware to version 4.3.5 or later while on Non-fixed APIC releases 5.3.x/6.0.9d/6.1(3g) and below. Upgrade the APIC software first, then proceed with the CIMC upgrade for the releases 6.0.9e/ 6.1.4h and above, will avoid this issue. Follow the software advisory for this defect [CSCwo74485][73]. !!! note @@ -2923,4 +2925,3 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [ [74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 - diff --git a/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M5.json b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M5.json new file mode 100644 index 00000000..e66e5870 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M5.json @@ -0,0 +1,10 @@ +[ + { + "compatRsSuppHw": { + "attributes": { + "cimcVersion": "4.3(2.250016)", + "dn": "uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm3]" + } + } + } +] diff --git a/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M6.json b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M6.json new file mode 100644 index 00000000..74f8a93e --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/compatRsSuppHw_615_M6.json @@ -0,0 +1,10 @@ +[ + { + "compatRsSuppHw": { + "attributes": { + "cimcVersion": "4.3(4.252002)", + "dn": "uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4]" + } + } + } +] diff --git a/tests/checks/cimc_compatibilty_check/eqptCh_615_supported_423e.json b/tests/checks/cimc_compatibilty_check/eqptCh_615_supported_423e.json new file mode 100644 index 00000000..ce34bb09 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/eqptCh_615_supported_423e.json @@ -0,0 +1,42 @@ +[ + { + "eqptCh": { + "attributes": { + "cimcVersion": "4.2(3e)", + "descr": "APIC-SERVER-L3", + "dn": "topology/pod-1/node-1/sys/ch", + "model": "APIC-SERVER-L3" + } + } + }, + { + "eqptCh": { + "attributes": { + "cimcVersion": "4.2(3e)", + "descr": "APIC-SERVER-M3", + "dn": "topology/pod-1/node-2/sys/ch", + "model": "APIC-SERVER-M3" + } + } + }, + { + "eqptCh": { + "attributes": { + "cimcVersion": "4.2(3e)", + "descr": "APIC-SERVER-L4", + "dn": "topology/pod-1/node-3/sys/ch", + "model": "APIC-SERVER-L4" + } + } + }, + { + "eqptCh": { + "attributes": { + "cimcVersion": "4.2(3e)", + "descr": "APIC-SERVER-M4", + "dn": "topology/pod-1/node-4/sys/ch", + "model": "APIC-SERVER-M4" + } + } + } +] diff --git a/tests/checks/cimc_compatibilty_check/eqptCh_615_unsupported_423d.json b/tests/checks/cimc_compatibilty_check/eqptCh_615_unsupported_423d.json new file mode 100644 index 00000000..a7cdf801 --- /dev/null +++ b/tests/checks/cimc_compatibilty_check/eqptCh_615_unsupported_423d.json @@ -0,0 +1,22 @@ +[ + { + "eqptCh": { + "attributes": { + "cimcVersion": "4.2(3d)", + "descr": "APIC-SERVER-M3", + "dn": "topology/pod-1/node-1/sys/ch", + "model": "APIC-SERVER-M3" + } + } + }, + { + "eqptCh": { + "attributes": { + "cimcVersion": "4.2(3d)", + "descr": "APIC-SERVER-M4", + "dn": "topology/pod-1/node-2/sys/ch", + "model": "APIC-SERVER-M4" + } + } + } +] diff --git a/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py b/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py index 4db1b7ea..6c8b14f2 100644 --- a/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py +++ b/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py @@ -21,10 +21,45 @@ compatRsSuppHwM4_605_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.0(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4].json' compatRsSuppHwL4_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl4].json' compatRsSuppHwM4_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm4].json' +compatRsSuppHwL3_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicl3].json' +compatRsSuppHwM3_api = 'uni/fabric/compcat-default/ctlrfw-apic-6.1(5)/rssuppHw-[uni/fabric/compcat-default/ctlrhw-apicm3].json' + +release_note_supported_615_outputs = { + eqptCh_api: read_data(dir, "eqptCh_615_supported_423e.json"), + compatRsSuppHwL3_api: read_data(dir, "compatRsSuppHw_615_M5.json"), + compatRsSuppHwM3_api: read_data(dir, "compatRsSuppHw_615_M5.json"), + compatRsSuppHwL4_api: read_data(dir, "compatRsSuppHw_615_M6.json"), + compatRsSuppHwM4_api: read_data(dir, "compatRsSuppHw_615_M6.json"), +} @pytest.mark.parametrize( "icurl_outputs, tversion, cversion, expected_result", [ + # CIMC 4.2(3e) is explicitly supported for M5/M6 APICs by the 6.1(5) release notes. + ( + release_note_supported_615_outputs, + "6.1(5e)", + "5.2(8g)", + script.PASS, + ), + # The release-note exception must not bypass the CSCwo74485 upgrade ordering check. + ( + release_note_supported_615_outputs, + "6.1(5e)", + "5.3(1d)", + script.FAIL_UF, + ), + # Other CIMC versions below the catalog recommendation remain unsupported. + ( + { + eqptCh_api: read_data(dir, "eqptCh_615_unsupported_423d.json"), + compatRsSuppHwM3_api: read_data(dir, "compatRsSuppHw_615_M5.json"), + compatRsSuppHwM4_api: read_data(dir, "compatRsSuppHw_615_M6.json"), + }, + "6.1(5e)", + "5.2(8g)", + script.FAIL_UF, + ), #m4/l4 model check and targeting affected version and cversion affected and cimc < 4.3.5 ( {eqptCh_api: read_data(dir, "eqptCh_m4l4_model_old_cimc.json"), From b0da7a5cc81820cea0840141cf62863ddb6e47eb Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 24 Jul 2026 09:29:24 -0400 Subject: [PATCH 16/44] Added validation for low RTC battery fault F2421 (#351) Block upgrades when an active F2421 reports the exact RTC battery voltage low reason, while ignoring stale and unrelated fault instances. Include issue-derived coverage and validation guidance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 54 +++++++++++++ docs/docs/validations.md | 13 +++- .../rtc_battery_voltage_low_check/f2421.json | 37 +++++++++ .../test_rtc_battery_voltage_low_check.py | 78 +++++++++++++++++++ 4 files changed, 179 insertions(+), 3 deletions(-) create mode 100644 tests/checks/rtc_battery_voltage_low_check/f2421.json create mode 100644 tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 5e79f56e..64bf2aa5 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6404,6 +6404,59 @@ def apic_storage_inode_check(**kwargs): return Result(result=result, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) +@check_wrapper(check_title="Switch RTC Battery Voltage (F2421 equipment-diags-failed)") +def rtc_battery_voltage_low_check(**kwargs): + result = FAIL_UF + headers = ['Fault', 'Pod', 'Node', 'Supervisor', 'Severity', 'Lifecycle'] + data = [] + unformatted_headers = ['Fault', 'Fault DN', 'Description', 'Severity', 'Lifecycle'] + unformatted_data = [] + recommended_action = 'Contact Cisco TAC to replace the RTC battery before upgrading or power cycling the affected switch' + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-rtc-battery-voltage' + dn_regex = node_regex + r'/.+/supslot-(?P\d+)/sup\]/fault-F2421$' + fault_reason = 'The RTC battery voltage is low' + fault_api = 'faultInst.json?query-target-filter=and(eq(faultInst.code,"F2421"),wcard(faultInst.descr,"reason:The RTC battery voltage is low"))' + + faultInsts = icurl('class', fault_api) + for faultInst in faultInsts: + attributes = faultInst['faultInst']['attributes'] + lc = attributes['lc'] + if lc not in ['raised', 'soaking']: + continue + description = attributes['descr'] + if 'reason:' not in description or description.split('reason:', 1)[1] != fault_reason: + continue + dn = re.search(dn_regex, attributes['dn']) + if dn: + data.append([ + attributes['code'], + dn.group('pod'), + dn.group('node'), + dn.group('slot'), + attributes['severity'], + lc, + ]) + else: + unformatted_data.append([ + attributes['code'], + attributes['dn'], + description, + attributes['severity'], + lc, + ]) + if not data and not unformatted_data: + result = PASS + return Result( + result=result, + headers=headers, + data=data, + unformatted_headers=unformatted_headers, + unformatted_data=unformatted_data, + recommended_action=recommended_action, + doc_url=doc_url, + ) + + # Connection Based Check @check_wrapper(check_title="Multi-Pod Modular Spine Bootscript File") def multipod_modular_spine_bootscript_check(tversion, fabric_nodes, username, password, **kwargs): @@ -6816,6 +6869,7 @@ class CheckManager: equipment_disk_limits_exceeded, apic_vmm_inventory_sync_faults_check, apic_storage_inode_check, + rtc_battery_voltage_low_check, # Configurations vpc_paired_switches_check, diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f7886811..e2b3ab4d 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -87,6 +87,7 @@ Items | Faults | This Script [Equipment Disk Limits][f20] | F1820: 80% -minor
F1821: -major
F1822: -critical | :white_check_mark: | :no_entry_sign: [VMM Inventory Partially Synced][f21] | F0132: comp-ctrlr-operational-issues | :white_check_mark: | :no_entry_sign: [APIC Storage Inode Usage][f22] | F4388: 75% - 85% -warning
F4389: 85% - 90% -major
F4390: 90% or more -critical | :white_check_mark: | :no_entry_sign: +[Switch RTC Battery Voltage][f23] | F2421: RTC battery voltage is low | :white_check_mark: | :no_entry_sign: [f1]: #apic-disk-space-usage [f2]: #standby-apic-disk-space-usage @@ -110,6 +111,7 @@ Items | Faults | This Script [f20]: #equipment-disk-limits [f21]: #vmm-inventory-partially-synced [f22]: #apic-storage-inode-usage +[f23]: #switch-rtc-battery-voltage ### Configuration Checks @@ -1638,8 +1640,14 @@ To recover from this fault, try the following action subject : equipment-full type : operational ``` - - + +### Switch RTC Battery Voltage + +This check detects active F2421 equipment diagnostic faults whose reason is `The RTC battery voltage is low`. The RTC battery maintains the switch system clock while the switch is powered off. If the battery voltage is low, a power cycle during an upgrade can reset the clock and prevent certificate validation, which can stop the switch from rejoining the fabric. + +The RTC battery should be replaced before upgrading or power cycling an affected switch. Contact Cisco TAC to coordinate replacement and confirm that the fault has cleared. + + ## Configuration Check Details ### VPC-paired Leaf switches @@ -2923,4 +2931,3 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [ [74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 - diff --git a/tests/checks/rtc_battery_voltage_low_check/f2421.json b/tests/checks/rtc_battery_voltage_low_check/f2421.json new file mode 100644 index 00000000..defb7c0b --- /dev/null +++ b/tests/checks/rtc_battery_voltage_low_check/f2421.json @@ -0,0 +1,37 @@ +{ + "totalCount": "1", + "imdata": [ + { + "faultInst": { + "attributes": { + "ack": "no", + "alert": "no", + "cause": "equipment-diags-failed", + "changeSet": "operStQual:The RTC battery voltage is low", + "childAction": "", + "code": "F2421", + "created": "2026-02-09T16:27:10.975+00:00", + "delegated": "no", + "descr": "Diagnostics test failed. reason:The RTC battery voltage is low", + "dn": "topology/pod-1/node-122/sys/diag/rule-rtc-test-trig-forever/subj-[topology/pod-1/node-122/sys/ch/supslot-1/sup]/fault-F2421", + "domain": "infra", + "extMngdBy": "", + "highestSeverity": "minor", + "lastTransition": "2026-02-09T16:27:10.975+00:00", + "lc": "raised", + "modTs": "never", + "occur": "1", + "origSeverity": "minor", + "prevSeverity": "cleared", + "rn": "fault-F2421", + "rule": "diag-rtctest-failed", + "severity": "minor", + "status": "", + "subject": "equipment-diags-failed", + "title": "", + "type": "operational" + } + } + } + ] +} diff --git a/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py b/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py new file mode 100644 index 00000000..17a44766 --- /dev/null +++ b/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py @@ -0,0 +1,78 @@ +import copy +import importlib +import logging +import os + +import pytest + +from helpers.utils import read_data + + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) +test_function = "rtc_battery_voltage_low_check" +faultInst_api = ( + 'faultInst.json?query-target-filter=and(eq(faultInst.code,"F2421"),' + 'wcard(faultInst.descr,"reason:The RTC battery voltage is low"))' +) +active_faults = read_data(dir, "f2421.json")["imdata"] +retaining_faults = copy.deepcopy(active_faults) +retaining_faults[0]["faultInst"]["attributes"]["lc"] = "retaining" +malformed_faults = copy.deepcopy(active_faults) +malformed_faults[0]["faultInst"]["attributes"]["dn"] = ( + "topology/pod-1/node-122/sys/diag/rule-rtc-test-trig-forever/" + "subj-[topology/pod-1/node-122/sys/ch/supslot-1/sup-bad]/fault-F2421" +) +non_exact_reason_faults = [] +for reason in [ + "The RTC battery voltage is low or unavailable", + "Warning: The RTC battery voltage is low", + "Power-on self-test failed", +]: + fault = copy.deepcopy(active_faults[0]) + fault["faultInst"]["attributes"]["descr"] = ( + "Diagnostics test failed. reason:" + reason + ) + non_exact_reason_faults.append(fault) + + +@pytest.mark.parametrize( + "icurl_outputs, expected_result, expected_data, expected_unformatted_data", + [ + ( + {faultInst_api: active_faults}, + script.FAIL_UF, + [["F2421", "1", "122", "1", "minor", "raised"]], + [], + ), + ({faultInst_api: []}, script.PASS, [], []), + ({faultInst_api: retaining_faults}, script.PASS, [], []), + ({faultInst_api: non_exact_reason_faults}, script.PASS, [], []), + ( + {faultInst_api: active_faults + malformed_faults}, + script.FAIL_UF, + [["F2421", "1", "122", "1", "minor", "raised"]], + [[ + "F2421", + "topology/pod-1/node-122/sys/diag/rule-rtc-test-trig-forever/" + "subj-[topology/pod-1/node-122/sys/ch/supslot-1/sup-bad]/fault-F2421", + "Diagnostics test failed. reason:The RTC battery voltage is low", + "minor", + "raised", + ]], + ), + ], +) +def test_logic( + run_check, + mock_icurl, + expected_result, + expected_data, + expected_unformatted_data, +): + result = run_check() + assert result.result == expected_result + assert result.data == expected_data + assert result.unformatted_data == expected_unformatted_data From 0d222c080f1987aa37a5a685906170c959998fc4 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 24 Jul 2026 12:37:42 -0400 Subject: [PATCH 17/44] docs: clarify recommended action metadata Explain how APIC 6.2+ consumes recommended_action and when standalone users should act on it. Fixes #389. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/docs/usage.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/docs/usage.md b/docs/docs/usage.md index e3f05889..556b4c43 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -71,6 +71,12 @@ Each check has a unique result which will help determine how to proceed. The res - **N/A** - The check completed successfully, and the ACI fabric is not susceptible because the needed configuration is not deployed. - **ERROR** - The check did not complete successfully, and needs further investigation. +### Recommended action in JSON results + +The per-check JSON schema was added for the built-in pre-upgrade validation workflow in APIC 6.2 and later. The APIC presentation layer expects `recommended_action` as static rule metadata, so the field can be populated even when `ruleStatus` is `passed`. + +Use `ruleStatus` to determine whether the recommendation is actionable. A populated `recommended_action` does not indicate a problem when `ruleStatus` is `passed`; standalone consumers should act on it only when `ruleStatus` is `failed`. + ## Logs A single log bundle will be generated with each run of the script From 9a362e4471c5dab9062101874028a1eb1e0b608d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 12:25:06 -0400 Subject: [PATCH 18/44] docs: link APIC 6.2 validator reference Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/usage.md b/docs/docs/usage.md index 556b4c43..934ded00 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -73,7 +73,7 @@ Each check has a unique result which will help determine how to proceed. The res ### Recommended action in JSON results -The per-check JSON schema was added for the built-in pre-upgrade validation workflow in APIC 6.2 and later. The APIC presentation layer expects `recommended_action` as static rule metadata, so the field can be populated even when `ruleStatus` is `passed`. +The per-check JSON schema was added for the built-in pre-upgrade validation workflow in APIC 6.2 and later. The APIC presentation layer expects `recommended_action` as static rule metadata, so the field can be populated even when `ruleStatus` is `passed`. See Cisco’s [Pre-upgrade validator examples for APIC 6.2(1)](https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/g-pre-upgrade-checklists/examples-of-pre-upgrade-validator-apic.html#Cisco_Reference.dita_53db0ff6-cc90-4543-80e0-091fc41dc962__section_vq4_5lv_qhc). Use `ruleStatus` to determine whether the recommendation is actionable. A populated `recommended_action` does not indicate a problem when `ruleStatus` is `passed`; standalone consumers should act on it only when `ruleStatus` is `failed`. From ff5932f0e170363767aac33aace8f7fd3ec04643 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 12:38:15 -0400 Subject: [PATCH 19/44] docs: broaden JSON results heading Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/docs/usage.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/docs/usage.md b/docs/docs/usage.md index 934ded00..d825bdd2 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -60,7 +60,7 @@ TOTAL : 36 Result Bundle: /data/techsupport/preupgrade_validator_2021-07-30T13-28-25-0700.tgz ``` -## Results +## Result types and JSON output Each check has a unique result which will help determine how to proceed. The results are explained as follows: @@ -71,8 +71,6 @@ Each check has a unique result which will help determine how to proceed. The res - **N/A** - The check completed successfully, and the ACI fabric is not susceptible because the needed configuration is not deployed. - **ERROR** - The check did not complete successfully, and needs further investigation. -### Recommended action in JSON results - The per-check JSON schema was added for the built-in pre-upgrade validation workflow in APIC 6.2 and later. The APIC presentation layer expects `recommended_action` as static rule metadata, so the field can be populated even when `ruleStatus` is `passed`. See Cisco’s [Pre-upgrade validator examples for APIC 6.2(1)](https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/all/apic-installation-aci-upgrade-downgrade/Cisco-APIC-Installation-ACI-Upgrade-Downgrade-Guide/g-pre-upgrade-checklists/examples-of-pre-upgrade-validator-apic.html#Cisco_Reference.dita_53db0ff6-cc90-4543-80e0-091fc41dc962__section_vq4_5lv_qhc). Use `ruleStatus` to determine whether the recommendation is actionable. A populated `recommended_action` does not indicate a problem when `ruleStatus` is `passed`; standalone consumers should act on it only when `ruleStatus` is `failed`. From 17709f426a7565056ad4a69b2bb1d7d08d9415dc Mon Sep 17 00:00:00 2001 From: "GM.EXE" Date: Mon, 3 Aug 2026 13:29:51 -0400 Subject: [PATCH 20/44] docs: correct APIC database object threshold (#415) Fixes #382. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/docs/validations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f7886811..f1c13fa3 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -521,7 +521,7 @@ The script performs 2 different checks depending on the version you are running. For current versions below 6.1(3): - The script checks all APICs' class's object count for a subset of services (DMEs) via a file scan. -- If the count is found to be above `150*1000*1000`, then that class will be flagged for further investigation. +- If the count is found to be above `1000*1000*1.5` (1,500,000), then that class will be flagged for further investigation. For current version is 6.1(3f): From 82163fc9868f3dc2cbd082b37ca2ef6c9453e730 Mon Sep 17 00:00:00 2001 From: "GM.EXE" Date: Mon, 3 Aug 2026 14:14:35 -0400 Subject: [PATCH 21/44] Issue 402 fn64251 chat guidance (#417) * Update FN64251 validation guidance (#402) Replace the retired serial-number validation tool with the chat interface embedded in FN64251 and document the direct Field Notice workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Clarify FN64251 affected-product guidance (#402) Document the shipment-date cutoff and inconclusive V01 VID, and direct users to TAC when a listed component is not known to be post-cutoff. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 7 ++++++- docs/docs/validations.md | 8 ++++---- .../test_clock_signal_component_failure_check.py | 16 ++++++++++++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 5e79f56e..a0e8e2ec 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -5630,7 +5630,12 @@ def clock_signal_component_failure_check(**kwargs): result = PASS headers = ['Pod', "Node", "Slot", "Model", "Serial Number"] data = [] - recommended_action = 'Run the SN string through the Serial Number Validation tool (linked within doc url) to check for FN64251.\n\tSN String:\n\t' + recommended_action = ( + 'Review the listed serial numbers using FN64251. Products shipped after December 5, 2016 are not affected ' + 'and can be ignored. For products shipped on or before December 5, 2016, or with an unknown ship date, ' + 'contact Cisco TAC to confirm whether they are affected. A V01 Version ID (VID) is only possibly affected ' + 'and is not conclusive because some unaffected products also use V01.\n\tSN String:\n\t' + ) doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#nexus-950x-fm-or-lc-might-fail-to-boot-after-reload' eqptFC_api = 'eqptFC.json' diff --git a/docs/docs/validations.md b/docs/docs/validations.md index f1c13fa3..4e416371 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -2645,7 +2645,7 @@ The script checks if your upgrade is susceptible to this defect from both versio ### Nexus 950X FM or LC Might Fail to boot after reload A clock signal component manufactured by one supplier, and included in some Cisco products, has been seen to degrade over time in some units. -Although the Cisco products with these components are currently performing normally, we expect product failures to increase over the years, beginning after the unit has been in operation for approximately 18 months. Additional details are document in [FN64251][39] +Although the Cisco products with these components are currently performing normally, we expect product failures to increase over the years, beginning after the unit has been in operation for approximately 18 months. Additional details are documented in [FN64251][39]. The matching defect is [CSCvg26013][40]. @@ -2660,7 +2660,9 @@ Line Card - N9K-X9732C-EX -If alerted, check if identified Serial Numbers are affected using the [Serial Number Validation Tool][41]. +If alerted, review the serial numbers reported by the check against [FN64251][39]. Products shipped after December 5, 2016 are not affected and can be ignored. For products shipped on or before December 5, 2016, or with an unknown ship date, contact Cisco TAC with the reported serial numbers to confirm whether they are affected. + +The Field Notice identifies V01 as possibly affected, but the VID is not conclusive because some unaffected products also use V01. The VID of a working module can be obtained with the `show inventory` command; a failed module will not be recognized. ### Stale Decommissioned Spine @@ -2887,7 +2889,6 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [ [38]: https://www.cisco.com/c/en/us/td/docs/dcn/aci/apic/6x/verified-scalability/cisco-aci-verified-scalability-guide-612.html [39]: https://www.cisco.com/c/en/us/support/docs/field-notices/642/fn64251.html [40]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCvg26013 -[41]: https://snvui.cisco.com/snv/FN64251 [42]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwf58763 [43]: https://www.cisco.com/c/en/us/support/docs/field-notices/740/fn74050.html [44]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwd65255 @@ -2923,4 +2924,3 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [ [74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 - diff --git a/tests/checks/clock_signal_component_failure_check/test_clock_signal_component_failure_check.py b/tests/checks/clock_signal_component_failure_check/test_clock_signal_component_failure_check.py index a1b017e5..7d4b1fcd 100644 --- a/tests/checks/clock_signal_component_failure_check/test_clock_signal_component_failure_check.py +++ b/tests/checks/clock_signal_component_failure_check/test_clock_signal_component_failure_check.py @@ -19,7 +19,7 @@ @pytest.mark.parametrize( - "icurl_outputs, expected_result", + "icurl_outputs, expected_result, expected_serials", # Positive cases, one or both classes return an affected model [ ( @@ -28,6 +28,7 @@ eqptLC_api: read_data(dir, "eqptLC_POS.json") }, script.MANUAL, + ["FOC235053QS", "FOC23506V60", "FOC23506V3J", "FOC235053QU", "FOC235053MR", "FDO23260QX5"], ), ( { @@ -35,6 +36,7 @@ eqptLC_api: read_data(dir, "eqptLC_NEG.json") }, script.MANUAL, + ["FOC235053QS", "FOC23506V60", "FOC23506V3J", "FOC235053QU", "FOC235053MR"], ), ( { @@ -42,6 +44,7 @@ eqptLC_api: read_data(dir, "eqptLC_POS.json") }, script.MANUAL, + ["FDO23260QX5"], ), # Both classes return empty ( @@ -50,9 +53,18 @@ eqptLC_api: read_data(dir, "eqptLC_NEG.json") }, script.PASS, + [], ) ], ) -def test_logic(run_check, mock_icurl, expected_result): +def test_logic(run_check, mock_icurl, expected_result, expected_serials): result = run_check() assert result.result == expected_result + if expected_result == script.MANUAL: + assert "shipped after December 5, 2016 are not affected" in result.recommended_action + assert "on or before December 5, 2016" in result.recommended_action + assert "contact Cisco TAC" in result.recommended_action + assert "V01 Version ID (VID) is only possibly affected" in result.recommended_action + assert all(serial in result.recommended_action for serial in expected_serials) + assert "chat interface" not in result.recommended_action + assert "Serial Number Validation tool" not in result.recommended_action From 405c880a749caeab15e0108ce212b13272c57569 Mon Sep 17 00:00:00 2001 From: "GM.EXE" Date: Mon, 3 Aug 2026 14:24:34 -0400 Subject: [PATCH 22/44] Support compact equipment disk usage changeSet (#369) (#414) Parse both observed F1820/F1821/F1822 changeSet serializations while preserving fault-driven failure behavior and malformed values as NA. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 16 +++++---- docs/docs/validations.md | 2 ++ .../faultInst_compact.json | 15 ++++++++ .../test_equipment_disk_limits_exceeded.py | 34 +++++++++++++++++-- 4 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 tests/checks/equipment_disk_limits_exceeded/faultInst_compact.json diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index a0e8e2ec..5c29ae0b 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -5850,7 +5850,8 @@ def equipment_disk_limits_exceeded(**kwargs): recommended_action = 'Review the reference document for commands to validate disk usage' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#equipment-disk-limits' - usage_regex = r"avail \(New: (?P\d+)\).+used \(New: (?P\d+)\)" + avail_regex = r"(?:^|,\s*)avail(?:\s+\(New:\s*|:\s*)(?P\d+)(?:\)|(?=,|$))" + used_regex = r"(?:^|,\s*)used(?:\s+\(New:\s*|:\s*)(?P\d+)(?:\)|(?=,|$))" f182x_api = 'faultInst.json' f182x_api += '?query-target-filter=or(eq(faultInst.code,"F1820"),eq(faultInst.code,"F1821"),eq(faultInst.code,"F1822"))' faults = icurl('class', f182x_api) @@ -5859,11 +5860,14 @@ def equipment_disk_limits_exceeded(**kwargs): percent = "NA" attributes = faultInst['faultInst']['attributes'] - usage_match = re.search(usage_regex, attributes['changeSet']) - if usage_match: - avail = int(usage_match.group('avail')) - used = int(usage_match.group('used')) - percent = round((used / (avail + used)) * 100) + avail_match = re.search(avail_regex, attributes['changeSet']) + used_match = re.search(used_regex, attributes['changeSet']) + if avail_match and used_match: + avail = int(avail_match.group('value')) + used = int(used_match.group('value')) + total = avail + used + if total: + percent = int(round((used / total) * 100)) dn_match = re.search(node_regex, attributes['dn']) if dn_match: diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 4e416371..1eeb5fe2 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -1528,6 +1528,8 @@ This fault occurs when the disk usage of a partiton increases beyond its thresho This fault also occurs when the MTS buffer memory usage increases beyond its threshold. /proc/isan/sw/mts/mem/stats is checked when this scenario occurs. +The check calculates utilization from the available and used values reported by each fault. Both APIC `changeSet` formats are supported. + Recommended Action: 1. Check `df -h` output on affected node to see the usage of the partition. diff --git a/tests/checks/equipment_disk_limits_exceeded/faultInst_compact.json b/tests/checks/equipment_disk_limits_exceeded/faultInst_compact.json new file mode 100644 index 00000000..cfeb9e82 --- /dev/null +++ b/tests/checks/equipment_disk_limits_exceeded/faultInst_compact.json @@ -0,0 +1,15 @@ +[ + { + "faultInst": { + "attributes": { + "cause": "equipment-disk-limits-exceeded", + "changeSet": "avail:1959076, memAlert:minor, name:ifc:cfg, path:/mnt/ifc/cfg, used:8207252", + "code": "F1820", + "descr": "Disk usage for /mnt/ifc/cfg is above normal", + "dn": "topology/pod-1/node-107/sys/eqptcapacity/fspartition-ifc:cfg/fault-F1820", + "lc": "raised", + "rule": "eqptcapacity-fspartition-fs-partition-limits-exceeded-minor" + } + } + } +] diff --git a/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py b/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py index 72fd0aa9..ad1cff4d 100644 --- a/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py +++ b/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py @@ -16,18 +16,46 @@ @pytest.mark.parametrize( - "icurl_outputs, expected_result", + "icurl_outputs, expected_result, expected_data, expected_unformatted_data", [ ( {f182x_api: read_data(dir, "faultInst_neg.json")}, script.PASS, + [], + [], ), ( {f182x_api: read_data(dir, "faultInst_pos.json")}, script.FAIL_UF, - ) + [ + ["1", "101", "F1820", 98, "Disk usage for /mnt/ifc/log is high on node 101 of fabric POD1 with a hostname leaf1"], + ["1", "102", "F1821", 97, "Disk usage for /mnt/ifc/cfg is high on node 102 of fabric POD1 with a hostname leaf2"], + ["1", "104", "F1821", 100, "Disk usage for / is high on node 104 of fabric POD1 with a hostname LEAF-104"], + ], + [[ + "topology/pod-1/node-[103]/sys/eqptcapacity/fspartition-ifc:cfg/fault-F1821", + "NA", + "Disk usage for /mnt/ifc/cfg is high on node 103 of fabric POD1 with a hostname leaf3", + ]], + ), + ( + {f182x_api: read_data(dir, "faultInst_compact.json")}, + script.FAIL_UF, + [["1", "107", "F1820", 81, "Disk usage for /mnt/ifc/cfg is above normal"]], + [], + ), ], ) -def test_logic(run_check, mock_icurl, expected_result): +def test_logic( + run_check, + mock_icurl, + expected_result, + expected_data, + expected_unformatted_data, +): result = run_check() assert result.result == expected_result + assert result.data == expected_data + assert result.unformatted_data == expected_unformatted_data + for row in result.data: + assert isinstance(row[3], int) From 1b413bf2bad493d37be114ad312330db6e214409 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 14:42:47 -0400 Subject: [PATCH 23/44] fix: classify low RTC battery as outage risk Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitlab-ci.yml | 2 ++ aci-preupgrade-validation-script.py | 2 +- .../test_rtc_battery_voltage_low_check.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de14fdea..7c123ea8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,8 +82,10 @@ test:integration: - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key artifacts: when: always + expire_in: 7 days paths: - $CI_PROJECT_DIR/tests/*.log + - $CI_PROJECT_DIR/tests/*.tgz cache: key: files: diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 64bf2aa5..2cd88694 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6406,7 +6406,7 @@ def apic_storage_inode_check(**kwargs): @check_wrapper(check_title="Switch RTC Battery Voltage (F2421 equipment-diags-failed)") def rtc_battery_voltage_low_check(**kwargs): - result = FAIL_UF + result = FAIL_O headers = ['Fault', 'Pod', 'Node', 'Supervisor', 'Severity', 'Lifecycle'] data = [] unformatted_headers = ['Fault', 'Fault DN', 'Description', 'Severity', 'Lifecycle'] diff --git a/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py b/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py index 17a44766..c468d61b 100644 --- a/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py +++ b/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py @@ -43,7 +43,7 @@ [ ( {faultInst_api: active_faults}, - script.FAIL_UF, + script.FAIL_O, [["F2421", "1", "122", "1", "minor", "raised"]], [], ), @@ -52,7 +52,7 @@ ({faultInst_api: non_exact_reason_faults}, script.PASS, [], []), ( {faultInst_api: active_faults + malformed_faults}, - script.FAIL_UF, + script.FAIL_O, [["F2421", "1", "122", "1", "minor", "raised"]], [[ "F2421", From f93f5ffaa44507ce94e72c30e9aa0e54be3e97d0 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 3 Aug 2026 14:53:05 -0400 Subject: [PATCH 24/44] fix: make RTC battery query APIC-compatible Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- aci-preupgrade-validation-script.py | 2 +- .../test_rtc_battery_voltage_low_check.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 2cd88694..b5f8c83c 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6415,7 +6415,7 @@ def rtc_battery_voltage_low_check(**kwargs): doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-rtc-battery-voltage' dn_regex = node_regex + r'/.+/supslot-(?P\d+)/sup\]/fault-F2421$' fault_reason = 'The RTC battery voltage is low' - fault_api = 'faultInst.json?query-target-filter=and(eq(faultInst.code,"F2421"),wcard(faultInst.descr,"reason:The RTC battery voltage is low"))' + fault_api = 'faultInst.json?query-target-filter=eq(faultInst.code,"F2421")' faultInsts = icurl('class', fault_api) for faultInst in faultInsts: diff --git a/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py b/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py index c468d61b..10da9961 100644 --- a/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py +++ b/tests/checks/rtc_battery_voltage_low_check/test_rtc_battery_voltage_low_check.py @@ -13,10 +13,7 @@ log = logging.getLogger(__name__) dir = os.path.dirname(os.path.abspath(__file__)) test_function = "rtc_battery_voltage_low_check" -faultInst_api = ( - 'faultInst.json?query-target-filter=and(eq(faultInst.code,"F2421"),' - 'wcard(faultInst.descr,"reason:The RTC battery voltage is low"))' -) +faultInst_api = 'faultInst.json?query-target-filter=eq(faultInst.code,"F2421")' active_faults = read_data(dir, "f2421.json")["imdata"] retaining_faults = copy.deepcopy(active_faults) retaining_faults[0]["faultInst"]["attributes"]["lc"] = "retaining" From 199cec61ab47d3bafbcd6116cbbfe5c927871076 Mon Sep 17 00:00:00 2001 From: muthu-ku Date: Sat, 8 Aug 2026 04:09:47 +0530 Subject: [PATCH 25/44] Added validation check for CSCwt58626 (#405) * Added validation check for CSCwt58626 * updated comments * updated merge conflict * updated missing import for stale_dbgacEpgSummaryTask_check * updated the review comments * updated pytest case * changed result for failure and error case * updated the doc * updated the document comment * updated comments * handled error cases * Added tversion missing check for other scripts * updated the review comments * updated the review comments * updated script * removed commented lines in pytest file --------- Co-authored-by: GM.EXE --- aci-preupgrade-validation-script.py | 93 +++++++- docs/docs/validations.md | 9 + ...nband_management_policy_misconfig_check.py | 12 +- .../fvnsEncapBlk_empty.json | 1 + .../fvnsEncapBlk_invalid_vlan_entry.json | 41 ++++ ...sing_attrs_with_non_overlap_vlan_pool.json | 39 +++ ..._missing_attrs_with_overlap_vlan_pool.json | 39 +++ .../fvnsEncapBlk_no_overlap.json | 28 +++ .../fvnsEncapBlk_overlap_malformed_dn.json | 40 ++++ ...pBlk_overlap_mixed_valid_malformed_dn.json | 41 ++++ ...nsEncapBlk_overlap_multiple_vlan_pool.json | 62 +++++ ...fvnsEncapBlk_overlap_single_vlan_pool.json | 41 ++++ .../lldpInst_empty.json | 1 + .../lldpInst_infra_vlan_multiple_entry.json | 102 ++++++++ .../lldpInst_infra_vlan_single_entry.json | 12 + ...t_infravlan_overlap_access_policy_check.py | 223 ++++++++++++++++++ .../test_rogue_ep_coop_exception_mac_check.py | 11 +- 17 files changed, 790 insertions(+), 5 deletions(-) create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_empty.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_invalid_vlan_entry.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_non_overlap_vlan_pool.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_overlap_vlan_pool.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_no_overlap.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_malformed_dn.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_mixed_valid_malformed_dn.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_multiple_vlan_pool.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_single_vlan_pool.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/lldpInst_empty.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_multiple_entry.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_single_entry.json create mode 100644 tests/checks/infravlan_overlap_access_policy_check/test_infravlan_overlap_access_policy_check.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 5c29ae0b..b8b68dff 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -22,7 +22,7 @@ from textwrap import TextWrapper from getpass import getpass from collections import defaultdict, OrderedDict -from datetime import datetime +from datetime import datetime, timedelta from argparse import ArgumentParser from itertools import chain import threading @@ -6302,6 +6302,9 @@ def is_affected_target(ver): in_61 = ver.newer_than("6.1(1a)") and ver.older_than("6.1(4h)") return in_60 or in_61 + if not tversion or not cversion: + return Result(result=MANUAL, msg=TVER_MISSING) + pre_apic_upg = is_affected_source(cversion) and is_affected_target(tversion) # Before APIC upgrade post_apic_upg = is_affected_target(cversion) and is_affected_target(tversion) and cversion.same_as(tversion) # After APIC upgrade (and before switch) @@ -6486,6 +6489,9 @@ def inband_management_policy_misconfig_check(cversion, tversion, **kwargs): recommended_action = "Contact Cisco TAC to remove any identified misconfigured 'mgmtRsInBStNode' objects" doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#inband-management-policy-misconfiguration" + if not tversion or not cversion: + return Result(result=MANUAL, msg=TVER_MISSING) + if (cversion.older_than("5.2(8d)")) and (tversion.newer_than("6.0(4c)") or tversion.same_as("6.0(4c)")): mgmtRsInBStNodes = icurl('class', 'mgmtRsInBStNode.json?query-target-filter=and(or(eq(mgmtRsInBStNode.addr,"0.0.0.0"),eq(mgmtRsInBStNode.gw,"0.0.0.0")),or(eq(mgmtRsInBStNode.v6Addr,"::"),eq(mgmtRsInBStNode.v6Gw,"::")))') for mgmtRsInBStNode in mgmtRsInBStNodes: @@ -6694,7 +6700,12 @@ def stale_dbgacEpgSummaryTask_check(tversion, **kwargs): if tversion and ((tversion.major1 == "6" and tversion.major2 == "1" and tversion.newer_than("6.1(5e)")) or tversion.newer_than("6.2(1g)")): return Result(result=NA, msg=VER_NOT_AFFECTED, doc_url=doc_url) - threshold = datetime.utcnow() - timedelta(hours=24) + try: + from datetime import timezone + threshold = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=24) + except ImportError: + threshold = datetime.utcnow() - timedelta(hours=24) + for obj in icurl("class", 'dbgacEpgSummaryTask.json?query-target-filter=eq(dbgacEpgSummaryTask.operSt,"processing")'): attr = obj["dbgacEpgSummaryTask"]["attributes"] dn = attr.get("dn", "") @@ -6711,6 +6722,81 @@ def stale_dbgacEpgSummaryTask_check(tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) +@check_wrapper(check_title="InfraVLAN Overlap in Access Policy VLAN Pools") +def infravlan_overlap_access_policy_check(tversion, **kwargs): + result = FAIL_UF + msg = "" + headers = ["InfraVLAN", "Encap Block", "VLAN Pool DN"] + unformatted_headers = ["InfraVLAN", "Encap Block", "VLAN Pool DN", "VLAN Pool RN"] + + data = [] + unformatted_data = [] + recommended_action = "Select a non-affected target version or contact Cisco TAC for Support before upgrade." + + doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#infravlan-overlap-access-policy-check" + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING) + + if not (tversion.same_as("6.2(1g)") or ( + not tversion.older_than("6.1(3f)") and not tversion.newer_than("6.1(5e)") + )): + return Result(result=NA, msg=VER_NOT_AFFECTED) + + dn_regex1 = r'uni/infra/vlanns-\[.+\]-(static|dynamic)/from-\[vlan-\d+\]-to-\[vlan-\d+\]' + + dn_regex2 = r'uni/vmmp-[^/]+/dom-[^/]+/.+/from-\[vlan-\d+\]-to-\[vlan-\d+\]' + + infra_vlan = None + has_error = False + lldpInsts = icurl('class', 'lldpInst.json?query-target-filter=wcard(lldpInst.dn,"/node-1/")') + for lldpInst in lldpInsts: + infra_vlan_id = lldpInst.get('lldpInst', {}).get('attributes', {}).get('infraVlan') + if not infra_vlan_id: + continue + match = re.search(r'\d+', str(infra_vlan_id)) + if match: + infra_vlan = int(match.group(0)) + break + + if infra_vlan is None: + return Result(result=ERROR, msg="Unable to determine InfraVLAN from lldpInst.") + + encap_blocks = icurl('class', 'fvnsEncapBlk.json?query-target-filter=eq(fvnsEncapBlk.role,"external")') + for obj in encap_blocks: + blk_attr = obj.get('fvnsEncapBlk', {}).get('attributes', {}) + dn = blk_attr.get('dn', '') + rn = blk_attr.get('rn', '') + from_encap = blk_attr.get('from') + to_encap = blk_attr.get('to') + + if not dn or not from_encap or not to_encap: + has_error = True + + try: + from_vlan = int(str(from_encap).split('-')[-1]) + to_vlan = int(str(to_encap).split('-')[-1]) + except (ValueError, TypeError): + has_error = True + continue + + if min(from_vlan, to_vlan) <= infra_vlan <= max(from_vlan, to_vlan): + row = [str(infra_vlan), "{} to {}".format(from_encap, to_encap), dn] + if (re.search(dn_regex1, dn) or re.search(dn_regex2, dn)): + data.append(row) + else: + unformatted_data.append(row + [rn]) + + if not data and not unformatted_data: + result = PASS + if has_error: + result = ERROR + msg = "Overlap check for InfraVLAN {} could not be determined because one or more VLAN pool blocks contain improper data or Error while fetching data.".format(infra_vlan) + + + return Result(result=result, msg=msg, headers=headers, data=data, unformatted_headers=unformatted_headers, unformatted_data=unformatted_data, recommended_action=recommended_action, doc_url=doc_url) + + # ---- Script Execution ---- @@ -6886,7 +6972,8 @@ class CheckManager: wred_affected_model_check, n9k_c93180yc_fx3_switch_memory_check, stale_dbgacEpgSummaryTask_check, - + infravlan_overlap_access_policy_check, + ] ssh_checks = [ # General diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 1eeb5fe2..6695fc99 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -206,6 +206,7 @@ Items | Defect | This Script [WRED with Affected FM Models][d35] | CSCwt50713 | :white_check_mark: | :no_entry_sign: [N9K-C93180YC-FX3 Switch Memory Less Than 32GB][d36] | CSCwm42741 | :white_check_mark: | :no_entry_sign: [Stale dbgacEpgSummaryTask Objects][d37] | CSCwt69100 | :white_check_mark: | :no_entry_sign: +[InfraVLAN Overlap in Access Policy VLAN Pools][d38] | CSCwt58626 | :white_check_mark: | :no_entry_sign: [d1]: #ep-announce-compatibility [d2]: #eventmgr-db-size-defect-susceptibility @@ -244,6 +245,7 @@ Items | Defect | This Script [d35]: #wred-with-affected-fm-models [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb [d37]: #stale-dbgacepgsummarytask-objects +[d38]: #infravlan-overlap-access-policy-check ## General Check Details @@ -2850,6 +2852,12 @@ Affected versions: 6.1(5e) and below, or 6.2(1g). Contact Cisco TAC for next steps. For more details, refer to the workaround in [CSCwt69100][75]. +### Infravlan Overlap Access Policy Check + +Due to the bug [CSCwt58626][77] , If Apic upgrade planned for target versions 6.1(3f), 6.1(3g), 6.1(4h), 6.1(5e) and 6.2(1g), be aware of fault F4701 being raised if the InfraVLAN overlaps with any user-configured VLAN pool in Access Policies. This also affects vlan pool created by NDO/MSO, VMM, Kubernetes. After the upgrade, domains associated with those VLAN pools cannot be linked to new EPGs, although existing EPGs continue to function. + +To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does not overlap with any configured block, or select a non-impacted fixed version. After upgrading to a fixed version this fault and Restriction have been removed. + [0]: https://github.com/datacenter/ACI-Pre-Upgrade-Validation-Script [1]: https://www.cisco.com/c/dam/en/us/td/docs/Website/datacenter/apicmatrix/index.html [2]: https://www.cisco.com/c/en/us/support/switches/nexus-9000-series-switches/products-release-notes-list.html @@ -2926,3 +2934,4 @@ Contact Cisco TAC for next steps. For more details, refer to the workaround in [ [74]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwm42741 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 +[77]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt58626 diff --git a/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py b/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py index 656cb4e4..da2fd25b 100644 --- a/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py +++ b/tests/checks/inband_management_policy_misconfig_check/test_inband_management_policy_misconfig_check.py @@ -13,6 +13,13 @@ @pytest.mark.parametrize( "icurl_outputs, cversion, tversion, expected_result, expected_data", [ + + # tversion missing + ({}, "5.2(7g)", None, script.MANUAL, []), + # cversion missing + ({}, None, "5.2(7g)", script.MANUAL, []), + # cversion and tversion missing + ({}, None, None, script.MANUAL, []), # Current version is affected, Target version = 6.0(4c), valid data ( { @@ -158,6 +165,9 @@ ], ) def test_logic(run_check, mock_icurl, cversion, tversion, expected_result, expected_data): - result = run_check(cversion=script.AciVersion(cversion), tversion=script.AciVersion(tversion)) + result = run_check( + cversion=script.AciVersion(cversion) if cversion else None, + tversion=script.AciVersion(tversion) if tversion else None, + ) assert result.result == expected_result assert result.data == expected_data \ No newline at end of file diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_empty.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_empty.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_empty.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_invalid_vlan_entry.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_invalid_vlan_entry.json new file mode 100644 index 00000000..6dd48914 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_invalid_vlan_entry.json @@ -0,0 +1,41 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-abc]-to-[vlan-4094]", + "role": "external", + "from": "vlan-abc", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool_invalid]-static/from-[vlan-abc]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-10]-to-[vlan-20]", + "role": "external", + "from": "vlan-10", + "to": "vlan-20", + "dn": "uni/infra/vlanns-[vlan_pool_invalid]-static/from-[vlan-10]-to-[vlan-20]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-3000]-to-[vlan-4094]", + "role": "external", + "from": null, + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool_non_string]-static/from-[vlan-3000]-to-[vlan-4094]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_non_overlap_vlan_pool.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_non_overlap_vlan_pool.json new file mode 100644 index 00000000..7a440680 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_non_overlap_vlan_pool.json @@ -0,0 +1,39 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-100]-to-[vlan-200]", + "role": "external", + "from": "vlan-100", + "to": "vlan-200" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-30]-to-[vlan-40]", + "role": "external", + "from": "", + "to": "vlan-40" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-10]-to-[vlan-20]", + "role": "external", + "from": "vlan-10", + "to": "vlan-20", + "dn": "uni/infra/vlanns-[vlan_pool3]-static/from-[vlan-10]-to-[vlan-20]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_overlap_vlan_pool.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_overlap_vlan_pool.json new file mode 100644 index 00000000..c98fc7f2 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_missing_attrs_with_overlap_vlan_pool.json @@ -0,0 +1,39 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-3000]-to-[vlan-4094]", + "role": "external", + "from": "vlan-3000", + "to": "vlan-4094" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-3001]-to-[vlan-4094]", + "role": "external", + "from": "", + "to": "vlan-4094" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-4000]-to-[vlan-4094]", + "role": "external", + "from": "vlan-4000", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool3]-static/from-[vlan-4000]-to-[vlan-4094]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_no_overlap.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_no_overlap.json new file mode 100644 index 00000000..a0a858b6 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_no_overlap.json @@ -0,0 +1,28 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-200]-to-[vlan-300]", + "role": "external", + "from": "vlan-200", + "to": "vlan-300", + "dn": "uni/infra/vlanns-[vlan_pool1]-static/from-[vlan-200]-to-[vlan-300]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-500]-to-[vlan-1000]", + "role": "external", + "from": "vlan-500", + "to": "vlan-1000", + "dn": "uni/infra/vlanns-[vlan_pool2]-static/from-[vlan-500]-to-[vlan-1000]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_malformed_dn.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_malformed_dn.json new file mode 100644 index 00000000..f0f7c46d --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_malformed_dn.json @@ -0,0 +1,40 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-4000]-to-[vlan-4094]", + "role": "external", + "from": "vlan-4000", + "to": "vlan-4094", + "dn": "uni/infra/vlanpool/vlan-4000-4094" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-400]-to-[vlan-4094]", + "role": "external", + "from": "vlan-400", + "to": "vlan-4094", + "dn": "uni/infra/vlanpool/vlan-400-4094" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "role": "external", + "from": "vlan-4000", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool_missing_rn]-static/from-[vlan-4000]-to-[vlan-4094]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_mixed_valid_malformed_dn.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_mixed_valid_malformed_dn.json new file mode 100644 index 00000000..489995d1 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_mixed_valid_malformed_dn.json @@ -0,0 +1,41 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-100]-to-[vlan-4094]", + "role": "external", + "from": "vlan-100", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool1]-static/from-[vlan-100]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-4000]-to-[vlan-4094]", + "role": "external", + "from": "vlan-4000", + "to": "vlan-4094", + "dn": "uni/infra/vlanpool/vlan-4000-4094" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-400]-to-[vlan-4094]", + "role": "external", + "from": "vlan-400", + "to": "vlan-4094", + "dn": "uni/infra/vlanpool/vlan-400-4094" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_multiple_vlan_pool.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_multiple_vlan_pool.json new file mode 100644 index 00000000..6b8717bd --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_multiple_vlan_pool.json @@ -0,0 +1,62 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "role": "external", + "from": "vlan-100", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool1]-static/from-[vlan-100]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "role": "internal", + "from": "vlan-200", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool2]-static/from-[vlan-200]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "role": "external", + "from": "vlan-4000", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool3]-static/from-[vlan-4000]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "role": "external", + "from": "vlan-1751", + "to": "vlan-4094", + "dn": "uni/vmmp-VMware/dom-k8s-scale-vmm/usrcustomaggr-k8srkesetup1/from-[vlan-1751]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "role": "external", + "from": "vlan-1752", + "to": "vlan-4094", + "dn": "uni/vmmp-VMware/dom-k8s-scale-vmm/usrcustomaggr-k8srkesetup1/from-[vlan-1752]-to-[vlan-4094]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_single_vlan_pool.json b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_single_vlan_pool.json new file mode 100644 index 00000000..baafbc43 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/fvnsEncapBlk_overlap_single_vlan_pool.json @@ -0,0 +1,41 @@ +[ + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-100]-to-[vlan-4094]", + "role": "external", + "from": "vlan-100", + "to": "vlan-4094", + "dn": "uni/infra/vlanns-[vlan_pool]-static/from-[vlan-100]-to-[vlan-4094]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-1751]-to-[vlan-1751]", + "role": "external", + "from": "vlan-1751", + "to": "vlan-1751", + "dn": "uni/vmmp-VMware/dom-k8s-scale-vmm/usrcustomaggr-k8srkesetup1/from-[vlan-1751]-to-[vlan-1751]" + } + } + }, + { + "fvnsEncapBlk": { + "attributes": { + "allocMode": "static", + "annotation": "orchestrator:aci-containers-controller", + "rn": "from-[vlan-1752]-to-[vlan-1752]", + "role": "external", + "from": "vlan-1752", + "to": "vlan-1752", + "dn": "uni/vmmp-VMware/dom-k8s-scale-vmm/usrcustomaggr-k8srkesetup1/from-[vlan-1752]-to-[vlan-1752]" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/lldpInst_empty.json b/tests/checks/infravlan_overlap_access_policy_check/lldpInst_empty.json new file mode 100644 index 00000000..0637a088 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/lldpInst_empty.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_multiple_entry.json b/tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_multiple_entry.json new file mode 100644 index 00000000..6a1bb8f6 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_multiple_entry.json @@ -0,0 +1,102 @@ +[ + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-201/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-201" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-202/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-202" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-203/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-203" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-204/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-204" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-205/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-205" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-206/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-206" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-207/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-207" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-208/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-208" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-209/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-209" + } + } + }, + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-210/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-210" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_single_entry.json b/tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_single_entry.json new file mode 100644 index 00000000..5b14d8d4 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/lldpInst_infra_vlan_single_entry.json @@ -0,0 +1,12 @@ +[ + { + "lldpInst": { + "attributes": { + "adminSt": "enabled", + "dn": "topology/pod-1/node-201/sys/lldp/inst", + "infraVlan": "vlan-4093", + "sysDesc": "topology/pod-1/node-201" + } + } + } +] diff --git a/tests/checks/infravlan_overlap_access_policy_check/test_infravlan_overlap_access_policy_check.py b/tests/checks/infravlan_overlap_access_policy_check/test_infravlan_overlap_access_policy_check.py new file mode 100644 index 00000000..0c692f79 --- /dev/null +++ b/tests/checks/infravlan_overlap_access_policy_check/test_infravlan_overlap_access_policy_check.py @@ -0,0 +1,223 @@ +import os +import pytest +import logging +import importlib +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") + +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) + +test_function = "infravlan_overlap_access_policy_check" + +# icurl queries +lldpInst_api = "lldpInst.json?query-target-filter=wcard(lldpInst.dn,\"/node-1/\")" +fvnsEncapBlk_api = "fvnsEncapBlk.json?query-target-filter=eq(fvnsEncapBlk.role,\"external\")" + + +@pytest.mark.parametrize( + "icurl_outputs, tversion, expected_result, expected_data, expected_unformatted_data", + [ + # Case 1: tversion missing. Expected: MANUAL. + ( + {}, + None, + script.MANUAL, + [], + [], + ), + # Case 2: Version not affected (6.0(9f) is below 6.1(3f)). Expected: NA. + ( + {}, + "6.0(9f)", + script.NA, + [], + [], + ), + # Case 3: Version not affected (6.1(3e) is just below lower boundary 6.1(3f)). Expected: NA. + ( + {}, + "6.1(3e)", + script.NA, + [], + [], + ), + # Case 4: Version not affected (6.1(6a) is above upper boundary 6.1(5e) and not 6.2(1g)). Expected: NA. + ( + {}, + "6.1(6a)", + script.NA, + [], + [], + ), + # Case 5: lldpInst returns no data so infraVlan cannot be determined. Expected: ERROR. + ( + { + lldpInst_api: read_data(dir, "lldpInst_empty.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_single_vlan_pool.json"), + }, + "6.2(1g)", + script.ERROR, + [], + [], + ), + # Case 6: InfraVLAN overlaps on lower boundary version 6.1(3f). Expected: FAIL_UF. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_single_vlan_pool.json"), + }, + "6.1(3f)", + script.FAIL_UF, + [["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool]-static/from-[vlan-100]-to-[vlan-4094]"]], + [], + ), + # Case 7: InfraVLAN overlaps on a mid-range version 6.1(4a). Expected: FAIL_UF. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_single_vlan_pool.json"), + }, + "6.1(4a)", + script.FAIL_UF, + [["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool]-static/from-[vlan-100]-to-[vlan-4094]"]], + [], + ), + # Case 8: InfraVLAN overlaps on upper boundary version 6.1(5e). Expected: FAIL_UF. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_single_vlan_pool.json"), + }, + "6.1(5e)", + script.FAIL_UF, + [["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool]-static/from-[vlan-100]-to-[vlan-4094]"]], + [], + ), + # Case 9: InfraVLAN overlaps on standalone affected version 6.2(1g). Expected: FAIL_UF. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_single_vlan_pool.json"), + }, + "6.2(1g)", + script.FAIL_UF, + [["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool]-static/from-[vlan-100]-to-[vlan-4094]"]], + [], + ), + # Case 10: InfraVLAN does not overlap on affected version 6.2(1g). Expected: PASS. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_no_overlap.json"), + }, + "6.2(1g)", + script.PASS, + [], + [], + ), + # Case 11: Single lldpInst entry with overlap on affected version 6.2(1g). Expected: FAIL_UF. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_single_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_single_vlan_pool.json"), + }, + "6.2(1g)", + script.FAIL_UF, + [["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool]-static/from-[vlan-100]-to-[vlan-4094]"]], + [], + ), + # Case 12: InfraVLAN overlaps on multiple VLAN pools including VMM domain blocks on 6.2(1g). Expected: FAIL_UF. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_multiple_vlan_pool.json"), + }, + "6.2(1g)", + script.FAIL_UF, + [ + ["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool1]-static/from-[vlan-100]-to-[vlan-4094]"], + ["4093", "vlan-200 to vlan-4094", "uni/infra/vlanns-[vlan_pool2]-static/from-[vlan-200]-to-[vlan-4094]"], + ["4093", "vlan-4000 to vlan-4094", "uni/infra/vlanns-[vlan_pool3]-static/from-[vlan-4000]-to-[vlan-4094]"], + ["4093", "vlan-1751 to vlan-4094", "uni/vmmp-VMware/dom-k8s-scale-vmm/usrcustomaggr-k8srkesetup1/from-[vlan-1751]-to-[vlan-4094]"], + ["4093", "vlan-1752 to vlan-4094", "uni/vmmp-VMware/dom-k8s-scale-vmm/usrcustomaggr-k8srkesetup1/from-[vlan-1752]-to-[vlan-4094]"], + ], + [], + ), + # Case 13: InfraVLAN exists and empty fvnsEncapBlk for vlan pools on standalone affected version 6.2(1g). Expected: PASS. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_empty.json"), + }, + "6.2(1g)", + script.PASS, + [], + [], + ), + # Case 14: Overlap found with malformed DN format. Expected: FAIL_UF with row captured in unformatted_data. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_malformed_dn.json"), + }, + "6.2(1g)", + script.FAIL_UF, + [["4093", "vlan-4000 to vlan-4094", "uni/infra/vlanns-[vlan_pool_missing_rn]-static/from-[vlan-4000]-to-[vlan-4094]"]], + [["4093", "vlan-4000 to vlan-4094", "uni/infra/vlanpool/vlan-4000-4094", "from-[vlan-4000]-to-[vlan-4094]"], + ["4093", "vlan-400 to vlan-4094", "uni/infra/vlanpool/vlan-400-4094", "from-[vlan-400]-to-[vlan-4094]"]], + ), + # Case 15: Overlap found with both valid and malformed DN format. Expected: FAIL_UF with valid in data and malformed in unformatted_data. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_overlap_mixed_valid_malformed_dn.json"), + }, + "6.2(1g)", + script.FAIL_UF, + [["4093", "vlan-100 to vlan-4094", "uni/infra/vlanns-[vlan_pool1]-static/from-[vlan-100]-to-[vlan-4094]"]], + [["4093", "vlan-4000 to vlan-4094", "uni/infra/vlanpool/vlan-4000-4094", "from-[vlan-4000]-to-[vlan-4094]"], + ["4093", "vlan-400 to vlan-4094", "uni/infra/vlanpool/vlan-400-4094", "from-[vlan-400]-to-[vlan-4094]"]], + ), + # Case 16: Missing dn/from/to attributes in fvnsEncapBlk. Expected: FAIL_UF when there is overlap found. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_missing_attrs_with_overlap_vlan_pool.json"), + }, + "6.2(1g)", + script.FAIL_UF, + [["4093", "vlan-4000 to vlan-4094", "uni/infra/vlanns-[vlan_pool3]-static/from-[vlan-4000]-to-[vlan-4094]"]], + [["4093", "vlan-3000 to vlan-4094", "", "from-[vlan-3000]-to-[vlan-4094]"]] + ), + #Case 17: Missing dn/from/to attributes in fvnsEncapBlk with no overlap. Expected: ERROR. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_missing_attrs_with_non_overlap_vlan_pool.json"), + }, + "6.2(1g)", + script.ERROR, + [], + [], + ), + # Case 18: invalid vlan from/to types including null. Expected: ERROR due parsing failure path with no overlaps. + ( + { + lldpInst_api: read_data(dir, "lldpInst_infra_vlan_multiple_entry.json"), + fvnsEncapBlk_api: read_data(dir, "fvnsEncapBlk_invalid_vlan_entry.json"), + }, + "6.2(1g)", + script.ERROR, + [], + [], + ) + ], +) +def test_logic(run_check, mock_icurl, tversion, expected_result, expected_data, expected_unformatted_data): + result = run_check(tversion=script.AciVersion(tversion) if tversion else None) + assert result.result == expected_result + assert result.data == expected_data + assert result.unformatted_data == expected_unformatted_data + \ No newline at end of file diff --git a/tests/checks/rogue_ep_coop_exception_mac_check/test_rogue_ep_coop_exception_mac_check.py b/tests/checks/rogue_ep_coop_exception_mac_check/test_rogue_ep_coop_exception_mac_check.py index f19dd7d2..9d06edfe 100644 --- a/tests/checks/rogue_ep_coop_exception_mac_check/test_rogue_ep_coop_exception_mac_check.py +++ b/tests/checks/rogue_ep_coop_exception_mac_check/test_rogue_ep_coop_exception_mac_check.py @@ -20,6 +20,12 @@ @pytest.mark.parametrize( "icurl_outputs, tversion, cversion, expected_result, expected_data", [ + # tversion missing + ({}, None, "5.2(3e)", script.MANUAL, []), + # cversion missing + ({}, "5.2(3e)", None, script.MANUAL, []), + # cversion and tversion missing + ({}, None, None, script.MANUAL, []), # NA cases (not affected) # tversion (affected source) ({}, "5.3(2f)", "5.2(3e)", script.NA, []), # cversion (affected source) @@ -197,6 +203,9 @@ ) def test_rogue_ep_coop_exception_mac_check(run_check, mock_icurl, tversion, cversion, expected_result, expected_data): """Test rogue_ep_coop_exception_mac_check with various scenarios.""" - result = run_check(cversion=script.AciVersion(cversion), tversion=script.AciVersion(tversion) if tversion else None) + result = run_check( + cversion=script.AciVersion(cversion) if cversion else None, + tversion=script.AciVersion(tversion) if tversion else None, + ) assert result.result == expected_result assert result.data == expected_data From 25531fc17aeb9d008c3a81e0360d1efbfc34e306 Mon Sep 17 00:00:00 2001 From: "GM.EXE" Date: Fri, 7 Aug 2026 18:42:35 -0400 Subject: [PATCH 26/44] mark as v4.2.0 --- aci-preupgrade-validation-script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index b8b68dff..af8ddb3c 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -38,7 +38,7 @@ import os import re -SCRIPT_VERSION = "v4.2.0-dev" +SCRIPT_VERSION = "v4.2.0" DEFAULT_TIMEOUT = 600 # sec # result constants DONE = 'DONE' @@ -7145,4 +7145,4 @@ def main(_args=None): msg = "Abort due to unexpected error - {}".format(e) prints(msg) log.error(msg, exc_info=True) - sys.exit(1) \ No newline at end of file + sys.exit(1) From 9aa983d529df1dca7951b2eb0a01bebe7fce05f9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 15:45:42 -0400 Subject: [PATCH 27/44] feat(ci): split integration tests with cversion/tversion override - Add resource_group for sequential execution across jobs - Split single integration test into 6 version-specific jobs: - 5.2(4d) -> 5.2(8f): 5.x internal upgrades - 5.2(4d) -> 6.0(2h): 5.x to 6.0.x upgrades - 5.2(6e) -> 6.0(5a): 5.x to 6.0.x upgrades - 5.2(8d) -> 6.1(1f): 5.x to 6.1.x upgrades - 5.3(2d) -> 6.1(4h): 5.x to 6.1.x upgrades - 6.0(2a) -> 6.1(4h): 6.x internal upgrades - Add --cversion and --tversion flags to runner.py invocation - Add timeout (45m) and retry on runner failure - Use YAML template to reduce duplication This enables testing version-dependent check logic across multiple upgrade paths while protecting scale fabrics via resource_group sequential execution. --- .gitlab-ci.yml | 85 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de14fdea..e377bac1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,9 +60,13 @@ test:py38: - .cache/pip -test:integration: +# ============================================================================= +# Integration Test Template +# ============================================================================= +.integration_template: &integration_template stage: integration_test image: python:3.8.10-slim + resource_group: aci_fabric_tests before_script: - echo 'ACQUIRE {http::proxy "$apt_proxy"}' >> /etc/apt/apt.conf - echo -e "$apt_source" > /etc/apt/sources.list @@ -73,13 +77,10 @@ test:integration: - pip install virtualenv - virtualenv venv38-integration - source venv38-integration/bin/activate - script: - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/requirements.txt/raw?ref=main" -o requirements.txt' - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/runner.py/raw?ref=main" -o runner.py' - - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - pip install -r requirements.txt - - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key artifacts: when: always paths: @@ -91,3 +92,79 @@ test:integration: paths: - venv38-integration/ - .cache/pip + timeout: 45m + retry: + max: 1 + when: runner_system_failure + +# ============================================================================= +# Integration Tests - 5.x Internal Upgrades +# ============================================================================= +test:integration:5.2.4d-to-5.2.8f: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(4d)" + FABRIC_TVERSION: "5.2(8f)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 5.x to 6.0.x Upgrades +# ============================================================================= +test:integration:5.2.4d-to-6.0.2h: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(4d)" + FABRIC_TVERSION: "6.0(2h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +test:integration:5.2.6e-to-6.0.5a: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(6e)" + FABRIC_TVERSION: "6.0(5a)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 5.x to 6.1.x Upgrades +# ============================================================================= +test:integration:5.2.8d-to-6.1.1f: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(8d)" + FABRIC_TVERSION: "6.1(1f)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +test:integration:5.3.2d-to-6.1.4h: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.3(2d)" + FABRIC_TVERSION: "6.1(4h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 6.x Internal Upgrades +# ============================================================================= +test:integration:6.0.2a-to-6.1.4h: + <<: *integration_template + variables: + FABRIC_CVERSION: "6.0(2a)" + FABRIC_TVERSION: "6.1(4h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" From f21b604fa5589fd6ae2fd2c14ec0db841dd2f871 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 19:18:12 -0400 Subject: [PATCH 28/44] fix(ci): avoid duplicate integration test directory change --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e377bac1..451b79be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,6 @@ test:integration:5.2.4d-to-5.2.8f: FABRIC_CVERSION: "5.2(4d)" FABRIC_TVERSION: "5.2(8f)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -119,7 +118,6 @@ test:integration:5.2.4d-to-6.0.2h: FABRIC_CVERSION: "5.2(4d)" FABRIC_TVERSION: "6.0(2h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -129,7 +127,6 @@ test:integration:5.2.6e-to-6.0.5a: FABRIC_CVERSION: "5.2(6e)" FABRIC_TVERSION: "6.0(5a)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -142,7 +139,6 @@ test:integration:5.2.8d-to-6.1.1f: FABRIC_CVERSION: "5.2(8d)" FABRIC_TVERSION: "6.1(1f)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -152,7 +148,6 @@ test:integration:5.3.2d-to-6.1.4h: FABRIC_CVERSION: "5.3(2d)" FABRIC_TVERSION: "6.1(4h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -165,6 +160,5 @@ test:integration:6.0.2a-to-6.1.4h: FABRIC_CVERSION: "6.0(2a)" FABRIC_TVERSION: "6.1(4h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" From 486022754f71443d9598b4513d8a64d16dd751e2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 10 Aug 2026 08:47:02 -0400 Subject: [PATCH 29/44] fix(ci): harden integration test coverage --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 451b79be..601ba9c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ # since gitlab ci can only cache local items. variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + INTEGRATION_RUN_ID: "$CI_PROJECT_ID-$CI_PIPELINE_ID-$CI_JOB_ID" stages: - unit_test @@ -85,6 +86,7 @@ test:py38: when: always paths: - $CI_PROJECT_DIR/tests/*.log + - $CI_PROJECT_DIR/tests/*.tgz cache: key: files: @@ -97,6 +99,15 @@ test:py38: max: 1 when: runner_system_failure +# ============================================================================= +# Integration Test - Actual Fabric Versions +# ============================================================================= +test:integration:actual-current-version: + <<: *integration_template + script: + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key + # ============================================================================= # Integration Tests - 5.x Internal Upgrades # ============================================================================= From 74dca5d389f160eede38ff6cfc6bb338453d4909 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 15:45:42 -0400 Subject: [PATCH 30/44] feat(ci): split integration tests with cversion/tversion override - Add resource_group for sequential execution across jobs - Split single integration test into 6 version-specific jobs: - 5.2(4d) -> 5.2(8f): 5.x internal upgrades - 5.2(4d) -> 6.0(2h): 5.x to 6.0.x upgrades - 5.2(6e) -> 6.0(5a): 5.x to 6.0.x upgrades - 5.2(8d) -> 6.1(1f): 5.x to 6.1.x upgrades - 5.3(2d) -> 6.1(4h): 5.x to 6.1.x upgrades - 6.0(2a) -> 6.1(4h): 6.x internal upgrades - Add --cversion and --tversion flags to runner.py invocation - Add timeout (45m) and retry on runner failure - Use YAML template to reduce duplication This enables testing version-dependent check logic across multiple upgrade paths while protecting scale fabrics via resource_group sequential execution. --- .gitlab-ci.yml | 85 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c123ea8..b4e01a44 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,9 +60,13 @@ test:py38: - .cache/pip -test:integration: +# ============================================================================= +# Integration Test Template +# ============================================================================= +.integration_template: &integration_template stage: integration_test image: python:3.8.10-slim + resource_group: aci_fabric_tests before_script: - echo 'ACQUIRE {http::proxy "$apt_proxy"}' >> /etc/apt/apt.conf - echo -e "$apt_source" > /etc/apt/sources.list @@ -73,13 +77,10 @@ test:integration: - pip install virtualenv - virtualenv venv38-integration - source venv38-integration/bin/activate - script: - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/requirements.txt/raw?ref=main" -o requirements.txt' - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/runner.py/raw?ref=main" -o runner.py' - - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - pip install -r requirements.txt - - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key artifacts: when: always expire_in: 7 days @@ -93,3 +94,79 @@ test:integration: paths: - venv38-integration/ - .cache/pip + timeout: 45m + retry: + max: 1 + when: runner_system_failure + +# ============================================================================= +# Integration Tests - 5.x Internal Upgrades +# ============================================================================= +test:integration:5.2.4d-to-5.2.8f: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(4d)" + FABRIC_TVERSION: "5.2(8f)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 5.x to 6.0.x Upgrades +# ============================================================================= +test:integration:5.2.4d-to-6.0.2h: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(4d)" + FABRIC_TVERSION: "6.0(2h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +test:integration:5.2.6e-to-6.0.5a: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(6e)" + FABRIC_TVERSION: "6.0(5a)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 5.x to 6.1.x Upgrades +# ============================================================================= +test:integration:5.2.8d-to-6.1.1f: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(8d)" + FABRIC_TVERSION: "6.1(1f)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +test:integration:5.3.2d-to-6.1.4h: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.3(2d)" + FABRIC_TVERSION: "6.1(4h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 6.x Internal Upgrades +# ============================================================================= +test:integration:6.0.2a-to-6.1.4h: + <<: *integration_template + variables: + FABRIC_CVERSION: "6.0(2a)" + FABRIC_TVERSION: "6.1(4h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" From de0c2ed06df76a5660d5a9fb6835910e5a82bcbd Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 19:18:12 -0400 Subject: [PATCH 31/44] fix(ci): avoid duplicate integration test directory change --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4e01a44..e62312e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,7 +108,6 @@ test:integration:5.2.4d-to-5.2.8f: FABRIC_CVERSION: "5.2(4d)" FABRIC_TVERSION: "5.2(8f)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -121,7 +120,6 @@ test:integration:5.2.4d-to-6.0.2h: FABRIC_CVERSION: "5.2(4d)" FABRIC_TVERSION: "6.0(2h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -131,7 +129,6 @@ test:integration:5.2.6e-to-6.0.5a: FABRIC_CVERSION: "5.2(6e)" FABRIC_TVERSION: "6.0(5a)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -144,7 +141,6 @@ test:integration:5.2.8d-to-6.1.1f: FABRIC_CVERSION: "5.2(8d)" FABRIC_TVERSION: "6.1(1f)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -154,7 +150,6 @@ test:integration:5.3.2d-to-6.1.4h: FABRIC_CVERSION: "5.3(2d)" FABRIC_TVERSION: "6.1(4h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -167,6 +162,5 @@ test:integration:6.0.2a-to-6.1.4h: FABRIC_CVERSION: "6.0(2a)" FABRIC_TVERSION: "6.1(4h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" From d1819c2b682e83e2c72f4cd7dbf41ec1acee9273 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 10 Aug 2026 08:47:02 -0400 Subject: [PATCH 32/44] fix(ci): harden integration test coverage --- .gitlab-ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e62312e8..50a64999 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ # since gitlab ci can only cache local items. variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + INTEGRATION_RUN_ID: "$CI_PROJECT_ID-$CI_PIPELINE_ID-$CI_JOB_ID" stages: - unit_test @@ -99,6 +100,15 @@ test:py38: max: 1 when: runner_system_failure +# ============================================================================= +# Integration Test - Actual Fabric Versions +# ============================================================================= +test:integration:actual-current-version: + <<: *integration_template + script: + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key + # ============================================================================= # Integration Tests - 5.x Internal Upgrades # ============================================================================= From 9a8448e41465e0edf2b630b8b48c9322c5b60bd1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 12 Aug 2026 18:37:30 -0400 Subject: [PATCH 33/44] Retry empty APIC database counter reads --- aci-preupgrade-validation-script.py | 13 +++++++- .../test_apic_database_size_check.py | 33 +++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 418dfd88..f9306a72 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6129,7 +6129,18 @@ def apic_database_size_check(cversion, **kwargs): for attempt in range(1, counter_read_attempts + 1): try: class_stats = run_cmd(collect_stats_cmd, splitlines=True) - break + if class_stats: + break + if attempt < counter_read_attempts: + log.warning( + 'Counter read returned no data for APIC %s %s ' + '(attempt %s/%s)', + id, + dme, + attempt, + counter_read_attempts, + ) + time.sleep(counter_read_retry_delay) except subprocess.CalledProcessError as error: error_output = error.output if isinstance(error_output, bytes): diff --git a/tests/checks/apic_database_size_check/test_apic_database_size_check.py b/tests/checks/apic_database_size_check/test_apic_database_size_check.py index f1cbeb54..a4b058b1 100644 --- a/tests/checks/apic_database_size_check/test_apic_database_size_check.py +++ b/tests/checks/apic_database_size_check/test_apic_database_size_check.py @@ -464,6 +464,39 @@ def transient_run_cmd(cmd, splitlines=False): assert sleep_calls.count(1) == 4 +def test_empty_counter_read_succeeds_on_retry( + run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs, monkeypatch +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + successful_outputs = { + apic2_vmm_cat: mitcounters_neg, + apic2_pm_cat: mitcounters_neg, + apic2_evm_cat: mitcounters_neg, + apic2_pd_cat: mitcounters_neg, + } + call_counts = {} + sleep_calls = [] + + def transient_run_cmd(cmd, splitlines=False): + call_counts[cmd] = call_counts.get(cmd, 0) + 1 + if call_counts[cmd] == 1: + return [] + output = successful_outputs[cmd] + return output.splitlines() if splitlines else output + + monkeypatch.setattr(script, "run_cmd", transient_run_cmd) + monkeypatch.setattr(script.time, "sleep", sleep_calls.append) + + result = run_check(cversion=script.AciVersion("6.0(8f)")) + + assert result.result == script.PASS + assert all(call_count == 2 for call_count in call_counts.values()) + assert sleep_calls.count(1) == 4 + + def test_object_counters_are_sorted_before_top_four_and_thresholded( run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs ): From 5979f4637cf16a333ebdecc1f74a7ed13bf66c0b Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 13 Aug 2026 10:07:01 -0400 Subject: [PATCH 34/44] Expand APIC 6.1(5) CIMC release-note support --- aci-preupgrade-validation-script.py | 43 +++++++++++++++++-- docs/docs/validations.md | 2 +- .../test_cimc_compatibilty_check.py | 43 +++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 8b5af5cb..936f8e9e 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3736,6 +3736,43 @@ def vpc_paired_switches_check(vpc_node_ids, fabric_nodes, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) +# APIC 6.1(5) release notes, verified 2026-08-10. +CIMC_RELEASE_NOTE_SUPPORT_615_M5 = ( + "4.3(2.260007)", + "4.3(2.250016)", + "4.3(2.240077)", + "4.3(2.240009)", + "4.3(2.230207)", + "4.2(3e)", + "4.2(3b)", + "4.2(2a)", + "4.1(3m)", + "4.1(3f)", + "4.1(3d)", + "4.1(3c)", +) + +CIMC_RELEASE_NOTE_SUPPORT_615_M6 = ( + "6.0(2.260044)", + "6.0(1.250192)", + "6.0(1.250131)", + "4.3(6.250053)", + "4.3(4.252002)", + "4.3(4.241063)", + "4.3(2.240009)", + "4.3(2.230207)", + "4.2(3e)", + "4.2(3b)", +) + +CIMC_RELEASE_NOTE_SUPPORT = { + ("6.1(5)", "apicl3"): CIMC_RELEASE_NOTE_SUPPORT_615_M5, + ("6.1(5)", "apicm3"): CIMC_RELEASE_NOTE_SUPPORT_615_M5, + ("6.1(5)", "apicl4"): CIMC_RELEASE_NOTE_SUPPORT_615_M6, + ("6.1(5)", "apicm4"): CIMC_RELEASE_NOTE_SUPPORT_615_M6, +} + + @check_wrapper(check_title="APIC CIMC Compatibility") def cimc_compatibilty_check(tversion, cversion, **kwargs): result = FAIL_UF @@ -3780,10 +3817,8 @@ def cimc_compatibilty_check(tversion, cversion, **kwargs): if compatMo and recommended_cimc: if not is_firstver_gt_secondver(current_cimc, "3.0(3a)"): warning = "Multi-step Upgrade may be required, check UCS CIMC Matrix." - release_note_supported = ( - tversion.simple_version == "6.1(5)" - and model in ("apicl3", "apicm3", "apicl4", "apicm4") - and current_cimc == "4.2(3e)" + release_note_supported = current_cimc in CIMC_RELEASE_NOTE_SUPPORT.get( + (tversion.simple_version, model), () ) if not release_note_supported and not is_firstver_gt_secondver(current_cimc, recommended_cimc): nodeid = eqptCh['eqptCh']['attributes']['dn'].split('/')[2] diff --git a/docs/docs/validations.md b/docs/docs/validations.md index e47f6b2e..ed80300c 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -260,7 +260,7 @@ The script checks the minimum recommended CIMC version for the given APIC model As the `compatRsSuppHw` object recommendation is strictly tied to the target software image, it is possible that the [Release Note Documentation][4] for your model/target version has a different recommendation than what the software recommends. Always check the release note of your Target version and APIC model to ensure you are getting the latest recommendations. -The APIC 6.1(5) release notes explicitly support CIMC 4.2(3e) on UCS C220/C240 M5 (APIC-L3/M3) and UCS C225 M6 (APIC-L4/M4). The check accepts that exact release-note-supported combination even when the image catalog recommends a newer CIMC version. +The APIC 6.1(5) release notes explicitly support multiple CIMC releases on UCS C220/C240 M5 (APIC-L3/M3) and UCS C225 M6 (APIC-L4/M4) that may be older than the image catalog recommendation. The check uses an embedded, model-specific list for those release-note-supported combinations before applying the image catalog recommendation to other CIMC releases. Due to the defect CSCwo74485, APIC-SERVER-M4/L4 systems will fail to boot correctly after upgrading CIMC firmware to version 4.3.5 or later while on Non-fixed APIC releases 5.3.x/6.0.9d/6.1(3g) and below. Upgrade the APIC software first, then proceed with the CIMC upgrade for the releases 6.0.9e/ 6.1.4h and above, will avoid this issue. Follow the software advisory for this defect [CSCwo74485][73]. diff --git a/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py b/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py index 6c8b14f2..1092eca9 100644 --- a/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py +++ b/tests/checks/cimc_compatibilty_check/test_cimc_compatibilty_check.py @@ -32,6 +32,40 @@ compatRsSuppHwM4_api: read_data(dir, "compatRsSuppHw_615_M6.json"), } +release_note_model_data = { + "apicl3": ("APIC-SERVER-L3", compatRsSuppHwL3_api, "compatRsSuppHw_615_M5.json"), + "apicm3": ("APIC-SERVER-M3", compatRsSuppHwM3_api, "compatRsSuppHw_615_M5.json"), + "apicl4": ("APIC-SERVER-L4", compatRsSuppHwL4_api, "compatRsSuppHw_615_M6.json"), + "apicm4": ("APIC-SERVER-M4", compatRsSuppHwM4_api, "compatRsSuppHw_615_M6.json"), +} + + +def release_note_supported_outputs(model, cimc_version): + apic_model, compat_api, compat_fixture = release_note_model_data[model] + return { + eqptCh_api: [ + { + "eqptCh": { + "attributes": { + "cimcVersion": cimc_version, + "descr": apic_model, + "dn": "topology/pod-1/node-1/sys/ch", + "model": apic_model, + } + } + } + ], + compat_api: read_data(dir, compat_fixture), + } + + +release_note_supported_cases = [ + release_note_supported_outputs(model, cimc_version) + for (target, model), cimc_versions in script.CIMC_RELEASE_NOTE_SUPPORT.items() + if target == "6.1(5)" + for cimc_version in cimc_versions +] + @pytest.mark.parametrize( "icurl_outputs, tversion, cversion, expected_result", [ @@ -137,3 +171,12 @@ def test_logic(run_check, mock_icurl, tversion, cversion, expected_result): result = run_check(tversion=script.AciVersion(tversion), cversion=script.AciVersion(cversion) if cversion is not None else None) assert result.result == expected_result + + +@pytest.mark.parametrize("icurl_outputs", release_note_supported_cases) +def test_release_note_supported_versions(run_check, mock_icurl): + result = run_check( + tversion=script.AciVersion("6.1(5e)"), + cversion=script.AciVersion("5.2(8g)"), + ) + assert result.result == script.PASS From 1028194ee008fbb1f137dc2841ddd78e1ecbf6dc Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 15:45:42 -0400 Subject: [PATCH 35/44] feat(ci): split integration tests with cversion/tversion override - Add resource_group for sequential execution across jobs - Split single integration test into 6 version-specific jobs: - 5.2(4d) -> 5.2(8f): 5.x internal upgrades - 5.2(4d) -> 6.0(2h): 5.x to 6.0.x upgrades - 5.2(6e) -> 6.0(5a): 5.x to 6.0.x upgrades - 5.2(8d) -> 6.1(1f): 5.x to 6.1.x upgrades - 5.3(2d) -> 6.1(4h): 5.x to 6.1.x upgrades - 6.0(2a) -> 6.1(4h): 6.x internal upgrades - Add --cversion and --tversion flags to runner.py invocation - Add timeout (45m) and retry on runner failure - Use YAML template to reduce duplication This enables testing version-dependent check logic across multiple upgrade paths while protecting scale fabrics via resource_group sequential execution. --- .gitlab-ci.yml | 85 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de14fdea..e377bac1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,9 +60,13 @@ test:py38: - .cache/pip -test:integration: +# ============================================================================= +# Integration Test Template +# ============================================================================= +.integration_template: &integration_template stage: integration_test image: python:3.8.10-slim + resource_group: aci_fabric_tests before_script: - echo 'ACQUIRE {http::proxy "$apt_proxy"}' >> /etc/apt/apt.conf - echo -e "$apt_source" > /etc/apt/sources.list @@ -73,13 +77,10 @@ test:integration: - pip install virtualenv - virtualenv venv38-integration - source venv38-integration/bin/activate - script: - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/requirements.txt/raw?ref=main" -o requirements.txt' - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/runner.py/raw?ref=main" -o runner.py' - - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - pip install -r requirements.txt - - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key artifacts: when: always paths: @@ -91,3 +92,79 @@ test:integration: paths: - venv38-integration/ - .cache/pip + timeout: 45m + retry: + max: 1 + when: runner_system_failure + +# ============================================================================= +# Integration Tests - 5.x Internal Upgrades +# ============================================================================= +test:integration:5.2.4d-to-5.2.8f: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(4d)" + FABRIC_TVERSION: "5.2(8f)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 5.x to 6.0.x Upgrades +# ============================================================================= +test:integration:5.2.4d-to-6.0.2h: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(4d)" + FABRIC_TVERSION: "6.0(2h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +test:integration:5.2.6e-to-6.0.5a: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(6e)" + FABRIC_TVERSION: "6.0(5a)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 5.x to 6.1.x Upgrades +# ============================================================================= +test:integration:5.2.8d-to-6.1.1f: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.2(8d)" + FABRIC_TVERSION: "6.1(1f)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +test:integration:5.3.2d-to-6.1.4h: + <<: *integration_template + variables: + FABRIC_CVERSION: "5.3(2d)" + FABRIC_TVERSION: "6.1(4h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" + +# ============================================================================= +# Integration Tests - 6.x Internal Upgrades +# ============================================================================= +test:integration:6.0.2a-to-6.1.4h: + <<: *integration_template + variables: + FABRIC_CVERSION: "6.0(2a)" + FABRIC_TVERSION: "6.1(4h)" + script: + - cd tests + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" From adf97ae15d9160caea09e77c5a7187d24c6dd90e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 7 Aug 2026 19:18:12 -0400 Subject: [PATCH 36/44] fix(ci): avoid duplicate integration test directory change --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e377bac1..451b79be 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -106,7 +106,6 @@ test:integration:5.2.4d-to-5.2.8f: FABRIC_CVERSION: "5.2(4d)" FABRIC_TVERSION: "5.2(8f)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -119,7 +118,6 @@ test:integration:5.2.4d-to-6.0.2h: FABRIC_CVERSION: "5.2(4d)" FABRIC_TVERSION: "6.0(2h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -129,7 +127,6 @@ test:integration:5.2.6e-to-6.0.5a: FABRIC_CVERSION: "5.2(6e)" FABRIC_TVERSION: "6.0(5a)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -142,7 +139,6 @@ test:integration:5.2.8d-to-6.1.1f: FABRIC_CVERSION: "5.2(8d)" FABRIC_TVERSION: "6.1(1f)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -152,7 +148,6 @@ test:integration:5.3.2d-to-6.1.4h: FABRIC_CVERSION: "5.3(2d)" FABRIC_TVERSION: "6.1(4h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" @@ -165,6 +160,5 @@ test:integration:6.0.2a-to-6.1.4h: FABRIC_CVERSION: "6.0(2a)" FABRIC_TVERSION: "6.1(4h)" script: - - cd tests - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key --cversion "$FABRIC_CVERSION" --tversion "$FABRIC_TVERSION" From b43f42e5a6acd7786ac107a90ec21a03f79c80ac Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 10 Aug 2026 08:47:02 -0400 Subject: [PATCH 37/44] fix(ci): harden integration test coverage --- .gitlab-ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 451b79be..601ba9c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ # since gitlab ci can only cache local items. variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + INTEGRATION_RUN_ID: "$CI_PROJECT_ID-$CI_PIPELINE_ID-$CI_JOB_ID" stages: - unit_test @@ -85,6 +86,7 @@ test:py38: when: always paths: - $CI_PROJECT_DIR/tests/*.log + - $CI_PROJECT_DIR/tests/*.tgz cache: key: files: @@ -97,6 +99,15 @@ test:py38: max: 1 when: runner_system_failure +# ============================================================================= +# Integration Test - Actual Fabric Versions +# ============================================================================= +test:integration:actual-current-version: + <<: *integration_template + script: + - 'curl --header "PRIVATE-TOKEN: $integration_repo_token" "$repo_url/test_fabrics.yaml/raw?ref=main" -o test_fabrics.yaml' + - python3 runner.py -f ../aci-preupgrade-validation-script.py -i test_fabrics.yaml -k $decode_key + # ============================================================================= # Integration Tests - 5.x Internal Upgrades # ============================================================================= From e9a99ea979bab28241177c6f970204fca013ba5c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 13 Aug 2026 12:08:12 -0400 Subject: [PATCH 38/44] Move CIMC support constants to module configuration --- aci-preupgrade-validation-script.py | 70 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 936f8e9e..30924abd 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -52,6 +52,39 @@ # message constants TVER_MISSING = "Target version not supplied. Skipping." VER_NOT_AFFECTED = "Version not affected." +# APIC 6.1(5) release notes, verified 2026-08-10. +CIMC_RELEASE_NOTE_SUPPORT_615_M5 = ( + "4.3(2.260007)", + "4.3(2.250016)", + "4.3(2.240077)", + "4.3(2.240009)", + "4.3(2.230207)", + "4.2(3e)", + "4.2(3b)", + "4.2(2a)", + "4.1(3m)", + "4.1(3f)", + "4.1(3d)", + "4.1(3c)", +) +CIMC_RELEASE_NOTE_SUPPORT_615_M6 = ( + "6.0(2.260044)", + "6.0(1.250192)", + "6.0(1.250131)", + "4.3(6.250053)", + "4.3(4.252002)", + "4.3(4.241063)", + "4.3(2.240009)", + "4.3(2.230207)", + "4.2(3e)", + "4.2(3b)", +) +CIMC_RELEASE_NOTE_SUPPORT = { + ("6.1(5)", "apicl3"): CIMC_RELEASE_NOTE_SUPPORT_615_M5, + ("6.1(5)", "apicm3"): CIMC_RELEASE_NOTE_SUPPORT_615_M5, + ("6.1(5)", "apicl4"): CIMC_RELEASE_NOTE_SUPPORT_615_M6, + ("6.1(5)", "apicm4"): CIMC_RELEASE_NOTE_SUPPORT_615_M6, +} # regex constants node_regex = r'topology/pod-(?P\d+)/node-(?P\d+)' port_regex = node_regex + r'/sys/phys-\[(?P.+)\]' @@ -3736,43 +3769,6 @@ def vpc_paired_switches_check(vpc_node_ids, fabric_nodes, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) -# APIC 6.1(5) release notes, verified 2026-08-10. -CIMC_RELEASE_NOTE_SUPPORT_615_M5 = ( - "4.3(2.260007)", - "4.3(2.250016)", - "4.3(2.240077)", - "4.3(2.240009)", - "4.3(2.230207)", - "4.2(3e)", - "4.2(3b)", - "4.2(2a)", - "4.1(3m)", - "4.1(3f)", - "4.1(3d)", - "4.1(3c)", -) - -CIMC_RELEASE_NOTE_SUPPORT_615_M6 = ( - "6.0(2.260044)", - "6.0(1.250192)", - "6.0(1.250131)", - "4.3(6.250053)", - "4.3(4.252002)", - "4.3(4.241063)", - "4.3(2.240009)", - "4.3(2.230207)", - "4.2(3e)", - "4.2(3b)", -) - -CIMC_RELEASE_NOTE_SUPPORT = { - ("6.1(5)", "apicl3"): CIMC_RELEASE_NOTE_SUPPORT_615_M5, - ("6.1(5)", "apicm3"): CIMC_RELEASE_NOTE_SUPPORT_615_M5, - ("6.1(5)", "apicl4"): CIMC_RELEASE_NOTE_SUPPORT_615_M6, - ("6.1(5)", "apicm4"): CIMC_RELEASE_NOTE_SUPPORT_615_M6, -} - - @check_wrapper(check_title="APIC CIMC Compatibility") def cimc_compatibilty_check(tversion, cversion, **kwargs): result = FAIL_UF From a6714a0ec3d7aa48b40c423de77e5fc8ba9559c5 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 14 Aug 2026 10:05:17 -0400 Subject: [PATCH 39/44] Enforce API-only check classification --- aci-preupgrade-validation-script.py | 2 +- tests/test_CheckManager.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index f5664765..cf4dc7ff 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -7101,7 +7101,6 @@ class CheckManager: l3out_route_map_direction_check, l3out_route_map_missing_target_check, l3out_overlapping_loopback_check, - intersight_upgrade_status_check, isis_redis_metric_mpod_msite_check, bgp_golf_route_target_type_check, docker0_subnet_overlap_check, @@ -7171,6 +7170,7 @@ class CheckManager: cli_checks = [ # General apic_database_size_check, + intersight_upgrade_status_check, # Bugs apic_ca_cert_validation, diff --git a/tests/test_CheckManager.py b/tests/test_CheckManager.py index fe4a7eea..94266c76 100644 --- a/tests/test_CheckManager.py +++ b/tests/test_CheckManager.py @@ -1,4 +1,5 @@ import pytest +import ast import importlib import logging import time @@ -172,6 +173,56 @@ def test_total_checks(api_only, debug_function, expected_total): assert cm.total_checks == expected_total +def test_api_checks_do_not_run_ssh_or_cli_commands(): + with open(script.__file__, "r") as source_file: + module = ast.parse(source_file.read()) + + functions = { + node.name: node + for node in module.body + if isinstance(node, ast.FunctionDef) + } + approved_api_boundaries = {"icurl"} + forbidden_calls = {"Connection", "run_cmd", "os.system", "os.popen"} + forbidden_prefixes = ("subprocess.", "pexpect.") + + def get_call_name(node): + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + parent = get_call_name(node.value) + return "{}.{}".format(parent, node.attr) if parent else node.attr + return "" + + def find_forbidden_calls(function_name, visited=None): + if visited is None: + visited = set() + if function_name in visited or function_name in approved_api_boundaries: + return set() + visited.add(function_name) + + findings = set() + function = functions.get(function_name) + if function is None: + return findings + for node in ast.walk(function): + if not isinstance(node, ast.Call): + continue + call_name = get_call_name(node.func) + if call_name in forbidden_calls or call_name.startswith(forbidden_prefixes): + findings.add(call_name) + elif call_name in functions: + findings.update(find_forbidden_calls(call_name, visited)) + return findings + + violations = { + check.__name__: sorted(find_forbidden_calls(check.__name__)) + for check in CheckManager.api_checks + if find_forbidden_calls(check.__name__) + } + assert violations == {} + + def test_exception_in_initialize(): """Exception in initialize is not captured by CheckManager. The exception should go up to the script's main() and abort the script From f96e4bfbf82a1731bd20575405f444eaab7ffd07 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 14 Aug 2026 10:21:23 -0400 Subject: [PATCH 40/44] Allow direct icurl in API checks --- aci-preupgrade-validation-script.py | 11 ++++++----- tests/test_CheckManager.py | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index cf4dc7ff..1c375a2c 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3844,10 +3844,11 @@ def intersight_upgrade_status_check(**kwargs): recommended_action = 'Wait a few minutes for the upgrade to complete' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#intersight-device-connector-upgrade-status' - cmd = ['icurl', '-gks', 'https://127.0.0.1/connector/UpgradeStatus'] - - log.info('cmd = ' + ' '.join(cmd)) - response = subprocess.check_output(cmd) + # The API-only container permits direct icurl access to this endpoint. + log.info('cmd = icurl -gks https://127.0.0.1/connector/UpgradeStatus') + response = subprocess.check_output([ + 'icurl', '-gks', 'https://127.0.0.1/connector/UpgradeStatus' + ]) try: resp_json = json.loads(response) @@ -7101,6 +7102,7 @@ class CheckManager: l3out_route_map_direction_check, l3out_route_map_missing_target_check, l3out_overlapping_loopback_check, + intersight_upgrade_status_check, isis_redis_metric_mpod_msite_check, bgp_golf_route_target_type_check, docker0_subnet_overlap_check, @@ -7170,7 +7172,6 @@ class CheckManager: cli_checks = [ # General apic_database_size_check, - intersight_upgrade_status_check, # Bugs apic_ca_cert_validation, diff --git a/tests/test_CheckManager.py b/tests/test_CheckManager.py index 94266c76..d9440da8 100644 --- a/tests/test_CheckManager.py +++ b/tests/test_CheckManager.py @@ -173,7 +173,7 @@ def test_total_checks(api_only, debug_function, expected_total): assert cm.total_checks == expected_total -def test_api_checks_do_not_run_ssh_or_cli_commands(): +def test_api_checks_only_use_approved_external_commands(): with open(script.__file__, "r") as source_file: module = ast.parse(source_file.read()) @@ -194,6 +194,15 @@ def get_call_name(node): return "{}.{}".format(parent, node.attr) if parent else node.attr return "" + def is_literal_icurl_call(node): + if not get_call_name(node.func).startswith("subprocess.") or not node.args: + return False + command = node.args[0] + if not isinstance(command, (ast.List, ast.Tuple)) or not command.elts: + return False + executable = command.elts[0] + return isinstance(executable, ast.Str) and executable.s == "icurl" + def find_forbidden_calls(function_name, visited=None): if visited is None: visited = set() @@ -209,7 +218,10 @@ def find_forbidden_calls(function_name, visited=None): if not isinstance(node, ast.Call): continue call_name = get_call_name(node.func) - if call_name in forbidden_calls or call_name.startswith(forbidden_prefixes): + if ( + call_name in forbidden_calls + or call_name.startswith(forbidden_prefixes) + ) and not is_literal_icurl_call(node): findings.add(call_name) elif call_name in functions: findings.update(find_forbidden_calls(call_name, visited)) From 8af80b7c5488ee05f1168195038be60cfa214dcf Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 14 Aug 2026 10:30:04 -0400 Subject: [PATCH 41/44] Preserve Intersight icurl command handling --- aci-preupgrade-validation-script.py | 9 ++++----- tests/test_CheckManager.py | 25 ++++++++++++++++++++----- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 1c375a2c..f5664765 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -3844,11 +3844,10 @@ def intersight_upgrade_status_check(**kwargs): recommended_action = 'Wait a few minutes for the upgrade to complete' doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#intersight-device-connector-upgrade-status' - # The API-only container permits direct icurl access to this endpoint. - log.info('cmd = icurl -gks https://127.0.0.1/connector/UpgradeStatus') - response = subprocess.check_output([ - 'icurl', '-gks', 'https://127.0.0.1/connector/UpgradeStatus' - ]) + cmd = ['icurl', '-gks', 'https://127.0.0.1/connector/UpgradeStatus'] + + log.info('cmd = ' + ' '.join(cmd)) + response = subprocess.check_output(cmd) try: resp_json = json.loads(response) diff --git a/tests/test_CheckManager.py b/tests/test_CheckManager.py index d9440da8..82d6b116 100644 --- a/tests/test_CheckManager.py +++ b/tests/test_CheckManager.py @@ -194,14 +194,29 @@ def get_call_name(node): return "{}.{}".format(parent, node.attr) if parent else node.attr return "" - def is_literal_icurl_call(node): + def is_literal_icurl_command(node): + if not isinstance(node, (ast.List, ast.Tuple)) or not node.elts: + return False + executable = node.elts[0] + return isinstance(executable, ast.Str) and executable.s == "icurl" + + def is_literal_icurl_call(node, function): if not get_call_name(node.func).startswith("subprocess.") or not node.args: return False command = node.args[0] - if not isinstance(command, (ast.List, ast.Tuple)) or not command.elts: + if is_literal_icurl_command(command): + return True + if not isinstance(command, ast.Name): return False - executable = command.elts[0] - return isinstance(executable, ast.Str) and executable.s == "icurl" + return any( + isinstance(candidate, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == command.id + for target in candidate.targets + ) + and is_literal_icurl_command(candidate.value) + for candidate in ast.walk(function) + ) def find_forbidden_calls(function_name, visited=None): if visited is None: @@ -221,7 +236,7 @@ def find_forbidden_calls(function_name, visited=None): if ( call_name in forbidden_calls or call_name.startswith(forbidden_prefixes) - ) and not is_literal_icurl_call(node): + ) and not is_literal_icurl_call(node, function): findings.add(call_name) elif call_name in functions: findings.update(find_forbidden_calls(call_name, visited)) From a3fe429416dd2f971eb0988fe929560a092f0818 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 14 Aug 2026 11:52:52 -0400 Subject: [PATCH 42/44] Fix FX3 memory query filter limit --- aci-preupgrade-validation-script.py | 8 +--- .../procMemUsage_lt32gb_unaffected.json | 11 +++++ ...st_n9k_c93180yc_fx3_switch_memory_check.py | 47 +++++++++++++++++-- 3 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb_unaffected.json diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index f5664765..4573ffdc 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6828,12 +6828,8 @@ def n9k_c93180yc_fx3_switch_memory_check(fabric_nodes, **kwargs): result = NA msg = 'No N9K-C93180YC-FX3 switches found. Skipping.' else: - node_ids = [node['fabricNode']['attributes']['id'] for node in affected_nodes] - node_filter = 'or({})'.format(','.join( - 'wcard(procMemUsage.dn,"node-{}/")'.format(nid) for nid in node_ids - )) - query = 'procMemUsage.json?query-target-filter=and({},wcard(procMemUsage.dn,"memusage-sup"),lt(procMemUsage.Total,"{}"))'.format( - node_filter, min_memory_kb + query = 'procMemUsage.json?query-target-filter=and(wcard(procMemUsage.dn,"memusage-sup"),lt(procMemUsage.Total,"{}"))'.format( + min_memory_kb ) proc_mem_mos = icurl('class', query) diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb_unaffected.json b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb_unaffected.json new file mode 100644 index 00000000..c9504f09 --- /dev/null +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/procMemUsage_lt32gb_unaffected.json @@ -0,0 +1,11 @@ +[ + { + "procMemUsage": { + "attributes": { + "dn": "topology/pod-1/node-102/sys/procmem/memusage-sup", + "Modname": "sup", + "Total": "16000000" + } + } + } +] \ No newline at end of file diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py index 45f46006..fc9df742 100644 --- a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py @@ -11,8 +11,25 @@ test_function = "n9k_c93180yc_fx3_switch_memory_check" -# icurl queries - filtered by affected node IDs and memory threshold -proc_mem_query_node101 = 'procMemUsage.json?query-target-filter=and(or(wcard(procMemUsage.dn,"node-101/")),wcard(procMemUsage.dn,"memusage-sup"),lt(procMemUsage.Total,"32000000"))' +# icurl query - affected node IDs are filtered from the response +proc_mem_query = 'procMemUsage.json?query-target-filter=and(wcard(procMemUsage.dn,"memusage-sup"),lt(procMemUsage.Total,"32000000"))' + + +def make_fx3_nodes(count): + return [ + { + "fabricNode": { + "attributes": { + "dn": "topology/pod-1/node-{}".format(node_id), + "id": str(node_id), + "name": "leaf{}".format(node_id), + "model": "N9K-C93180YC-FX3", + "role": "leaf", + } + } + } + for node_id in range(101, 101 + count) + ] @pytest.mark.parametrize( @@ -38,7 +55,7 @@ ( read_data(dir, "fabricNode_one.json"), { - proc_mem_query_node101: [], + proc_mem_query: [], }, script.PASS, '', @@ -48,7 +65,27 @@ ( read_data(dir, "fabricNode_two.json"), { - proc_mem_query_node101: [], + proc_mem_query: [], + }, + script.PASS, + '', + [], + ), + # Low-memory results for other switch models are ignored + ( + read_data(dir, "fabricNode_two.json"), + { + proc_mem_query: read_data(dir, "procMemUsage_lt32gb_unaffected.json"), + }, + script.PASS, + '', + [], + ), + # Query remains below APIC's 20-expression limit on large fabrics + ( + make_fx3_nodes(55), + { + proc_mem_query: [], }, script.PASS, '', @@ -58,7 +95,7 @@ ( read_data(dir, "fabricNode_one.json"), { - proc_mem_query_node101: read_data(dir, "procMemUsage_lt32gb.json"), + proc_mem_query: read_data(dir, "procMemUsage_lt32gb.json"), }, script.FAIL_O, ( From f652abfc0c441fadd2a42d9eed727af78810500a Mon Sep 17 00:00:00 2001 From: "GM.EXE" Date: Tue, 18 Aug 2026 15:17:36 -0400 Subject: [PATCH 43/44] Document scalability spillover risk during upgrades (#429) * docs: explain scalability spillover risk --------- Co-authored-by: takishida <38262981+takishida@users.noreply.github.com> --- docs/docs/validations.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 3eb15dcc..717c1757 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -1393,7 +1393,9 @@ The fault F3545 occurs when the switch fails to activate a contract rule (zoning The script checks faults raised under `eqptcapacityEntity`, which are TCA (Threshold Crossed Alert) faults for various objects monitored in the **Capacity Dashboard** from `Operations > Capacity Dashboard > Leaf Capacity` on the Cisco APIC GUI. -It is important to ensure that any capacity does not exceed its limit. When it's exceeding the limit, it may cause inconsistency on resources that are deployed before and after an upgrade just like it was warned for [Policy CAM Programming for Contracts (F3545) and L3Out Subnets Programming for Contracts (F3544)][f15]. +A raised TCA indicates that a configured capacity threshold was crossed; it does not necessarily indicate a current outage. The script reports `FAIL - OUTAGE WARNING!!` because switch reboots during an upgrade can cause endpoints, routes, contracts, and other programmed resources to be temporarily redistributed to the remaining switches. This spillover can push a resource that is already near its limit, such as Policy CAM at 90%, beyond supported capacity and cause programming failures or traffic disruption. Similar post-reboot resource inconsistencies are described under [Policy CAM Programming for Contracts (F3545) and L3Out Subnets Programming for Contracts (F3544)][f15]. + +Before upgrading, review the affected node and resource under `Operations > Capacity Dashboard > Leaf Capacity` and examine the capacity headroom based on your network design, server connectivity and so on. Examples of what's monitored via `Operations > Capacity Dashboard > Leaf Capacity` are the number of endpoints such as MAC (Learned), IPv4 (Learned), Policy CAM, LPM, host routes, VLANs and so on. From 4f24dd3f4053d818abb0d0e21af0a18e97805914 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 18 Aug 2026 16:59:36 -0400 Subject: [PATCH 44/44] test: expect string-normalized result data Fixes #432 --- .../test_equipment_disk_limits_exceeded.py | 10 +++++----- .../test_n9k_c93180yc_fx3_switch_memory_check.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py b/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py index ad1cff4d..d4238cb1 100644 --- a/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py +++ b/tests/checks/equipment_disk_limits_exceeded/test_equipment_disk_limits_exceeded.py @@ -28,9 +28,9 @@ {f182x_api: read_data(dir, "faultInst_pos.json")}, script.FAIL_UF, [ - ["1", "101", "F1820", 98, "Disk usage for /mnt/ifc/log is high on node 101 of fabric POD1 with a hostname leaf1"], - ["1", "102", "F1821", 97, "Disk usage for /mnt/ifc/cfg is high on node 102 of fabric POD1 with a hostname leaf2"], - ["1", "104", "F1821", 100, "Disk usage for / is high on node 104 of fabric POD1 with a hostname LEAF-104"], + ["1", "101", "F1820", "98", "Disk usage for /mnt/ifc/log is high on node 101 of fabric POD1 with a hostname leaf1"], + ["1", "102", "F1821", "97", "Disk usage for /mnt/ifc/cfg is high on node 102 of fabric POD1 with a hostname leaf2"], + ["1", "104", "F1821", "100", "Disk usage for / is high on node 104 of fabric POD1 with a hostname LEAF-104"], ], [[ "topology/pod-1/node-[103]/sys/eqptcapacity/fspartition-ifc:cfg/fault-F1821", @@ -41,7 +41,7 @@ ( {f182x_api: read_data(dir, "faultInst_compact.json")}, script.FAIL_UF, - [["1", "107", "F1820", 81, "Disk usage for /mnt/ifc/cfg is above normal"]], + [["1", "107", "F1820", "81", "Disk usage for /mnt/ifc/cfg is above normal"]], [], ), ], @@ -58,4 +58,4 @@ def test_logic( assert result.data == expected_data assert result.unformatted_data == expected_unformatted_data for row in result.data: - assert isinstance(row[3], int) + assert isinstance(row[3], str) diff --git a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py index fc9df742..6458b218 100644 --- a/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py +++ b/tests/checks/n9k_c93180yc_fx3_switch_memory_check/test_n9k_c93180yc_fx3_switch_memory_check.py @@ -103,7 +103,7 @@ def make_fx3_nodes(count): 'One or more switches with less than 32GB of memory may experience service instability. ' 'Upgrade the switch memory to at least 32GB.' ), - [["101", "leaf101", "N9K-C93180YC-FX3", 16.0]], + [["101", "leaf101", "N9K-C93180YC-FX3", "16.0"]], ), ], )