diff --git a/roles/ocp4_workload_openshift_virtualization/defaults/main.yml b/roles/ocp4_workload_openshift_virtualization/defaults/main.yml index 5f62d57b..49973972 100644 --- a/roles/ocp4_workload_openshift_virtualization/defaults/main.yml +++ b/roles/ocp4_workload_openshift_virtualization/defaults/main.yml @@ -41,6 +41,11 @@ ocp4_workload_openshift_virtualization_enabled_nested_kvm: false # Only set to false if a lab needs students to create it ocp4_workload_openshift_virtualization_deploy_hyperconverged: true +# Retry HCO readiness queries through temporary API/webhook outages during node +# rollouts. Defaults allow approximately 20 minutes, plus API request duration. +ocp4_workload_openshift_virtualization_hco_wait_retries: 120 +ocp4_workload_openshift_virtualization_hco_wait_delay: 10 + # Install virtctl - requires hyperconverged to be deployed ocp4_workload_openshift_virtualization_install_virtctl: true diff --git a/roles/ocp4_workload_openshift_virtualization/readme.adoc b/roles/ocp4_workload_openshift_virtualization/readme.adoc index 2cd32548..a5da12d5 100644 --- a/roles/ocp4_workload_openshift_virtualization/readme.adoc +++ b/roles/ocp4_workload_openshift_virtualization/readme.adoc @@ -1,3 +1,18 @@ = ocp4_workload_openshift_virtualization - Deploy OpenShift Virtualization to an OpenShift Cluster See defaults/main.yml for variables to set. + +== HyperConverged readiness + +The role retries HyperConverged API queries through temporary conversion-webhook +or API outages, such as those caused by node reboots during a MachineConfig rollout. +Readiness requires `Available=True`, `Progressing=False`, and `Degraded=False`. +Missing resources or conditions do not count as ready. Persistent errors or an +unhealthy HyperConverged instance fail the task when its retry budget is exhausted. + +* `ocp4_workload_openshift_virtualization_hco_wait_retries`: `120` +* `ocp4_workload_openshift_virtualization_hco_wait_delay`: `10` seconds + +The default retry budget is approximately 20 minutes, plus API request duration. +This role does not apply MachineConfigs or sequence external MachineConfig changes; +callers managing such changes should wait for their rollout where appropriate. diff --git a/roles/ocp4_workload_openshift_virtualization/tasks/workload.yml b/roles/ocp4_workload_openshift_virtualization/tasks/workload.yml index 4d70e97c..3d94a4f2 100644 --- a/roles/ocp4_workload_openshift_virtualization/tasks/workload.yml +++ b/roles/ocp4_workload_openshift_virtualization/tasks/workload.yml @@ -42,13 +42,27 @@ kind: HyperConverged name: kubevirt-hyperconverged namespace: openshift-cnv - wait: true - wait_sleep: 10 - wait_timeout: 1200 - wait_condition: - reason: "ReconcileCompleted" - type: "Available" - status: "True" + # Retry the API request itself: the module's internal wait exits on webhook + # conversion errors, which can occur while operator nodes are rebooting. + register: r_openshift_virtualization_hco + vars: + _hco_conditions: >- + {{ (r_openshift_virtualization_hco.resources | default([]) + | first | default({})).get('status', {}).get('conditions', []) }} + until: + - r_openshift_virtualization_hco is succeeded + - r_openshift_virtualization_hco.resources | default([]) | length == 1 + - >- + _hco_conditions | selectattr('type', 'equalto', 'Available') + | map(attribute='status') | list == ['True'] + - >- + _hco_conditions | selectattr('type', 'equalto', 'Progressing') + | map(attribute='status') | list == ['False'] + - >- + _hco_conditions | selectattr('type', 'equalto', 'Degraded') + | map(attribute='status') | list == ['False'] + retries: "{{ ocp4_workload_openshift_virtualization_hco_wait_retries }}" + delay: "{{ ocp4_workload_openshift_virtualization_hco_wait_delay }}" # For deployments on CNV with external Ceph use PVCs for boot image snapshots # HyperConverged must have been created with .spec.enableCommonBootImageImport: false diff --git a/tests/static/tox.ini b/tests/static/tox.ini index 0908da48..392d6fea 100644 --- a/tests/static/tox.ini +++ b/tests/static/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = yamllint, ansible-lint, cert-manager +envlist = yamllint, ansible-lint, cert-manager, virtualization-readiness skipsdist = True skip_missing_interpreters = True @@ -25,3 +25,11 @@ deps = Jinja2 changedir = {toxinidir}/../.. commands = python -m unittest discover -s tests/unit -p test_cert_manager_azure.py -v + +# Exercise HCO polling with a local API stub, including transient failures. +[testenv:virtualization-readiness] +deps = + ansible-core + PyYAML +changedir = {toxinidir}/../.. +commands = python -m unittest discover -s tests/unit -p test_openshift_virtualization_readiness.py -v diff --git a/tests/unit/test_openshift_virtualization_readiness.py b/tests/unit/test_openshift_virtualization_readiness.py new file mode 100644 index 00000000..889ef339 --- /dev/null +++ b/tests/unit/test_openshift_virtualization_readiness.py @@ -0,0 +1,122 @@ +"""Exercise the production HCO retry task offline using a sequenced API stub.""" +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / 'roles/ocp4_workload_openshift_virtualization' +PREFIX = 'ocp4_workload_openshift_virtualization_' + +# Only the API module is replaced. Ansible evaluates the actual production +# register, vars, until, retries, and delay directives without cluster access. +STUB = ''' +from ansible.module_utils.basic import AnsibleModule +import json +from pathlib import Path + +module = AnsibleModule(argument_spec={ + 'api_version': {'type': 'str'}, 'kind': {'type': 'str'}, + 'name': {'type': 'str'}, 'namespace': {'type': 'str'}, + 'sequence_file': {'type': 'path', 'required': True}, + 'counter_file': {'type': 'path', 'required': True}, +}) +sequence = json.loads(Path(module.params['sequence_file']).read_text()) +counter = Path(module.params['counter_file']) +attempt = int(counter.read_text()) if counter.exists() else 0 +counter.write_text(str(attempt + 1)) +result = sequence[min(attempt, len(sequence) - 1)] +if result.get('failed'): + module.fail_json(**{k: v for k, v in result.items() if k != 'failed'}) +module.exit_json(changed=False, **result) +''' + + +def response(available='True', progressing='False', degraded='False'): + return {'resources': [{'status': {'conditions': [ + {'type': 'Available', 'status': available}, + {'type': 'Progressing', 'status': progressing}, + {'type': 'Degraded', 'status': degraded}, + ]}}]} + + +class HyperConvergedReadinessTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + tasks = yaml.safe_load((ROLE / 'tasks/workload.yml').read_text()) + block = next(t['block'] for t in tasks if t['name'] == 'Deploy and configure HyperConverged') + cls.task = next(t for t in block if t['name'] == 'Wait until HyperConverged is installed') + cls.defaults = yaml.safe_load((ROLE / 'defaults/main.yml').read_text()) + + def run_sequence(self, sequence, success, attempts, retries=2): + with tempfile.TemporaryDirectory(prefix='hco-readiness-test-') as directory: + tmp = Path(directory) + library = tmp / 'library' + library.mkdir() + (library / 'hco_info.py').write_text(STUB) + (tmp / 'sequence.json').write_text(json.dumps(sequence)) + task = copy.deepcopy(self.task) + task['hco_info'] = task.pop('kubernetes.core.k8s_info') + task['hco_info'].update(sequence_file=str(tmp / 'sequence.json'), + counter_file=str(tmp / 'counter')) + variables = dict(self.defaults) + variables.update({PREFIX + 'hco_wait_retries': retries, + PREFIX + 'hco_wait_delay': 0, + 'ansible_python_interpreter': sys.executable}) + play = [dict(hosts='localhost', connection='local', gather_facts=False, + vars=variables, tasks=[task])] + playbook = tmp / 'playbook.yml' + playbook.write_text(yaml.safe_dump(play, sort_keys=False)) + environment = dict(os.environ, ANSIBLE_LIBRARY=str(library), + ANSIBLE_LOCAL_TEMP=str(tmp / 'local'), + ANSIBLE_REMOTE_TEMP=str(tmp / 'remote'), ANSIBLE_NOCOLOR='1') + result = subprocess.run([sys.executable, '-m', 'ansible.cli.playbook', + '-i', 'localhost,', str(playbook)], cwd=tmp, + env=environment, capture_output=True, text=True, timeout=60) + output = result.stdout + result.stderr + self.assertEqual(result.returncode == 0, success, output) + self.assertEqual(int((tmp / 'counter').read_text()), attempts, output) + self.assertNotIn('Error while evaluating conditional', output) + + def test_recovery_after_api_error_and_incomplete_conditions(self): + sequence = [ + {'failed': True, 'msg': '500 conversion webhook: connection refused'}, + {}, {'resources': []}, {'resources': [{}]}, + {'resources': [{'status': {}}]}, + {'resources': [{'status': {'conditions': []}}]}, + response(available='False'), response(progressing='True'), + response(degraded='True'), response(available='Unknown'), response(), + ] + self.run_sequence(sequence, success=True, attempts=len(sequence), retries=len(sequence)) + + def test_unhealthy_or_failed_queries_exhaust_retry_budget(self): + missing_condition = response() + missing_condition['resources'][0]['status']['conditions'].pop() + failed_with_stale_resource = response() + failed_with_stale_resource.update(failed=True, msg='API request failed') + for result in [ + {'failed': True, 'msg': '500 webhook unavailable'}, {}, {'resources': []}, + {'resources': [{}]}, missing_condition, failed_with_stale_resource, + response(available='False'), response(progressing='True'), + response(degraded='True'), response(available='Unknown'), + ]: + with self.subTest(result=result): + self.run_sequence([result], success=False, attempts=3) + + def test_ready_resource_succeeds_without_retry(self): + self.run_sequence([response()], success=True, attempts=1) + + def test_default_retry_budget(self): + self.assertEqual(self.defaults[PREFIX + 'hco_wait_retries'], 120) + self.assertEqual(self.defaults[PREFIX + 'hco_wait_delay'], 10) + self.assertNotIn('wait', self.task['kubernetes.core.k8s_info']) + + +if __name__ == '__main__': + unittest.main()