Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ ocp4_workload_openshift_virtualization_enabled_nested_kvm: false
# Only set to false if a lab needs students to create it
ocp4_workload_openshift_virtualization_deploy_hyperconverged: true

# Retry HCO readiness queries through temporary API/webhook outages during node
# rollouts. Defaults allow approximately 20 minutes, plus API request duration.
ocp4_workload_openshift_virtualization_hco_wait_retries: 120
ocp4_workload_openshift_virtualization_hco_wait_delay: 10

# Install virtctl - requires hyperconverged to be deployed
ocp4_workload_openshift_virtualization_install_virtctl: true

Expand Down
15 changes: 15 additions & 0 deletions roles/ocp4_workload_openshift_virtualization/readme.adoc
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
= ocp4_workload_openshift_virtualization - Deploy OpenShift Virtualization to an OpenShift Cluster

See defaults/main.yml for variables to set.

== HyperConverged readiness

The role retries HyperConverged API queries through temporary conversion-webhook
or API outages, such as those caused by node reboots during a MachineConfig rollout.
Readiness requires `Available=True`, `Progressing=False`, and `Degraded=False`.
Missing resources or conditions do not count as ready. Persistent errors or an
unhealthy HyperConverged instance fail the task when its retry budget is exhausted.

* `ocp4_workload_openshift_virtualization_hco_wait_retries`: `120`
* `ocp4_workload_openshift_virtualization_hco_wait_delay`: `10` seconds

The default retry budget is approximately 20 minutes, plus API request duration.
This role does not apply MachineConfigs or sequence external MachineConfig changes;
callers managing such changes should wait for their rollout where appropriate.
28 changes: 21 additions & 7 deletions roles/ocp4_workload_openshift_virtualization/tasks/workload.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,27 @@
kind: HyperConverged
name: kubevirt-hyperconverged
namespace: openshift-cnv
wait: true
wait_sleep: 10
wait_timeout: 1200
wait_condition:
reason: "ReconcileCompleted"
type: "Available"
status: "True"
# Retry the API request itself: the module's internal wait exits on webhook
# conversion errors, which can occur while operator nodes are rebooting.
register: r_openshift_virtualization_hco
vars:
_hco_conditions: >-
{{ (r_openshift_virtualization_hco.resources | default([])
| first | default({})).get('status', {}).get('conditions', []) }}
until:
- r_openshift_virtualization_hco is succeeded
- r_openshift_virtualization_hco.resources | default([]) | length == 1
- >-
_hco_conditions | selectattr('type', 'equalto', 'Available')
| map(attribute='status') | list == ['True']
- >-
_hco_conditions | selectattr('type', 'equalto', 'Progressing')
| map(attribute='status') | list == ['False']
- >-
_hco_conditions | selectattr('type', 'equalto', 'Degraded')
| map(attribute='status') | list == ['False']
retries: "{{ ocp4_workload_openshift_virtualization_hco_wait_retries }}"
delay: "{{ ocp4_workload_openshift_virtualization_hco_wait_delay }}"

# For deployments on CNV with external Ceph use PVCs for boot image snapshots
# HyperConverged must have been created with .spec.enableCommonBootImageImport: false
Expand Down
10 changes: 9 additions & 1 deletion tests/static/tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = yamllint, ansible-lint, cert-manager
envlist = yamllint, ansible-lint, cert-manager, virtualization-readiness
skipsdist = True
skip_missing_interpreters = True

Expand All @@ -25,3 +25,11 @@ deps =
Jinja2
changedir = {toxinidir}/../..
commands = python -m unittest discover -s tests/unit -p test_cert_manager_azure.py -v

# Exercise HCO polling with a local API stub, including transient failures.
[testenv:virtualization-readiness]
deps =
ansible-core
PyYAML
changedir = {toxinidir}/../..
commands = python -m unittest discover -s tests/unit -p test_openshift_virtualization_readiness.py -v
122 changes: 122 additions & 0 deletions tests/unit/test_openshift_virtualization_readiness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
"""Exercise the production HCO retry task offline using a sequenced API stub."""
import copy
import json
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest

import yaml

ROOT = Path(__file__).resolve().parents[2]
ROLE = ROOT / 'roles/ocp4_workload_openshift_virtualization'
PREFIX = 'ocp4_workload_openshift_virtualization_'

# Only the API module is replaced. Ansible evaluates the actual production
# register, vars, until, retries, and delay directives without cluster access.
STUB = '''
from ansible.module_utils.basic import AnsibleModule
import json
from pathlib import Path

module = AnsibleModule(argument_spec={
'api_version': {'type': 'str'}, 'kind': {'type': 'str'},
'name': {'type': 'str'}, 'namespace': {'type': 'str'},
'sequence_file': {'type': 'path', 'required': True},
'counter_file': {'type': 'path', 'required': True},
})
sequence = json.loads(Path(module.params['sequence_file']).read_text())
counter = Path(module.params['counter_file'])
attempt = int(counter.read_text()) if counter.exists() else 0
counter.write_text(str(attempt + 1))
result = sequence[min(attempt, len(sequence) - 1)]
if result.get('failed'):
module.fail_json(**{k: v for k, v in result.items() if k != 'failed'})
module.exit_json(changed=False, **result)
'''


def response(available='True', progressing='False', degraded='False'):
return {'resources': [{'status': {'conditions': [
{'type': 'Available', 'status': available},
{'type': 'Progressing', 'status': progressing},
{'type': 'Degraded', 'status': degraded},
]}}]}


class HyperConvergedReadinessTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
tasks = yaml.safe_load((ROLE / 'tasks/workload.yml').read_text())
block = next(t['block'] for t in tasks if t['name'] == 'Deploy and configure HyperConverged')
cls.task = next(t for t in block if t['name'] == 'Wait until HyperConverged is installed')
cls.defaults = yaml.safe_load((ROLE / 'defaults/main.yml').read_text())

def run_sequence(self, sequence, success, attempts, retries=2):
with tempfile.TemporaryDirectory(prefix='hco-readiness-test-') as directory:
tmp = Path(directory)
library = tmp / 'library'
library.mkdir()
(library / 'hco_info.py').write_text(STUB)
(tmp / 'sequence.json').write_text(json.dumps(sequence))
task = copy.deepcopy(self.task)
task['hco_info'] = task.pop('kubernetes.core.k8s_info')
task['hco_info'].update(sequence_file=str(tmp / 'sequence.json'),
counter_file=str(tmp / 'counter'))
variables = dict(self.defaults)
variables.update({PREFIX + 'hco_wait_retries': retries,
PREFIX + 'hco_wait_delay': 0,
'ansible_python_interpreter': sys.executable})
play = [dict(hosts='localhost', connection='local', gather_facts=False,
vars=variables, tasks=[task])]
playbook = tmp / 'playbook.yml'
playbook.write_text(yaml.safe_dump(play, sort_keys=False))
environment = dict(os.environ, ANSIBLE_LIBRARY=str(library),
ANSIBLE_LOCAL_TEMP=str(tmp / 'local'),
ANSIBLE_REMOTE_TEMP=str(tmp / 'remote'), ANSIBLE_NOCOLOR='1')
result = subprocess.run([sys.executable, '-m', 'ansible.cli.playbook',
'-i', 'localhost,', str(playbook)], cwd=tmp,
env=environment, capture_output=True, text=True, timeout=60)
output = result.stdout + result.stderr
self.assertEqual(result.returncode == 0, success, output)
self.assertEqual(int((tmp / 'counter').read_text()), attempts, output)
self.assertNotIn('Error while evaluating conditional', output)

def test_recovery_after_api_error_and_incomplete_conditions(self):
sequence = [
{'failed': True, 'msg': '500 conversion webhook: connection refused'},
{}, {'resources': []}, {'resources': [{}]},
{'resources': [{'status': {}}]},
{'resources': [{'status': {'conditions': []}}]},
response(available='False'), response(progressing='True'),
response(degraded='True'), response(available='Unknown'), response(),
]
self.run_sequence(sequence, success=True, attempts=len(sequence), retries=len(sequence))

def test_unhealthy_or_failed_queries_exhaust_retry_budget(self):
missing_condition = response()
missing_condition['resources'][0]['status']['conditions'].pop()
failed_with_stale_resource = response()
failed_with_stale_resource.update(failed=True, msg='API request failed')
for result in [
{'failed': True, 'msg': '500 webhook unavailable'}, {}, {'resources': []},
{'resources': [{}]}, missing_condition, failed_with_stale_resource,
response(available='False'), response(progressing='True'),
response(degraded='True'), response(available='Unknown'),
]:
with self.subTest(result=result):
self.run_sequence([result], success=False, attempts=3)

def test_ready_resource_succeeds_without_retry(self):
self.run_sequence([response()], success=True, attempts=1)

def test_default_retry_budget(self):
self.assertEqual(self.defaults[PREFIX + 'hco_wait_retries'], 120)
self.assertEqual(self.defaults[PREFIX + 'hco_wait_delay'], 10)
self.assertNotIn('wait', self.task['kubernetes.core.k8s_info'])


if __name__ == '__main__':
unittest.main()
Loading