From 6aca17669ce1e0252d4768c9d1d7406645727d44 Mon Sep 17 00:00:00 2001 From: Ori Braunshtein Date: Mon, 10 Aug 2026 09:06:16 +0300 Subject: [PATCH] feat: add OpenPERouter baremetal E2E deploy verify lane Baseline for the openperouter-e2e-metal test. Currently deploys the operator via OLM bundle on a baremetal OCP cluster and verifies all components (operator, webhook, controller, router) come up healthy. The full E2E lane will extend this with containerlab fabric setup and test execution. The bundle image is now built using bundle.Dockerfile rather than bundle.Dockerfile.openshift (like the other telco-5g projects using konflux here). The workflow inserts an iptables pre step before devscripts-setup. EXTRA_NETWORK_NAMES creates additional NICs on each node. The installer's bootstrap gather collects ALL NIC IPs and SSHes to each for log gathering. Since the extra networks are unreachable via SSH, this avoids timeouts on attempts to gather them. This is the reason we unpacked the dev-scripts chain instead of calling it directly. Co-Authored-By: Claude Opus 4.6 (1M context) Signed-off-by: Ori Braunshtein --- .../openshift-kni-openperouter-main.yaml | 30 ++++++- ...hift-kni-openperouter-main-presubmits.yaml | 83 ++++++++++++++++++ .../baremetalds/openperouter-e2e/OWNERS | 10 +++ ...ds-openperouter-e2e-workflow.metadata.json | 17 ++++ ...baremetalds-openperouter-e2e-workflow.yaml | 20 +++++ .../openperouter-e2e/iptables/OWNERS | 10 +++ ...alds-openperouter-e2e-iptables-commands.sh | 26 ++++++ ...penperouter-e2e-iptables-ref.metadata.json | 17 ++++ ...metalds-openperouter-e2e-iptables-ref.yaml | 13 +++ .../baremetalds/openperouter-e2e/test/OWNERS | 11 +++ ...emetalds-openperouter-e2e-test-commands.sh | 84 +++++++++++++++++++ ...ds-openperouter-e2e-test-ref.metadata.json | 17 ++++ ...baremetalds-openperouter-e2e-test-ref.yaml | 16 ++++ 13 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json create mode 100644 ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml diff --git a/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml b/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml index 529b5cd31f10c..f78e12bc2d45c 100644 --- a/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml +++ b/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml @@ -1,3 +1,8 @@ +base_images: + dev-scripts: + name: test + namespace: ocp-kni + tag: dev-scripts binary_build_commands: make build build_root: image_stream_tag: @@ -9,7 +14,8 @@ images: items: - dockerfile_path: Dockerfile.openshift to: openperouter-operator - - dockerfile_path: operator/bundle.Dockerfile.openshift + - context_dir: operator + dockerfile_path: bundle.Dockerfile to: openperouter-operator-bundle - build_args: - name: BASE_IMAGE @@ -19,10 +25,11 @@ images: operator: bundles: - as: operator-bundle - dockerfile_path: operator/bundle.Dockerfile.openshift + context_dir: operator + dockerfile_path: bundle.Dockerfile skip_building_index: true substitutions: - - pullspec: quay.io/redhat-user-workloads/telco-5g-tenant/openperouter-operator-[\d].* + - pullspec: quay.io/openperouter/router:main with: pipeline:openperouter-operator promotion: to: @@ -47,6 +54,23 @@ resources: requests: cpu: 100m memory: 200Mi +tests: +- as: openperouter-e2e-metal + cluster: build05 + steps: + cluster_profile: equinix-ocp-metal + env: + DEVSCRIPTS_CONFIG: | + IP_STACK=v4v6 + NETWORK_TYPE=OVNKubernetes + EXTRA_NETWORK_NAMES="toswitch1 toswitch2" + TOSWITCH1_NETWORK_SUBNET_V4='192.168.11.0/24' + TOSWITCH1_NETWORK_SUBNET_V6='2001:db8:11::/64' + TOSWITCH2_NETWORK_SUBNET_V4='192.168.12.0/24' + TOSWITCH2_NETWORK_SUBNET_V6='2001:db8:12::/64' + NUM_WORKERS=2 + ENABLE_LOCAL_REGISTRY=true + workflow: baremetalds-openperouter-e2e zz_generated_metadata: branch: main org: openshift-kni diff --git a/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml b/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml index e7cd5b88239fb..c3f5699420109 100644 --- a/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml @@ -111,3 +111,86 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build05 + context: ci/prow/openperouter-e2e-metal + decorate: true + decoration_config: {} + labels: + ci-operator.openshift.io/cloud: equinix-ocp-metal + ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal + ci-operator.openshift.io/cluster: build05 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-kni-openperouter-main-openperouter-e2e-metal + path_alias: github.com/openperouter/openperouter + rerun_command: /test openperouter-e2e-metal + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=openperouter-e2e-metal + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )openperouter-e2e-metal,?($|\s.*) diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS new file mode 100644 index 0000000000000..d90ed11dd94d7 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS @@ -0,0 +1,10 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json new file mode 100644 index 0000000000000..bcac85f36d446 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml new file mode 100644 index 0000000000000..1a766fa5ad8db --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml @@ -0,0 +1,20 @@ +workflow: + as: baremetalds-openperouter-e2e + steps: + pre: + - ref: ofcir-acquire + - ref: ipi-install-rbac + - ref: baremetalds-devscripts-ibm + - ref: baremetalds-devscripts-proxy + - ref: ipi-install-hosted-loki + - ref: rhcos-conf-osstream + - ref: baremetalds-openperouter-e2e-iptables + - ref: baremetalds-devscripts-setup + test: + - ref: baremetalds-openperouter-e2e-test + post: + - chain: baremetalds-ofcir-post + documentation: |- + Deploy OpenPerOuter on a baremetal OCP cluster and verify all components + come up healthy. Uses ip6tables REJECT rules to prevent bootstrap gather + from hanging on extra network IPv6 addresses. diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS new file mode 100644 index 0000000000000..d90ed11dd94d7 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS @@ -0,0 +1,10 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh new file mode 100644 index 0000000000000..74be172fa8e2c --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "************ openperouter iptables fix for bootstrap gather ************" + +# shellcheck source=/dev/null +source "${SHARED_DIR}/packet-conf.sh" + +# Add ip6tables REJECT rules on the dev-scripts host for extra network subnets. +# The installer's bootstrap gather tries to SSH to every NIC IP on every node. +# IPv6 addresses on the extra networks (toswitch1/toswitch2) are unreachable +# via SSH, causing ~10min timeout per address. REJECT returns immediate RST +# instead of dropping packets, turning the timeout into an instant failure. +ssh "${SSHOPTS[@]}" "root@${IP}" bash -s << 'EOFIPTABLES' +set -euo pipefail +iptables -I OUTPUT -d 192.168.11.0/24 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +iptables -I OUTPUT -d 192.168.12.0/24 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +ip6tables -I OUTPUT -d 2001:db8:11::/64 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +ip6tables -I OUTPUT -d 2001:db8:12::/64 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +echo "Added iptables REJECT rules for extra network subnets (IPv4 + IPv6)" +iptables -L OUTPUT -n | head -5 +ip6tables -L OUTPUT -n | head -5 +EOFIPTABLES diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json new file mode 100644 index 0000000000000..82f34e850ad9a --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml new file mode 100644 index 0000000000000..652ccd2f307af --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml @@ -0,0 +1,13 @@ +ref: + as: baremetalds-openperouter-e2e-iptables + from: dev-scripts + commands: baremetalds-openperouter-e2e-iptables-commands.sh + timeout: 300s + resources: + requests: + cpu: 100m + memory: 200Mi + documentation: |- + Add ip6tables REJECT rules on the dev-scripts host for extra network subnets. + This prevents the bootstrap gather from hanging on SSH timeouts to IPv6 + addresses on toswitch1/toswitch2 networks. diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS new file mode 100644 index 0000000000000..9827a2117fb39 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS @@ -0,0 +1,11 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke + diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh new file mode 100644 index 0000000000000..242c283435ca0 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "************ openperouter deploy-verify test ************" + +# shellcheck source=/dev/null +source "${SHARED_DIR}/packet-conf.sh" + +echo "### Copying openperouter PR source to remote host" +OPENPEROUTER_SRC="/go/src/github.com/openperouter/openperouter" +ssh "${SSHOPTS[@]}" "root@${IP}" "mkdir -p /root/openperouter" +scp "${SSHOPTS[@]}" "${OPENPEROUTER_SRC}/Makefile" "root@${IP}:/root/openperouter/" +scp "${SSHOPTS[@]}" -r "${OPENPEROUTER_SRC}/e2etests" "root@${IP}:/root/openperouter/" + +echo "### Deploy OpenPerOuter via OLM bundle and verify" +ssh "${SSHOPTS[@]}" "root@${IP}" bash -s -- "${OO_BUNDLE}" << 'EOFDEPLOY' +set -euo pipefail +OO_BUNDLE="$1" +export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig + +# Install operator-sdk via Makefile (uses PR source) +cd /root/openperouter +# .git not copied (submodule symlinks break scp), but make needs a git repo +git init +make operator-sdk +OPERATOR_SDK=$(pwd)/_cache/operator-sdk +${OPERATOR_SDK} version + +# Create and configure namespace +oc create namespace openshift-openperouter-system +oc label --overwrite ns openshift-openperouter-system \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/audit=privileged \ + pod-security.kubernetes.io/warn=privileged + +# Deploy via OLM bundle +${OPERATOR_SDK} run bundle -n openshift-openperouter-system "${OO_BUNDLE}" --timeout 5m + +# Wait for all operator deployments (operator + webhook) +oc wait --for condition=Available -n openshift-openperouter-system \ + deployment --all --timeout=300s + +# Create OpenPERouter CR +cat <<'EOF' | oc apply -f - +apiVersion: network.openperouter.io/v1alpha1 +kind: OpenPERouter +metadata: + name: openperouter + namespace: openshift-openperouter-system +spec: + logLevel: debug +EOF + +# Wait for controller and router daemonsets to be created and rolled out +for ds in controller router; do + echo "Waiting for daemonset $ds to be created..." + deadline=$((SECONDS + 300)) + until oc get daemonset "$ds" -n openshift-openperouter-system &>/dev/null; do + if (( SECONDS >= deadline )); then + echo "ERROR: Timed out waiting for daemonset $ds" + exit 1 + fi + sleep 5 + done + oc rollout status daemonset/"$ds" -n openshift-openperouter-system --timeout=300s +done + +echo "=== Deploy verification ===" +oc get pods -n openshift-openperouter-system -o wide +oc get daemonset -n openshift-openperouter-system + +# Verify all pods are Running and Ready +NOT_READY=$(oc get pods -n openshift-openperouter-system --no-headers | grep -v "Completed" | grep -v "1/1\|2/2\|3/3\|4/4\|5/5" || true) +if [ -n "$NOT_READY" ]; then + echo "ERROR: Some pods are not fully ready:" + echo "$NOT_READY" + exit 1 +fi + +echo "All openperouter pods are running and ready" +EOFDEPLOY diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json new file mode 100644 index 0000000000000..c17b6fc2b2caa --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml new file mode 100644 index 0000000000000..2ffdc2fc814ad --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml @@ -0,0 +1,16 @@ +ref: + as: baremetalds-openperouter-e2e-test + from: src + grace_period: 10m + commands: baremetalds-openperouter-e2e-test-commands.sh + timeout: 3600s + dependencies: + - env: OO_BUNDLE + name: operator-bundle + resources: + requests: + cpu: 100m + memory: 200Mi + documentation: |- + Deploy OpenPerOuter via OLM bundle, create CR, and verify that operator, + controller, and router daemonsets come up healthy.