diff --git a/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml b/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml index 529b5cd31f10c..f78e12bc2d45c 100644 --- a/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml +++ b/ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml @@ -1,3 +1,8 @@ +base_images: + dev-scripts: + name: test + namespace: ocp-kni + tag: dev-scripts binary_build_commands: make build build_root: image_stream_tag: @@ -9,7 +14,8 @@ images: items: - dockerfile_path: Dockerfile.openshift to: openperouter-operator - - dockerfile_path: operator/bundle.Dockerfile.openshift + - context_dir: operator + dockerfile_path: bundle.Dockerfile to: openperouter-operator-bundle - build_args: - name: BASE_IMAGE @@ -19,10 +25,11 @@ images: operator: bundles: - as: operator-bundle - dockerfile_path: operator/bundle.Dockerfile.openshift + context_dir: operator + dockerfile_path: bundle.Dockerfile skip_building_index: true substitutions: - - pullspec: quay.io/redhat-user-workloads/telco-5g-tenant/openperouter-operator-[\d].* + - pullspec: quay.io/openperouter/router:main with: pipeline:openperouter-operator promotion: to: @@ -47,6 +54,23 @@ resources: requests: cpu: 100m memory: 200Mi +tests: +- as: openperouter-e2e-metal + cluster: build05 + steps: + cluster_profile: equinix-ocp-metal + env: + DEVSCRIPTS_CONFIG: | + IP_STACK=v4v6 + NETWORK_TYPE=OVNKubernetes + EXTRA_NETWORK_NAMES="toswitch1 toswitch2" + TOSWITCH1_NETWORK_SUBNET_V4='192.168.11.0/24' + TOSWITCH1_NETWORK_SUBNET_V6='2001:db8:11::/64' + TOSWITCH2_NETWORK_SUBNET_V4='192.168.12.0/24' + TOSWITCH2_NETWORK_SUBNET_V6='2001:db8:12::/64' + NUM_WORKERS=2 + ENABLE_LOCAL_REGISTRY=true + workflow: baremetalds-openperouter-e2e zz_generated_metadata: branch: main org: openshift-kni diff --git a/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml b/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml index e7cd5b88239fb..c3f5699420109 100644 --- a/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml @@ -111,3 +111,86 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )images,?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build05 + context: ci/prow/openperouter-e2e-metal + decorate: true + decoration_config: {} + labels: + ci-operator.openshift.io/cloud: equinix-ocp-metal + ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal + ci-operator.openshift.io/cluster: build05 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-kni-openperouter-main-openperouter-e2e-metal + path_alias: github.com/openperouter/openperouter + rerun_command: /test openperouter-e2e-metal + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=openperouter-e2e-metal + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )openperouter-e2e-metal,?($|\s.*) diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS new file mode 100644 index 0000000000000..d90ed11dd94d7 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS @@ -0,0 +1,10 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json new file mode 100644 index 0000000000000..bcac85f36d446 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml new file mode 100644 index 0000000000000..1a766fa5ad8db --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml @@ -0,0 +1,20 @@ +workflow: + as: baremetalds-openperouter-e2e + steps: + pre: + - ref: ofcir-acquire + - ref: ipi-install-rbac + - ref: baremetalds-devscripts-ibm + - ref: baremetalds-devscripts-proxy + - ref: ipi-install-hosted-loki + - ref: rhcos-conf-osstream + - ref: baremetalds-openperouter-e2e-iptables + - ref: baremetalds-devscripts-setup + test: + - ref: baremetalds-openperouter-e2e-test + post: + - chain: baremetalds-ofcir-post + documentation: |- + Deploy OpenPerOuter on a baremetal OCP cluster and verify all components + come up healthy. Uses ip6tables REJECT rules to prevent bootstrap gather + from hanging on extra network IPv6 addresses. diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS new file mode 100644 index 0000000000000..d90ed11dd94d7 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS @@ -0,0 +1,10 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh new file mode 100644 index 0000000000000..74be172fa8e2c --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "************ openperouter iptables fix for bootstrap gather ************" + +# shellcheck source=/dev/null +source "${SHARED_DIR}/packet-conf.sh" + +# Add ip6tables REJECT rules on the dev-scripts host for extra network subnets. +# The installer's bootstrap gather tries to SSH to every NIC IP on every node. +# IPv6 addresses on the extra networks (toswitch1/toswitch2) are unreachable +# via SSH, causing ~10min timeout per address. REJECT returns immediate RST +# instead of dropping packets, turning the timeout into an instant failure. +ssh "${SSHOPTS[@]}" "root@${IP}" bash -s << 'EOFIPTABLES' +set -euo pipefail +iptables -I OUTPUT -d 192.168.11.0/24 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +iptables -I OUTPUT -d 192.168.12.0/24 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +ip6tables -I OUTPUT -d 2001:db8:11::/64 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +ip6tables -I OUTPUT -d 2001:db8:12::/64 -p tcp --dport 22 -j REJECT --reject-with tcp-reset +echo "Added iptables REJECT rules for extra network subnets (IPv4 + IPv6)" +iptables -L OUTPUT -n | head -5 +ip6tables -L OUTPUT -n | head -5 +EOFIPTABLES diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json new file mode 100644 index 0000000000000..82f34e850ad9a --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml new file mode 100644 index 0000000000000..652ccd2f307af --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml @@ -0,0 +1,13 @@ +ref: + as: baremetalds-openperouter-e2e-iptables + from: dev-scripts + commands: baremetalds-openperouter-e2e-iptables-commands.sh + timeout: 300s + resources: + requests: + cpu: 100m + memory: 200Mi + documentation: |- + Add ip6tables REJECT rules on the dev-scripts host for extra network subnets. + This prevents the bootstrap gather from hanging on SSH timeouts to IPv6 + addresses on toswitch1/toswitch2 networks. diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS new file mode 100644 index 0000000000000..9827a2117fb39 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS @@ -0,0 +1,11 @@ +approvers: +- fedepaol +- maiqueb +- oribon +- zeeke +reviewers: +- fedepaol +- maiqueb +- oribon +- zeeke + diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh new file mode 100644 index 0000000000000..242c283435ca0 --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +echo "************ openperouter deploy-verify test ************" + +# shellcheck source=/dev/null +source "${SHARED_DIR}/packet-conf.sh" + +echo "### Copying openperouter PR source to remote host" +OPENPEROUTER_SRC="/go/src/github.com/openperouter/openperouter" +ssh "${SSHOPTS[@]}" "root@${IP}" "mkdir -p /root/openperouter" +scp "${SSHOPTS[@]}" "${OPENPEROUTER_SRC}/Makefile" "root@${IP}:/root/openperouter/" +scp "${SSHOPTS[@]}" -r "${OPENPEROUTER_SRC}/e2etests" "root@${IP}:/root/openperouter/" + +echo "### Deploy OpenPerOuter via OLM bundle and verify" +ssh "${SSHOPTS[@]}" "root@${IP}" bash -s -- "${OO_BUNDLE}" << 'EOFDEPLOY' +set -euo pipefail +OO_BUNDLE="$1" +export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig + +# Install operator-sdk via Makefile (uses PR source) +cd /root/openperouter +# .git not copied (submodule symlinks break scp), but make needs a git repo +git init +make operator-sdk +OPERATOR_SDK=$(pwd)/_cache/operator-sdk +${OPERATOR_SDK} version + +# Create and configure namespace +oc create namespace openshift-openperouter-system +oc label --overwrite ns openshift-openperouter-system \ + pod-security.kubernetes.io/enforce=privileged \ + pod-security.kubernetes.io/audit=privileged \ + pod-security.kubernetes.io/warn=privileged + +# Deploy via OLM bundle +${OPERATOR_SDK} run bundle -n openshift-openperouter-system "${OO_BUNDLE}" --timeout 5m + +# Wait for all operator deployments (operator + webhook) +oc wait --for condition=Available -n openshift-openperouter-system \ + deployment --all --timeout=300s + +# Create OpenPERouter CR +cat <<'EOF' | oc apply -f - +apiVersion: network.openperouter.io/v1alpha1 +kind: OpenPERouter +metadata: + name: openperouter + namespace: openshift-openperouter-system +spec: + logLevel: debug +EOF + +# Wait for controller and router daemonsets to be created and rolled out +for ds in controller router; do + echo "Waiting for daemonset $ds to be created..." + deadline=$((SECONDS + 300)) + until oc get daemonset "$ds" -n openshift-openperouter-system &>/dev/null; do + if (( SECONDS >= deadline )); then + echo "ERROR: Timed out waiting for daemonset $ds" + exit 1 + fi + sleep 5 + done + oc rollout status daemonset/"$ds" -n openshift-openperouter-system --timeout=300s +done + +echo "=== Deploy verification ===" +oc get pods -n openshift-openperouter-system -o wide +oc get daemonset -n openshift-openperouter-system + +# Verify all pods are Running and Ready +NOT_READY=$(oc get pods -n openshift-openperouter-system --no-headers | grep -v "Completed" | grep -v "1/1\|2/2\|3/3\|4/4\|5/5" || true) +if [ -n "$NOT_READY" ]; then + echo "ERROR: Some pods are not fully ready:" + echo "$NOT_READY" + exit 1 +fi + +echo "All openperouter pods are running and ready" +EOFDEPLOY diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json new file mode 100644 index 0000000000000..c17b6fc2b2caa --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml", + "owners": { + "approvers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ], + "reviewers": [ + "fedepaol", + "maiqueb", + "oribon", + "zeeke" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml new file mode 100644 index 0000000000000..2ffdc2fc814ad --- /dev/null +++ b/ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml @@ -0,0 +1,16 @@ +ref: + as: baremetalds-openperouter-e2e-test + from: src + grace_period: 10m + commands: baremetalds-openperouter-e2e-test-commands.sh + timeout: 3600s + dependencies: + - env: OO_BUNDLE + name: operator-bundle + resources: + requests: + cpu: 100m + memory: 200Mi + documentation: |- + Deploy OpenPerOuter via OLM bundle, create CR, and verify that operator, + controller, and router daemonsets come up healthy.