WIP: add OpenPERouter baremetal E2E deploy-verify lane

ci-operator/config/openshift-kni/openperouter/openshift-kni-openperouter-main.yaml

@@ -1,3 +1,8 @@
+base_images:
+  dev-scripts:
+    name: test
+    namespace: ocp-kni
+    tag: dev-scripts
 binary_build_commands: make build
 build_root:
   image_stream_tag:
@@ -9,7 +14,8 @@ images:
   items:
   - dockerfile_path: Dockerfile.openshift
     to: openperouter-operator
-  - dockerfile_path: operator/bundle.Dockerfile.openshift
+  - context_dir: operator
+    dockerfile_path: bundle.Dockerfile
     to: openperouter-operator-bundle
   - build_args:
     - name: BASE_IMAGE
@@ -19,10 +25,11 @@ images:
 operator:
   bundles:
   - as: operator-bundle
-    dockerfile_path: operator/bundle.Dockerfile.openshift
+    context_dir: operator
+    dockerfile_path: bundle.Dockerfile
     skip_building_index: true
   substitutions:
-  - pullspec: quay.io/redhat-user-workloads/telco-5g-tenant/openperouter-operator-[\d].*
+  - pullspec: quay.io/openperouter/router:main
     with: pipeline:openperouter-operator
 promotion:
   to:
@@ -47,6 +54,23 @@ resources:
     requests:
       cpu: 100m
       memory: 200Mi
+tests:
+- as: openperouter-e2e-metal
+  cluster: build05
+  steps:
+    cluster_profile: equinix-ocp-metal
+    env:
+      DEVSCRIPTS_CONFIG: |
+        IP_STACK=v4v6
+        NETWORK_TYPE=OVNKubernetes
+        EXTRA_NETWORK_NAMES="toswitch1 toswitch2"
+        TOSWITCH1_NETWORK_SUBNET_V4='192.168.11.0/24'
+        TOSWITCH1_NETWORK_SUBNET_V6='2001:db8:11::/64'
+        TOSWITCH2_NETWORK_SUBNET_V4='192.168.12.0/24'
+        TOSWITCH2_NETWORK_SUBNET_V6='2001:db8:12::/64'
+        NUM_WORKERS=2
+        ENABLE_LOCAL_REGISTRY=true
+    workflow: baremetalds-openperouter-e2e
 zz_generated_metadata:
   branch: main
   org: openshift-kni

ci-operator/jobs/openshift-kni/openperouter/openshift-kni-openperouter-main-presubmits.yaml

@@ -111,3 +111,86 @@ presubmits:
         secret:
           secretName: result-aggregator
     trigger: (?m)^/test( | .* )images,?($|\s.*)
+  - agent: kubernetes
+    always_run: true
+    branches:
+    - ^main$
+    - ^main-
+    cluster: build05
+    context: ci/prow/openperouter-e2e-metal
+    decorate: true
+    decoration_config: {}
+    labels:
+      ci-operator.openshift.io/cloud: equinix-ocp-metal
+      ci-operator.openshift.io/cloud-cluster-profile: equinix-ocp-metal
+      ci-operator.openshift.io/cluster: build05
+      ci.openshift.io/generator: prowgen
+      pj-rehearse.openshift.io/can-be-rehearsed: "true"
+    name: pull-ci-openshift-kni-openperouter-main-openperouter-e2e-metal
+    path_alias: github.com/openperouter/openperouter
+    rerun_command: /test openperouter-e2e-metal
+    spec:
+      containers:
+      - args:
+        - --gcs-upload-secret=/secrets/gcs/service-account.json
+        - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+        - --lease-server-credentials-file=/etc/boskos/credentials
+        - --report-credentials-file=/etc/report/credentials
+        - --secret-dir=/secrets/ci-pull-credentials
+        - --target=openperouter-e2e-metal
+        command:
+        - ci-operator
+        env:
+        - name: HTTP_SERVER_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.podIP
+        image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest
+        imagePullPolicy: Always
+        name: ""
+        ports:
+        - containerPort: 8080
+          name: http
+        resources:
+          requests:
+            cpu: 10m
+        volumeMounts:
+        - mountPath: /etc/boskos
+          name: boskos
+          readOnly: true
+        - mountPath: /secrets/ci-pull-credentials
+          name: ci-pull-credentials
+          readOnly: true
+        - mountPath: /secrets/gcs
+          name: gcs-credentials
+          readOnly: true
+        - mountPath: /secrets/manifest-tool
+          name: manifest-tool-local-pusher
+          readOnly: true
+        - mountPath: /etc/pull-secret
+          name: pull-secret
+          readOnly: true
+        - mountPath: /etc/report
+          name: result-aggregator
+          readOnly: true
+      serviceAccountName: ci-operator
+      volumes:
+      - name: boskos
+        secret:
+          items:
+          - key: credentials
+            path: credentials
+          secretName: boskos-credentials
+      - name: ci-pull-credentials
+        secret:
+          secretName: ci-pull-credentials
+      - name: manifest-tool-local-pusher
+        secret:
+          secretName: manifest-tool-local-pusher
+      - name: pull-secret
+        secret:
+          secretName: registry-pull-credentials
+      - name: result-aggregator
+        secret:
+          secretName: result-aggregator
+    trigger: (?m)^/test( | .* )openperouter-e2e-metal,?($|\s.*)

ci-operator/step-registry/baremetalds/openperouter-e2e/OWNERS

@@ -0,0 +1,10 @@
+approvers:
+- fedepaol
+- maiqueb
+- oribon
+- zeeke
+reviewers:
+- fedepaol
+- maiqueb
+- oribon
+- zeeke

ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml",
+	"owners": {
+		"approvers": [
+			"fedepaol",
+			"maiqueb",
+			"oribon",
+			"zeeke"
+		],
+		"reviewers": [
+			"fedepaol",
+			"maiqueb",
+			"oribon",
+			"zeeke"
+		]
+	}
+}

ci-operator/step-registry/baremetalds/openperouter-e2e/baremetalds-openperouter-e2e-workflow.yaml

@@ -0,0 +1,20 @@
+workflow:
+  as: baremetalds-openperouter-e2e
+  steps:
+    pre:
+      - ref: ofcir-acquire
+      - ref: ipi-install-rbac
+      - ref: baremetalds-devscripts-ibm
+      - ref: baremetalds-devscripts-proxy
+      - ref: ipi-install-hosted-loki
+      - ref: rhcos-conf-osstream
+      - ref: baremetalds-openperouter-e2e-iptables
+      - ref: baremetalds-devscripts-setup
+    test:
+      - ref: baremetalds-openperouter-e2e-test
+    post:
+      - chain: baremetalds-ofcir-post
+  documentation: |-
+    Deploy OpenPerOuter on a baremetal OCP cluster and verify all components
+    come up healthy. Uses ip6tables REJECT rules to prevent bootstrap gather
+    from hanging on extra network IPv6 addresses.

ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/OWNERS

@@ -0,0 +1,10 @@
+approvers:
+- fedepaol
+- maiqueb
+- oribon
+- zeeke
+reviewers:
+- fedepaol
+- maiqueb
+- oribon
+- zeeke

ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-commands.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+echo "************ openperouter iptables fix for bootstrap gather ************"
+
+# shellcheck source=/dev/null
+source "${SHARED_DIR}/packet-conf.sh"
+
+# Add ip6tables REJECT rules on the dev-scripts host for extra network subnets.
+# The installer's bootstrap gather tries to SSH to every NIC IP on every node.
+# IPv6 addresses on the extra networks (toswitch1/toswitch2) are unreachable
+# via SSH, causing ~10min timeout per address. REJECT returns immediate RST
+# instead of dropping packets, turning the timeout into an instant failure.
+ssh "${SSHOPTS[@]}" "root@${IP}" bash -s << 'EOFIPTABLES'
+set -euo pipefail
+iptables -I OUTPUT -d 192.168.11.0/24 -p tcp --dport 22 -j REJECT --reject-with tcp-reset
+iptables -I OUTPUT -d 192.168.12.0/24 -p tcp --dport 22 -j REJECT --reject-with tcp-reset
+ip6tables -I OUTPUT -d 2001:db8:11::/64 -p tcp --dport 22 -j REJECT --reject-with tcp-reset
+ip6tables -I OUTPUT -d 2001:db8:12::/64 -p tcp --dport 22 -j REJECT --reject-with tcp-reset
+echo "Added iptables REJECT rules for extra network subnets (IPv4 + IPv6)"
+iptables -L OUTPUT -n | head -5
+ip6tables -L OUTPUT -n | head -5
+EOFIPTABLES

ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml",
+	"owners": {
+		"approvers": [
+			"fedepaol",
+			"maiqueb",
+			"oribon",
+			"zeeke"
+		],
+		"reviewers": [
+			"fedepaol",
+			"maiqueb",
+			"oribon",
+			"zeeke"
+		]
+	}
+}

ci-operator/step-registry/baremetalds/openperouter-e2e/iptables/baremetalds-openperouter-e2e-iptables-ref.yaml

@@ -0,0 +1,13 @@
+ref:
+  as: baremetalds-openperouter-e2e-iptables
+  from: dev-scripts
+  commands: baremetalds-openperouter-e2e-iptables-commands.sh
+  timeout: 300s
+  resources:
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  documentation: |-
+    Add ip6tables REJECT rules on the dev-scripts host for extra network subnets.
+    This prevents the bootstrap gather from hanging on SSH timeouts to IPv6
+    addresses on toswitch1/toswitch2 networks.

ci-operator/step-registry/baremetalds/openperouter-e2e/test/OWNERS

@@ -0,0 +1,11 @@
+approvers:
+- fedepaol
+- maiqueb
+- oribon
+- zeeke
+reviewers:
+- fedepaol
+- maiqueb
+- oribon
+- zeeke
+

ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-commands.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+echo "************ openperouter deploy-verify test ************"
+
+# shellcheck source=/dev/null
+source "${SHARED_DIR}/packet-conf.sh"
+
+echo "### Copying openperouter PR source to remote host"
+scp "${SSHOPTS[@]}" -r "/go/src/github.com/openperouter/openperouter" "root@${IP}:/root/openperouter"
+
+echo "### Deploy OpenPerOuter via OLM bundle and verify"
+ssh "${SSHOPTS[@]}" "root@${IP}" bash -s -- "${OO_BUNDLE}" << 'EOFDEPLOY'
+set -euo pipefail
+OO_BUNDLE="$1"
+export KUBECONFIG=/root/dev-scripts/ocp/ostest/auth/kubeconfig
+
+# Install operator-sdk via Makefile (uses PR source)
+cd /root/openperouter
+make operator-sdk
+OPERATOR_SDK=$(pwd)/_cache/operator-sdk
+${OPERATOR_SDK} version
+
+# Create and configure namespace
+oc create namespace openshift-openperouter-system
+oc label --overwrite ns openshift-openperouter-system \
+  pod-security.kubernetes.io/enforce=privileged \
+  pod-security.kubernetes.io/audit=privileged \
+  pod-security.kubernetes.io/warn=privileged
+
+# Deploy via OLM bundle
+${OPERATOR_SDK} run bundle -n openshift-openperouter-system "${OO_BUNDLE}" --timeout 5m
+
+# Wait for all operator deployments (operator + webhook)
+oc wait --for condition=Available -n openshift-openperouter-system \
+  deployment --all --timeout=300s
+
+# Create OpenPERouter CR
+cat <<'EOF' | oc apply -f -
+apiVersion: network.openperouter.io/v1alpha1
+kind: OpenPERouter
+metadata:
+  name: openperouter
+  namespace: openshift-openperouter-system
+spec:
+  logLevel: debug
+EOF
+
+# Wait for controller and router daemonsets to be created and rolled out
+for ds in controller router; do
+  echo "Waiting for daemonset $ds to be created..."
+  deadline=$((SECONDS + 300))
+  until oc get daemonset "$ds" -n openshift-openperouter-system &>/dev/null; do
+    if (( SECONDS >= deadline )); then
+      echo "ERROR: Timed out waiting for daemonset $ds"
+      exit 1
+    fi
+    sleep 5
+  done
+  oc rollout status daemonset/"$ds" -n openshift-openperouter-system --timeout=300s
+done
+
+echo "=== Deploy verification ==="
+oc get pods -n openshift-openperouter-system -o wide
+oc get daemonset -n openshift-openperouter-system
+
+# Verify all pods are Running and Ready
+NOT_READY=$(oc get pods -n openshift-openperouter-system --no-headers | grep -v "Completed" | grep -v "1/1\|2/2\|3/3\|4/4\|5/5" || true)
+if [ -n "$NOT_READY" ]; then
+  echo "ERROR: Some pods are not fully ready:"
+  echo "$NOT_READY"
+  exit 1
+fi
+
+echo "All openperouter pods are running and ready"
+EOFDEPLOY

ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml",
+	"owners": {
+		"approvers": [
+			"fedepaol",
+			"maiqueb",
+			"oribon",
+			"zeeke"
+		],
+		"reviewers": [
+			"fedepaol",
+			"maiqueb",
+			"oribon",
+			"zeeke"
+		]
+	}
+}

ci-operator/step-registry/baremetalds/openperouter-e2e/test/baremetalds-openperouter-e2e-test-ref.yaml

@@ -0,0 +1,16 @@
+ref:
+  as: baremetalds-openperouter-e2e-test
+  from: src
+  grace_period: 10m
+  commands: baremetalds-openperouter-e2e-test-commands.sh
+  timeout: 3600s
+  dependencies:
+  - env: OO_BUNDLE
+    name: operator-bundle
+  resources:
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  documentation: |-
+    Deploy OpenPerOuter via OLM bundle, create CR, and verify that operator,
+    controller, and router daemonsets come up healthy.